mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
perf(server): stop re-reading and re-decrypting unchanged checkpoint headers on every upgrade readiness probe (#21130)
Cloud repeats the upgrade readiness probe. Every probe loaded, decrypted and parsed the checkpoint header of each active native run and of each retained task session's last run, even when nothing had changed. The statement that reads a run now also reads its header's row revision (xmin, ctid, updated_at, stored size) and returns the material only for a revision the process has not inspected. A bounded memo in the checkpoint owner keeps the four identifiers the answer compares. It reuses an answer only for the same binding, row revision and secrets key, loads the key again for each answer it serves, never remembers a header that failed to decrypt, and forgets a header the latest probe did not inspect. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
1 parent
f2e449f8f6
commit
2174f2dfa5
6 files changed
+783
-24
No files matched your search
@@ -3545,6 +3545,21 @@ An upgrade readiness report contains preliminary native evidence, not permission
|
||||
to remove a deployment volume. Keep the checkpoint pool bounded and measure all
|
||||
pools when qualifying database cost; the current size of two is a default.
|
||||
|
||||
Cloud repeats the upgrade readiness probe, so the probe does not read or decrypt
|
||||
a checkpoint header that has not changed. The statement that reads each run also
|
||||
reads its header's row revision (`xmin`, `ctid`, `updated_at` and stored size)
|
||||
and returns the encrypted material only for a revision the process has not
|
||||
inspected. The memo in `native-runner-checkpoint.ts` keeps four identifiers per
|
||||
header, is bounded in entries and bytes, and forgets a header that the latest
|
||||
probe did not inspect. It loads the secrets key again for each answer it reuses, as a direct read
|
||||
does for each header, and refuses the answer when the key is missing or changed. It
|
||||
never remembers a header that failed to decrypt. Measure a probe with
|
||||
`pnpm --filter @paperclipai/server exec tsx scripts/benchmark-upgrade-readiness.ts --baseline-ref=<pre-fix-commit>`.
|
||||
The harness runs both revisions over the same PostgreSQL rows, reports bytes
|
||||
from the database, decryptions, CPU and sampled allocation per probe, and fails
|
||||
unless both give the same answer after a header changes, stops decrypting or is
|
||||
deleted.
|
||||
|
||||
Committed native Agent Chat turns recover their unanswered final reply through
|
||||
the existing response presentation worker. Recovery verifies the accepted result
|
||||
and final turn, agent, responsible user, native session and conversation generation.
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
// Compares one upgrade readiness probe between a pre-fix revision and the working tree
|
||||
// on the same PostgreSQL rows. See "Native runner restart recovery" in doc/DEVELOPING.md.
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import crypto, { randomBytes, randomUUID } from "node:crypto";
|
||||
import diagnosticsChannel from "node:diagnostics_channel";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { Session } from "node:inspector/promises";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import type { Socket } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { agentTaskSessions, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations,
|
||||
nativeRunnerCheckpoints, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db";
|
||||
import { createLocalEncryptedCipher } from "../src/secrets/local-encrypted-provider.js";
|
||||
import { createAgentLifecycle } from "../src/services/agent-lifecycle.js";
|
||||
import { readUpgradeReadiness } from "../src/services/heartbeat/upgrade-readiness.js";
|
||||
import { currentNativeControllerIdentity } from "../src/services/native-runtime/native-restart-recovery.js";
|
||||
import { DurablePrpControlPlane } from "../src/vendor/paperclip-runner/index.js";
|
||||
|
||||
const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const argument = (name: string) => process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
const baselineRef = argument("baseline-ref");
|
||||
assert.ok(baselineRef, "Pass a pinned pre-fix --baseline-ref");
|
||||
const iterations = Number(argument("iterations") ?? 12);
|
||||
// Active runs and the plaintext bytes of each header. Every scenario stays under the
|
||||
// 8 MiB inspection bound, above which the read declines to inspect any header.
|
||||
const scenarios = (argument("scenarios") ?? "10x300000,50x100000,200x24000").split(",").map(value => {
|
||||
const [runs, headerBytes] = value.split("x").map(Number);
|
||||
return { runs, headerBytes };
|
||||
});
|
||||
const retainedSessions = Number(argument("retained") ?? 20), retainedHeaderBytes = 24_000;
|
||||
const git = (...args: string[]) => execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" });
|
||||
const baselineRevision = git("rev-parse", "--verify", `${baselineRef}^{commit}`).trim();
|
||||
const treatmentRevision = git("rev-parse", "HEAD").trim();
|
||||
const sources = ["server/src/services/heartbeat/upgrade-readiness.ts",
|
||||
"server/src/services/native-runtime/native-runner-checkpoint.ts",
|
||||
"server/src/services/native-runtime/native-controller-recovery-inventory.ts"];
|
||||
const treatmentDiff = git("diff", "--stat", "HEAD", "--", ...sources).trim();
|
||||
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "b".repeat(64);
|
||||
let decryptions = 0;
|
||||
const createDecipheriv = crypto.createDecipheriv;
|
||||
crypto.createDecipheriv = ((...args: Parameters<typeof createDecipheriv>) => { decryptions++; return createDecipheriv(...args); }) as typeof createDecipheriv;
|
||||
syncBuiltinESMExports();
|
||||
const sockets = new Set<Socket>();
|
||||
diagnosticsChannel.subscribe("net.client.socket", message => { sockets.add((message as { socket: Socket }).socket); });
|
||||
const bytesReceived = () => [...sockets].reduce((sum, socket) => sum + socket.bytesRead, 0);
|
||||
const bytesSent = () => [...sockets].reduce((sum, socket) => sum + socket.bytesWritten, 0);
|
||||
|
||||
/** The baseline copies import each other; everything else resolves to the working tree. */
|
||||
async function loadBaseline(directory: string) {
|
||||
const copies = new Map(sources.map(source => [path.join(repoRoot, source), path.join(directory, path.basename(source))]));
|
||||
for (const source of sources) {
|
||||
const original = pathToFileURL(path.join(repoRoot, source));
|
||||
await fs.writeFile(copies.get(path.join(repoRoot, source))!, git("show", `${baselineRevision}:${source}`)
|
||||
.replace(/(from |import\()"(\.{1,2}\/[^"]+)"/g, (_match, prefix: string, specifier: string) => {
|
||||
const resolved = fileURLToPath(new URL(specifier, original));
|
||||
return `${prefix}${JSON.stringify(copies.get(resolved.replace(/\.js$/, ".ts"))?.replace(/\.ts$/, ".js") ?? resolved)}`;
|
||||
}));
|
||||
}
|
||||
return await import(pathToFileURL(path.join(directory, "upgrade-readiness.ts")).href) as { readUpgradeReadiness: typeof readUpgradeReadiness };
|
||||
}
|
||||
|
||||
function databaseCpuSeconds(port: string) {
|
||||
const rows = execFileSync("ps", ["-axo", "pid=,ppid=,cputime=,command="], { encoding: "utf8" }).split("\n")
|
||||
.map(line => line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/)).filter(match => match !== null);
|
||||
const postmaster = rows.find(row => /postgres/.test(row[4]) && row[4].includes(`-p ${port}`))?.[1];
|
||||
assert.ok(postmaster, "The PostgreSQL server process was not found");
|
||||
return rows.filter(row => row[1] === postmaster || row[2] === postmaster).reduce((sum, row) =>
|
||||
sum + row[3].split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0), 0);
|
||||
}
|
||||
|
||||
const keepAlive = setInterval(() => {}, 1000);
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db | undefined;
|
||||
let baselineDirectory: string | undefined;
|
||||
const profiler = new Session();
|
||||
try {
|
||||
database = await startEmbeddedPostgresTestDatabase("upgrade-readiness-benchmark-");
|
||||
db = createDb(database.connectionString);
|
||||
const port = new URL(database.connectionString).port;
|
||||
baselineDirectory = await fs.mkdtemp(path.join(fileURLToPath(new URL("./", import.meta.url)), ".upgrade-readiness-baseline-"));
|
||||
const baseline = await loadBaseline(baselineDirectory);
|
||||
profiler.connect();
|
||||
await profiler.post("HeapProfiler.enable");
|
||||
const owner = await currentNativeControllerIdentity();
|
||||
const cipher = createLocalEncryptedCipher("write");
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Synthetic upgrade readiness", issuePrefix: `U${companyId.replaceAll("-", "")}` });
|
||||
await createAgentLifecycle(db).requestHire(companyId, { id: agentId, name: "Fixture", role: "engineer", status: "idle",
|
||||
adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {} }, { runnerResolved: true });
|
||||
|
||||
// A header a real control plane wrote, then given each run's identity and a provider
|
||||
// attachment template of the scenario's size (the part that makes real headers large).
|
||||
const stateDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "upgrade-readiness-header-"));
|
||||
let template = "";
|
||||
const core = new DurablePrpControlPlane({ stateDirectory, identity: { runId: randomUUID(), runnerInstanceId: randomUUID(),
|
||||
normalizedSessionId: randomUUID(), environmentLeaseId: randomUUID(), turnId: "turn", itemId: "item" },
|
||||
expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
core.setCheckpointPersistence(async checkpoint => { template = checkpoint.records.get("header")!; });
|
||||
try { core.issueBootstrapTicket(); await core.flushCheckpoint(); }
|
||||
finally { await core.stop(); await fs.rm(stateDirectory, { recursive: true, force: true }); }
|
||||
type Run = { runId: string; issueId: string; runnerInstanceId: string; sessionId: string; workspaceId: string };
|
||||
const headerMaterial = (run: Run, bytes: number, key = cipher, environmentLeaseId = run.workspaceId) => {
|
||||
const header = JSON.parse(template);
|
||||
const contents = JSON.stringify({ ...header, state: { ...header.state,
|
||||
identity: { runId: run.runId, runnerInstanceId: run.runnerInstanceId, normalizedSessionId: run.sessionId,
|
||||
environmentLeaseId, turnId: "turn", itemId: "item" },
|
||||
runAttachTemplate: { provider: { instructions: randomBytes(Math.ceil(bytes * 0.75)).toString("base64").slice(0, bytes) } } } });
|
||||
return key.encrypt(JSON.stringify({ companyId, issueId: run.issueId, runId: run.runId, part: "header", contents }));
|
||||
};
|
||||
async function seed(count: number, headerBytes: number, active: boolean) {
|
||||
const runs: Run[] = Array.from({ length: count }, () => ({ runId: randomUUID(), issueId: randomUUID(),
|
||||
runnerInstanceId: randomUUID(), sessionId: randomUUID(), workspaceId: randomUUID() }));
|
||||
for (let offset = 0; offset < runs.length; offset += 10) {
|
||||
const batch = runs.slice(offset, offset + 10);
|
||||
await db!.insert(issues).values(batch.map(run => ({ id: run.issueId, companyId, title: "Synthetic work", status: "todo" })));
|
||||
await db!.insert(heartbeatRuns).values(batch.map(run => ({ id: run.runId, companyId, agentId, issueId: run.issueId,
|
||||
nativeIssueId: run.issueId, runtimeMode: "native", runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.sessionId,
|
||||
...(active ? { status: "running", processPid: process.pid, processStartedAt: new Date(),
|
||||
runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), workspaceId: run.workspaceId,
|
||||
leaseId: randomUUID(), providerLeaseId: "synthetic-sandbox", remoteCwd: "/workspace" } } }
|
||||
: { status: "succeeded", finishedAt: new Date(),
|
||||
runnerProfileJson: { nativeControllerRecovery: { schema: "paperclip.native-controller-recovery.v1",
|
||||
checkpoint: { runId: run.runId, runnerInstanceId: run.runnerInstanceId, normalizedSessionId: run.sessionId } } } }) })));
|
||||
await db!.insert(nativeRunFinalizations).values(batch.map(run => ({ runId: run.runId, companyId, issueId: run.issueId,
|
||||
...(active ? { phase: "observed", controllerBootId: owner.bootId, leaseOwner: "benchmark-lease", controllerGeneration: 1,
|
||||
leaseExpiresAt: new Date(Date.now() + 3_600_000) } : { phase: "committed" }) })));
|
||||
await db!.insert(nativeRunnerCheckpoints).values(batch.map(run => ({ companyId, runId: run.runId, issueId: run.issueId,
|
||||
part: "header", material: headerMaterial(run, headerBytes) })));
|
||||
if (!active) await db!.insert(agentTaskSessions).values(batch.map(run => ({ companyId, agentId,
|
||||
adapterType: "paperclip_runner", taskKey: run.issueId, lastRunId: run.runId })));
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
type Read = typeof readUpgradeReadiness;
|
||||
const comparable = (report: Awaited<ReturnType<Read>>) => ({ ...report, observedAt: null,
|
||||
nativeControllerRecovery: { ...report.nativeControllerRecovery, retained: { ...report.nativeControllerRecovery.retained, observedAt: null } } });
|
||||
let input = { activeRunIds: new Set<string>(), counters: () => ({ activeRuns: 0, pendingWakes: 0, draining: false }) };
|
||||
/** Header rows whose material reached this process in one read, and their JSON size. */
|
||||
let rows = 0, materials = 0, materialBytes = 0;
|
||||
const recorded = new Proxy(db, { get(target, key) {
|
||||
if (key !== "transaction") return Reflect.get(target, key);
|
||||
return (work: (tx: unknown) => Promise<unknown>, options: unknown) => (target.transaction as (...args: unknown[]) => Promise<unknown>)(
|
||||
(tx: { execute: (...args: unknown[]) => Promise<unknown> }) => work(new Proxy(tx, { get(inner, name) {
|
||||
if (name !== "execute") return Reflect.get(inner, name);
|
||||
return async (...args: unknown[]) => {
|
||||
const result = await inner.execute(...args);
|
||||
for (const row of Array.isArray(result) ? result as Array<{ material?: unknown }> : []) {
|
||||
rows++;
|
||||
if (row.material) { materials++; materialBytes += Buffer.byteLength(JSON.stringify(row.material)); }
|
||||
}
|
||||
return result;
|
||||
};
|
||||
} })), options);
|
||||
} });
|
||||
async function materialRead(read: Read) {
|
||||
rows = materials = materialBytes = 0;
|
||||
const before = decryptions;
|
||||
const report = comparable(await read(recorded, input));
|
||||
return { report, rows, materials, materialBytes, decryptions: decryptions - before };
|
||||
}
|
||||
async function measure(read: Read, calls: number, concurrency = 1) {
|
||||
await read(db!, input); // The measured reads repeat a state this revision has already read.
|
||||
const databaseCpu = databaseCpuSeconds(port);
|
||||
await profiler.post("HeapProfiler.startSampling", { samplingInterval: 16_384,
|
||||
includeObjectsCollectedByMajorGC: true, includeObjectsCollectedByMinorGC: true });
|
||||
const wall: number[] = [];
|
||||
const bytes = bytesReceived(), sent = bytesSent(), decrypted = decryptions;
|
||||
const cpu = process.cpuUsage(), loop = performance.eventLoopUtilization();
|
||||
for (let call = 0; call < calls; call++) {
|
||||
const start = performance.now();
|
||||
await Promise.all(Array.from({ length: concurrency }, () => read(db!, input)));
|
||||
wall.push(performance.now() - start);
|
||||
}
|
||||
const used = process.cpuUsage(cpu), busy = performance.eventLoopUtilization(loop).active;
|
||||
const { profile } = await profiler.post("HeapProfiler.stopSampling");
|
||||
const databaseCpuMs = (databaseCpuSeconds(port) - databaseCpu) * 1000;
|
||||
let allocated = 0, decryptAllocated = 0;
|
||||
const visit = (node: typeof profile.head, underDecrypt: boolean) => {
|
||||
const decrypt = underDecrypt || node.callFrame.functionName === "decryptPart";
|
||||
allocated += node.selfSize;
|
||||
if (decrypt) decryptAllocated += node.selfSize;
|
||||
for (const child of node.children) visit(child, decrypt);
|
||||
};
|
||||
visit(profile.head, false);
|
||||
const reads = calls * concurrency, round = (value: number) => Math.round(value * 100) / 100;
|
||||
return { reads, concurrency, medianWallMs: round([...wall].sort((a, b) => a - b)[Math.floor(wall.length / 2)]),
|
||||
bytesFromDatabasePerRead: Math.round((bytesReceived() - bytes) / reads),
|
||||
bytesToDatabasePerRead: Math.round((bytesSent() - sent) / reads), decryptionsPerRead: round((decryptions - decrypted) / reads),
|
||||
processCpuMsPerRead: round((used.user + used.system) / 1000 / reads), mainThreadBusyMsPerRead: round(busy / reads),
|
||||
sampledAllocationBytesPerRead: Math.round(allocated / reads), sampledDecryptAllocationBytesPerRead: Math.round(decryptAllocated / reads),
|
||||
databaseCpuMsPerRead: round(databaseCpuMs / reads) };
|
||||
}
|
||||
|
||||
const retained = await seed(retainedSessions, retainedHeaderBytes, false);
|
||||
const results = [];
|
||||
let last: Run[] = [];
|
||||
for (const scenario of scenarios) {
|
||||
if (last.length) await db.update(heartbeatRuns).set({ status: "failed" }).where(inArray(heartbeatRuns.id, last.map(run => run.runId)));
|
||||
last = await seed(scenario.runs, scenario.headerBytes, true);
|
||||
await db.$client.unsafe("ANALYZE");
|
||||
const ids = new Set(last.map(run => run.runId));
|
||||
input = { activeRunIds: ids, counters: () => ({ activeRuns: ids.size, pendingWakes: 0, draining: false }) };
|
||||
const reference = await materialRead(baseline.readUpgradeReadiness);
|
||||
assert.equal(reference.report.ready, true, "Every fixture run must be recoverable");
|
||||
assert.equal(reference.report.recoverableRuns, scenario.runs);
|
||||
assert.equal((reference.report.nativeControllerRecovery.retained as { selectedRetainedEvidence?: number }).selectedRetainedEvidence, retained.length);
|
||||
const first = await materialRead(readUpgradeReadiness);
|
||||
const repeated = await materialRead(readUpgradeReadiness);
|
||||
assert.deepEqual(first.report, reference.report);
|
||||
assert.deepEqual(repeated.report, reference.report);
|
||||
const pick = ({ report: _report, ...counts }: typeof reference) => counts;
|
||||
const before = await measure(baseline.readUpgradeReadiness, iterations);
|
||||
const after = await measure(readUpgradeReadiness, iterations);
|
||||
results.push({ scenario: { activeRuns: scenario.runs, headerPlaintextBytes: scenario.headerBytes, retainedSessions, retainedHeaderBytes },
|
||||
headerRows: { baseline: pick(reference), treatmentFirstRead: pick(first), treatmentRepeatedRead: pick(repeated) },
|
||||
unchangedRead: { baseline: before, treatment: after },
|
||||
eightConcurrentReads: { baseline: await measure(baseline.readUpgradeReadiness, 3, 8), treatment: await measure(readUpgradeReadiness, 3, 8) } });
|
||||
assert.deepEqual(comparable(await readUpgradeReadiness(db, input)), reference.report);
|
||||
}
|
||||
|
||||
// Same answer in every state, with the working tree warm from the reads above.
|
||||
const [changed, undecryptable, deleted] = last;
|
||||
const setHeader = (run: Run, material: Record<string, unknown>) => db!.update(nativeRunnerCheckpoints).set({ material })
|
||||
.where(eq(nativeRunnerCheckpoints.runId, run.runId));
|
||||
const otherKey = (() => {
|
||||
const saved = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "c".repeat(64);
|
||||
try { return createLocalEncryptedCipher("write"); } finally { process.env.PAPERCLIP_SECRETS_MASTER_KEY = saved; }
|
||||
})();
|
||||
const headerBytes = scenarios.at(-1)!.headerBytes;
|
||||
const states: Array<[string, () => Promise<unknown>, number]> = [
|
||||
["a header changes to another workspace identity (updated_at untouched)", () => setHeader(changed, headerMaterial(changed, headerBytes, cipher, randomUUID())), last.length - 1],
|
||||
["the changed header is restored", () => setHeader(changed, headerMaterial(changed, headerBytes)), last.length],
|
||||
["a header is encrypted under another key", () => setHeader(undecryptable, headerMaterial(undecryptable, headerBytes, otherKey)), last.length - 1],
|
||||
["a header is not encrypted material", () => setHeader(undecryptable, {}), last.length - 1],
|
||||
["a header is deleted", () => db!.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, deleted.runId)), last.length - 2],
|
||||
["the secrets key is replaced", async () => { process.env.PAPERCLIP_SECRETS_MASTER_KEY = "c".repeat(64); }, 0],
|
||||
["the secrets key is restored", async () => { process.env.PAPERCLIP_SECRETS_MASTER_KEY = "b".repeat(64); }, last.length - 2],
|
||||
];
|
||||
const equivalence = [];
|
||||
for (const [state, change, recoverableRuns] of states) {
|
||||
await change();
|
||||
const expected = comparable(await baseline.readUpgradeReadiness(db, input));
|
||||
assert.equal(expected.recoverableRuns, recoverableRuns, state);
|
||||
assert.equal(expected.ready, recoverableRuns === last.length, state);
|
||||
for (let read = 0; read < 2; read++) assert.deepEqual(comparable(await readUpgradeReadiness(db, input)), expected, state);
|
||||
equivalence.push({ state, ready: expected.ready, recoverableRuns: expected.recoverableRuns, blockingRuns: expected.blockingRuns, equal: true });
|
||||
}
|
||||
console.log(JSON.stringify({ baselineRevision, treatmentRevision, treatmentDiff: treatmentDiff || null,
|
||||
node: process.version, postgres: (await db.$client.unsafe("select version()"))[0].version, iterations,
|
||||
boundaries: { bytesFromDatabase: "bytes read from (and written to) the PostgreSQL sockets, protocol framing included",
|
||||
mainThreadBusy: "event-loop active time, which includes every synchronous stretch",
|
||||
sampledAllocation: "V8 sampling heap profile at 16 KiB, collected objects included; Buffer backing stores are outside the V8 heap and not counted",
|
||||
databaseCpu: "CPU time of the PostgreSQL server processes, 10 ms resolution" },
|
||||
results, equivalence }, null, 2));
|
||||
} finally {
|
||||
try {
|
||||
profiler.disconnect();
|
||||
await db?.$client.end({ timeout: 0 }); await database?.cleanup();
|
||||
if (baselineDirectory) await fs.rm(baselineDirectory, { recursive: true, force: true });
|
||||
} finally { clearInterval(keepAlive); }
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import crypto, { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { agents, agentTaskSessions, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues,
|
||||
nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeRunnerUpgradeHeaderMemo, directNativeRunnerUpgradeHeaderRead } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { readNativeControllerRecoveryInventory } from "../native-runtime/native-controller-recovery-inventory.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
import { readUpgradeReadiness } from "./upgrade-readiness.js";
|
||||
|
||||
const KEY = "a".repeat(64), OTHER_KEY = "c".repeat(64);
|
||||
type Run = { runId: string; issueId: string; runnerInstanceId: string; sessionId: string; workspaceId: string };
|
||||
|
||||
describe("upgrade readiness header memo", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: Db;
|
||||
/** `db`, counting the header material that reaches this process. */
|
||||
let observed: Db, materials = 0;
|
||||
/** Runs once, after a header query returns and before its rows are inspected: the next query, or the one after `skip`. */
|
||||
let afterQuery: { skip: number; run: () => void } | undefined;
|
||||
let template = "", companyId = "", agentId = "";
|
||||
let decryptions = 0;
|
||||
const seeded: string[] = [];
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("upgrade-readiness-header-memo-");
|
||||
db = createDb(database.connectionString);
|
||||
observed = new Proxy(db, { get(target, key) {
|
||||
if (key !== "transaction") return Reflect.get(target, key);
|
||||
return (work: (tx: unknown) => Promise<unknown>, options: unknown) => (target.transaction as (...args: unknown[]) => Promise<unknown>)(
|
||||
(tx: { execute: (...args: unknown[]) => Promise<unknown> }) => work(new Proxy(tx, { get(inner, name) {
|
||||
if (name !== "execute") return Reflect.get(inner, name);
|
||||
return async (...args: unknown[]) => {
|
||||
const rows = await inner.execute(...args);
|
||||
if (Array.isArray(rows)) materials += rows.filter(row => (row as { material?: unknown }).material).length;
|
||||
if (Array.isArray(rows) && rows.length && afterQuery && afterQuery.skip-- === 0) { afterQuery.run(); afterQuery = undefined; }
|
||||
return rows;
|
||||
};
|
||||
} })), options);
|
||||
} });
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", KEY);
|
||||
const createDecipheriv = crypto.createDecipheriv;
|
||||
vi.spyOn(crypto, "createDecipheriv").mockImplementation(((...args: Parameters<typeof createDecipheriv>) => {
|
||||
decryptions++;
|
||||
return createDecipheriv(...args);
|
||||
}) as typeof createDecipheriv);
|
||||
syncBuiltinESMExports();
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Header memo", issuePrefix: `H${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Header memo", role: "engineer", adapterType: "paperclip_runner" });
|
||||
// A header a real control plane wrote; each fixture run gets its own identity in it.
|
||||
const root = await mkdtemp(join(tmpdir(), "upgrade-header-memo-"));
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: root, identity: { runId: randomUUID(), runnerInstanceId: randomUUID(),
|
||||
normalizedSessionId: randomUUID(), environmentLeaseId: randomUUID(), turnId: "turn", itemId: "item" },
|
||||
expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
core.setCheckpointPersistence(async checkpoint => { template = checkpoint.records.get("header")!; });
|
||||
try { core.issueBootstrapTicket(); await core.flushCheckpoint(); }
|
||||
finally { await core.stop(); await rm(root, { recursive: true, force: true }); }
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterEach(async () => {
|
||||
afterQuery = undefined;
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", KEY);
|
||||
if (!seeded.length) return;
|
||||
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, companyId));
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(inArray(heartbeatRuns.id, seeded.splice(0)));
|
||||
});
|
||||
afterAll(async () => {
|
||||
vi.restoreAllMocks(); syncBuiltinESMExports(); vi.unstubAllEnvs();
|
||||
await db?.$client.end(); await database?.cleanup();
|
||||
});
|
||||
|
||||
type Header = Record<string, unknown>;
|
||||
function headerContents(run: Run, patch: (header: Header) => Header = header => header) {
|
||||
const header = JSON.parse(template) as Header & { state: Header };
|
||||
return JSON.stringify(patch({ ...header, state: { ...header.state, identity: { runId: run.runId,
|
||||
runnerInstanceId: run.runnerInstanceId, normalizedSessionId: run.sessionId, environmentLeaseId: run.workspaceId,
|
||||
turnId: "turn", itemId: "item" } } }));
|
||||
}
|
||||
function material(run: Run, contents = headerContents(run), key = KEY) {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key);
|
||||
try {
|
||||
return createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId: run.issueId, runId: run.runId, part: "header", contents }));
|
||||
} finally { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", KEY); }
|
||||
}
|
||||
const setHeader = (run: Run, value: Record<string, unknown>) => db.update(nativeRunnerCheckpoints).set({ material: value })
|
||||
.where(and(eq(nativeRunnerCheckpoints.runId, run.runId), eq(nativeRunnerCheckpoints.part, "header")));
|
||||
/** Active runs this server could hand to its replacement, or finished runs a task session still selects. */
|
||||
async function seed(count: number, active = true) {
|
||||
const owner = await currentNativeControllerIdentity();
|
||||
const runs: Run[] = Array.from({ length: count }, () => ({ runId: randomUUID(), issueId: randomUUID(),
|
||||
runnerInstanceId: randomUUID(), sessionId: randomUUID(), workspaceId: randomUUID() }));
|
||||
await db.insert(issues).values(runs.map(run => ({ id: run.issueId, companyId, title: "Synthetic work", status: "todo" })));
|
||||
await db.insert(heartbeatRuns).values(runs.map(run => ({ id: run.runId, companyId, agentId, issueId: run.issueId,
|
||||
nativeIssueId: run.issueId, runtimeMode: "native", runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.sessionId,
|
||||
...(active ? { status: "running", processPid: process.pid, processStartedAt: new Date(),
|
||||
runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), workspaceId: run.workspaceId,
|
||||
leaseId: randomUUID(), providerLeaseId: "synthetic-sandbox", remoteCwd: "/workspace" } } }
|
||||
: { status: "succeeded", finishedAt: new Date(), runnerProfileJson: { nativeControllerRecovery: {
|
||||
schema: "paperclip.native-controller-recovery.v1",
|
||||
checkpoint: { runId: run.runId, runnerInstanceId: run.runnerInstanceId, normalizedSessionId: run.sessionId } } } }) })));
|
||||
await db.insert(nativeRunFinalizations).values(runs.map(run => ({ runId: run.runId, companyId, issueId: run.issueId,
|
||||
...(active ? { phase: "observed", controllerBootId: owner.bootId, leaseOwner: "memo-lease", controllerGeneration: 1,
|
||||
leaseExpiresAt: new Date(Date.now() + 3_600_000) } : { phase: "committed" }) })));
|
||||
await db.insert(nativeRunnerCheckpoints).values(runs.map(run => ({ companyId, runId: run.runId, issueId: run.issueId,
|
||||
part: "header", material: material(run) })));
|
||||
if (!active) await db.insert(agentTaskSessions).values(runs.map(run => ({ companyId, agentId,
|
||||
adapterType: "paperclip_runner", taskKey: run.issueId, lastRunId: run.runId })));
|
||||
seeded.push(...runs.map(run => run.runId));
|
||||
return runs;
|
||||
}
|
||||
/** One probe, with the header material that reached this process and the decryptions it ran. */
|
||||
async function probe(runs: Run[], headers?: Parameters<typeof readUpgradeReadiness>[1]["headers"]) {
|
||||
const ids = new Set(runs.map(run => run.runId));
|
||||
const before = decryptions, read = materials;
|
||||
const report = await readUpgradeReadiness(observed, { activeRunIds: ids,
|
||||
counters: () => ({ activeRuns: ids.size, pendingWakes: 0, draining: false }), headers });
|
||||
return { materials: materials - read, decryptions: decryptions - before, report: { ...report, observedAt: null,
|
||||
nativeControllerRecovery: { ...report.nativeControllerRecovery, retained: { ...report.nativeControllerRecovery.retained, observedAt: null } } } };
|
||||
}
|
||||
/** The answer when every inspected header is read and decrypted, as before the memo. */
|
||||
const reference = async (runs: Run[]) => (await probe(runs, directNativeRunnerUpgradeHeaderRead())).report;
|
||||
|
||||
it("reads and decrypts no header on a repeated probe of unchanged runs", async () => {
|
||||
const runs = await seed(4), retained = await seed(3, false);
|
||||
const first = await probe(runs);
|
||||
expect(first.report).toMatchObject({ ready: true, recoverableRuns: 4, blockingRuns: 0,
|
||||
nativeControllerRecovery: { retained: { selectedRetainedEvidence: retained.length } } });
|
||||
// Every header that contributes evidence is read and decrypted once.
|
||||
expect(first).toMatchObject({ materials: 7, decryptions: 7 });
|
||||
for (let repeat = 0; repeat < 3; repeat++) {
|
||||
const again = await probe(runs);
|
||||
expect(again.report).toEqual(first.report);
|
||||
expect({ materials: again.materials, decryptions: again.decryptions }).toEqual({ materials: 0, decryptions: 0 });
|
||||
}
|
||||
expect(await reference(runs)).toEqual(first.report);
|
||||
});
|
||||
|
||||
it("re-inspects a checkpoint that changes between two probes", async () => {
|
||||
const runs = await seed(3), [changed] = runs;
|
||||
expect((await probe(runs)).report).toMatchObject({ ready: true, recoverableRuns: 3 });
|
||||
expect((await probe(runs)).materials).toBe(0);
|
||||
// The run's checkpoint now belongs to another workspace. Only the material changes: no updated_at.
|
||||
await setHeader(changed, material(changed, headerContents({ ...changed, workspaceId: randomUUID() })));
|
||||
const after = await probe(runs);
|
||||
expect(after.report).toMatchObject({ ready: false, recoverableRuns: 2, blockingRuns: 1 });
|
||||
expect(after).toMatchObject({ materials: 1, decryptions: 1 });
|
||||
expect(after.report).toEqual(await reference(runs));
|
||||
// Unchanged again: the new answer is the one remembered, also after the rows are frozen or moved.
|
||||
expect(await probe(runs)).toMatchObject({ materials: 0, report: { ready: false, recoverableRuns: 2 } });
|
||||
await db.execute(sql`vacuum (freeze) native_runner_checkpoints`);
|
||||
expect(await probe(runs)).toMatchObject({ materials: 0, report: { ready: false, recoverableRuns: 2 } });
|
||||
await db.execute(sql`vacuum (full) native_runner_checkpoints`);
|
||||
expect((await probe(runs)).report).toEqual(await reference(runs));
|
||||
// The same content written again is a new revision, so it is inspected again.
|
||||
await setHeader(changed, material(changed));
|
||||
expect(await probe(runs)).toMatchObject({ materials: 1, decryptions: 1, report: { ready: true, recoverableRuns: 3 } });
|
||||
await db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, changed.runId));
|
||||
await db.insert(nativeRunnerCheckpoints).values({ companyId, runId: changed.runId, issueId: changed.issueId, part: "header",
|
||||
material: material(changed, headerContents({ ...changed, sessionId: randomUUID() })) });
|
||||
expect(await probe(runs)).toMatchObject({ materials: 1, report: { ready: false, recoverableRuns: 2 } });
|
||||
});
|
||||
|
||||
it("gives the answer of reading every header in every state, warm or cold", async () => {
|
||||
const runs = await seed(4), retained = await seed(2, false), [target] = runs;
|
||||
const legacy = (header: Header) => ({ ...header, schema: "paperclip.runner.controller-checkpoint.v1" });
|
||||
const states: Array<[string, () => Promise<unknown>, { ready: boolean; recoverableRuns: number }]> = [
|
||||
["every header binds its run", async () => {}, { ready: true, recoverableRuns: 4 }],
|
||||
["a header names another runner", () => setHeader(target, material(target, headerContents({ ...target, runnerInstanceId: randomUUID() }))), { ready: false, recoverableRuns: 3 }],
|
||||
["a header is encrypted under another key", () => setHeader(target, material(target, headerContents(target), OTHER_KEY)), { ready: false, recoverableRuns: 3 }],
|
||||
["a header is bound to another run", () => setHeader(target, material({ ...target, runId: randomUUID() })), { ready: false, recoverableRuns: 3 }],
|
||||
["a header is not encrypted material", () => setHeader(target, {}), { ready: false, recoverableRuns: 3 }],
|
||||
["a header's ciphertext is damaged", async () => {
|
||||
const good = material(target) as Record<string, string>;
|
||||
await setHeader(target, { ...good, ciphertext: Buffer.from("damaged").toString("base64") + good.ciphertext.slice(12) });
|
||||
}, { ready: false, recoverableRuns: 3 }],
|
||||
["a header is not valid JSON", () => setHeader(target, material(target, "{")), { ready: false, recoverableRuns: 3 }],
|
||||
["a header has the earlier schema", () => setHeader(target, material(target, headerContents(target, legacy))), { ready: false, recoverableRuns: 3 }],
|
||||
["a header exceeds the inspection bound", () => setHeader(target, material(target, headerContents(target, header => ({ ...header, padding: "x".repeat(1_100_000) })))), { ready: false, recoverableRuns: 3 }],
|
||||
["a header is missing", () => db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, target.runId)), { ready: false, recoverableRuns: 3 }],
|
||||
["the header is written again", () => db.insert(nativeRunnerCheckpoints).values({ companyId, runId: target.runId, issueId: target.issueId, part: "header", material: material(target) }), { ready: true, recoverableRuns: 4 }],
|
||||
["a retained header names another session", () => setHeader(retained[0], material(retained[0], headerContents({ ...retained[0], sessionId: randomUUID() }))), { ready: true, recoverableRuns: 4 }],
|
||||
["the secrets key is replaced", async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", OTHER_KEY); }, { ready: false, recoverableRuns: 0 }],
|
||||
["the secrets key is invalid", async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "not-a-key"); }, { ready: false, recoverableRuns: 0 }],
|
||||
["the secrets key is restored", async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", KEY); }, { ready: true, recoverableRuns: 4 }],
|
||||
["a run ends", () => db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runs[3].runId)), { ready: false, recoverableRuns: 3 }],
|
||||
];
|
||||
for (const [state, change, expected] of states) {
|
||||
await change();
|
||||
const answer = await reference(runs);
|
||||
expect(answer, state).toMatchObject(expected);
|
||||
// Warm from the previous state, then warm from this one.
|
||||
expect((await probe(runs)).report, state).toEqual(answer);
|
||||
expect((await probe(runs)).report, state).toEqual(answer);
|
||||
// A reader that remembers nothing yet.
|
||||
expect((await probe(runs, createNativeRunnerUpgradeHeaderMemo().read())).report, state).toEqual(answer);
|
||||
}
|
||||
expect((await reference(runs)).nativeControllerRecovery.retained).toMatchObject({ selectedRetainedEvidence: 1,
|
||||
reasons: { checkpoint_binding_missing: 1 } });
|
||||
});
|
||||
|
||||
it("reads and tries a header that does not decrypt on every probe", async () => {
|
||||
const runs = await seed(2), [target] = runs;
|
||||
await setHeader(target, material(target, headerContents(target), OTHER_KEY));
|
||||
for (let repeat = 0; repeat < 3; repeat++) {
|
||||
// The other run's answer is remembered after the first probe; this one never is.
|
||||
expect(await probe(runs)).toMatchObject({ materials: repeat ? 1 : 2, decryptions: repeat ? 1 : 2,
|
||||
report: { ready: false, recoverableRuns: 1, blockingRuns: 1 } });
|
||||
}
|
||||
// It was written under the key that is now current: the next probe finds it readable.
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", OTHER_KEY);
|
||||
expect(await probe(runs)).toMatchObject({ materials: 2, decryptions: 2, report: { recoverableRuns: 1, blockingRuns: 1 } });
|
||||
});
|
||||
|
||||
const replaceKey = () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", OTHER_KEY); };
|
||||
const removeKey = () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(tmpdir(), `missing-secrets-key-${randomUUID()}`));
|
||||
};
|
||||
it.each([
|
||||
["replaced", "run", replaceKey, 0], ["removed", "run", removeKey, 0],
|
||||
["replaced", "retained session", replaceKey, 1], ["removed", "retained session", removeKey, 1],
|
||||
] as const)("refuses a remembered header when the secrets key is %s before the %s headers are inspected", async (_change, _query, change, skip) => {
|
||||
const restore = () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", undefined);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", KEY);
|
||||
};
|
||||
const runs = await seed(3);
|
||||
await seed(2, false);
|
||||
expect((await probe(runs)).report).toMatchObject({ ready: true, recoverableRuns: 3 });
|
||||
expect((await probe(runs)).materials).toBe(0);
|
||||
// The key changes while the probe waits for its query. A direct read decrypts after that, and fails.
|
||||
afterQuery = { skip, run: change };
|
||||
const warm = await probe(runs);
|
||||
restore();
|
||||
afterQuery = { skip, run: change };
|
||||
const direct = await probe(runs, directNativeRunnerUpgradeHeaderRead());
|
||||
// Headers inspected before the change still count; those inspected after it do not.
|
||||
expect(direct).toMatchObject({ materials: 5, report: { ...(skip ? { ready: true, recoverableRuns: 3 }
|
||||
: { ready: false, recoverableRuns: 0, blockingRuns: 3 }), nativeControllerRecovery: { retained: {
|
||||
selectedRetainedEvidence: 0, reasons: { checkpoint_binding_missing: 2 } } } } });
|
||||
expect(warm.report).toEqual(direct.report);
|
||||
expect(warm.decryptions).toBe(0);
|
||||
// Nothing from that probe is remembered: with the key back, every header is read again.
|
||||
restore();
|
||||
expect(await probe(runs)).toMatchObject({ materials: 5, decryptions: 5, report: { ready: true, recoverableRuns: 3 } });
|
||||
expect(await probe(runs)).toMatchObject({ materials: 0, report: { ready: true, recoverableRuns: 3 } });
|
||||
|
||||
// The same at the memo: an answer offered before the change is not served after it.
|
||||
const memo = createNativeRunnerUpgradeHeaderMemo();
|
||||
await probe(runs, memo.read());
|
||||
const read = memo.read();
|
||||
const offered = Object.entries(JSON.parse(read.known) as Record<string, string>);
|
||||
expect(offered).toHaveLength(5);
|
||||
change();
|
||||
for (const [binding, revision] of offered) {
|
||||
const [, issueId, runId] = binding.split("/");
|
||||
expect(read.inspect({ companyId, issueId, runId }, revision, null)).toBeNull();
|
||||
}
|
||||
restore();
|
||||
read.finish();
|
||||
expect(memo.size).toBe(0);
|
||||
});
|
||||
|
||||
it("reads retained headers of the inventory page only, and only once", async () => {
|
||||
await seed(3, false);
|
||||
const memo = createNativeRunnerUpgradeHeaderMemo();
|
||||
const page = async (headers = memo.read()) => {
|
||||
const before = materials;
|
||||
const inventory = await readNativeControllerRecoveryInventory(observed, { limit: 2, headers });
|
||||
headers.finish();
|
||||
return { materials: materials - before, inventory: { ...inventory, observedAt: null } };
|
||||
};
|
||||
const direct = await page(directNativeRunnerUpgradeHeaderRead());
|
||||
expect(direct).toMatchObject({ materials: 2, inventory: { inspected: 2, truncated: true, selectedRetainedEvidence: 2 } });
|
||||
// The third session only shows that another page exists; its header is not inspected.
|
||||
expect(await page()).toEqual(direct);
|
||||
expect(await page()).toEqual({ ...direct, materials: 0 });
|
||||
expect(memo.size).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps four identifiers per header, within its bounds, and forgets a run that ended", async () => {
|
||||
const runs = await seed(5);
|
||||
const memo = createNativeRunnerUpgradeHeaderMemo();
|
||||
expect((await probe(runs, memo.read())).report).toMatchObject({ ready: true, recoverableRuns: 5 });
|
||||
expect(memo.size).toBe(5);
|
||||
expect(memo.bytes).toBeLessThan(5 * 1024);
|
||||
const read = memo.read();
|
||||
for (const [binding, revision] of Object.entries(JSON.parse(read.known) as Record<string, string>)) {
|
||||
const [, issueId, runId] = binding.split("/");
|
||||
const run = runs.find(candidate => candidate.runId === runId)!;
|
||||
expect(read.inspect({ companyId, issueId, runId }, revision, null)).toEqual({ runId, runnerInstanceId: run.runnerInstanceId,
|
||||
normalizedSessionId: run.sessionId, environmentLeaseId: run.workspaceId });
|
||||
// Without the material, only the exact revision this read offered has an answer.
|
||||
expect(read.inspect({ companyId, issueId, runId }, `${revision}0`, null)).toBeNull();
|
||||
expect(read.inspect({ companyId, issueId: randomUUID(), runId }, revision, null)).toBeNull();
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded" }).where(inArray(heartbeatRuns.id, runs.slice(0, 2).map(run => run.runId)));
|
||||
await probe(runs.slice(2), memo.read());
|
||||
expect(memo.size).toBe(3);
|
||||
const bounded = createNativeRunnerUpgradeHeaderMemo({ maxEntries: 2 });
|
||||
expect((await probe(runs.slice(2), bounded.read())).report).toMatchObject({ recoverableRuns: 3 });
|
||||
expect(bounded.size).toBe(2);
|
||||
// The header that was not kept is read again; the answer does not change.
|
||||
expect(await probe(runs.slice(2), bounded.read())).toMatchObject({ materials: 1, decryptions: 1, report: { recoverableRuns: 3 } });
|
||||
const tiny = createNativeRunnerUpgradeHeaderMemo({ maxBytes: 64 });
|
||||
expect((await probe(runs.slice(2), tiny.read())).report).toMatchObject({ recoverableRuns: 3 });
|
||||
expect([tiny.size, tiny.bytes]).toEqual([0, 0]);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { createNativeRunnerUpgradeHeaderMemo, nativeRunnerCheckpointRevisionSql, nativeRunnerUpgradeHeaderSql, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES, type NativeRunnerUpgradeHeaderRead } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
|
||||
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
@@ -19,6 +19,9 @@ type Counters = { activeRuns: number; pendingWakes: number; unsavedWakes?: numbe
|
||||
/** `restartable`: the row and its events show no launched agent work, so a replacement
|
||||
* server starts the same work again. It is weaker than `recoverable` and never implies it. */
|
||||
type Candidate = { id: string; recoverable: boolean; restartable?: boolean; retainedUnowned?: boolean };
|
||||
/** Cloud repeats this probe, so an unchanged header is not read or decrypted again.
|
||||
* A row revision means something in one database only: each handle has its own memo. */
|
||||
const upgradeHeaders = new WeakMap<Db, ReturnType<typeof createNativeRunnerUpgradeHeaderMemo>>();
|
||||
|
||||
/** An observation for Cloud's optional rollout policy, never execution authority.
|
||||
* Only explicit upgrade probes read the database; ordinary drain reads stay local.
|
||||
@@ -27,15 +30,24 @@ type Candidate = { id: string; recoverable: boolean; restartable?: boolean; reta
|
||||
export async function readUpgradeReadiness(db: Db, input: {
|
||||
activeRunIds: ReadonlySet<string>;
|
||||
counters: () => Counters;
|
||||
/** Defaults to the header answers remembered for this database handle. */
|
||||
headers?: NativeRunnerUpgradeHeaderRead;
|
||||
}) {
|
||||
if (!input.headers && !upgradeHeaders.has(db)) upgradeHeaders.set(db, createNativeRunnerUpgradeHeaderMemo());
|
||||
const headers = input.headers ?? upgradeHeaders.get(db)!.read();
|
||||
const header = nativeRunnerUpgradeHeaderSql(headers, { companyId: sql`q."companyId"`, issueId: sql`q."issueId"`,
|
||||
runId: sql`q.id`, revision: sql`q."headerRevision"`, material: sql`case when q.recoverable or q."hasResult" then q.material end` });
|
||||
const inspectionDeadline = performance.now() + POST_RESULT_INSPECTION_BUDGET_MS;
|
||||
const before = input.counters();
|
||||
const beforeIds = [...input.activeRunIds];
|
||||
const owner = await currentNativeControllerIdentity();
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; remoteWorkspace: unknown; hasRemoteWorkspace: boolean; hasResult: boolean; pluginLeaseId: string | null; material: Record<string, unknown> | null }>(sql`
|
||||
select r.id, r.company_id as "companyId", r.native_issue_id as "issueId",
|
||||
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; remoteWorkspace: unknown; hasRemoteWorkspace: boolean; hasResult: boolean; pluginLeaseId: string | null; headerRevision: string | null; material: Record<string, unknown> | null }>(sql`
|
||||
select q.id, q."companyId", q."issueId", q."runnerId", q."sessionId", q.workspace, q."remoteWorkspace",
|
||||
q."hasRemoteWorkspace", q."pluginLeaseId", q."hasResult", q."retainedUnowned", q.recoverable, q.restartable,
|
||||
${header.revision} as "headerRevision", ${header.material} as material
|
||||
from (select r.id, r.company_id as "companyId", r.native_issue_id as "issueId",
|
||||
r.runner_instance_id as "runnerId", r.native_session_id as "sessionId",
|
||||
r.runner_profile_json->'nativeWorkspaceSync' as workspace,
|
||||
r.runner_profile_json->'remoteWorkspace' as "remoteWorkspace",
|
||||
@@ -49,7 +61,8 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
and r.process_pid is null and r.process_started_at is null and c.run_id is null
|
||||
) is true as "retainedUnowned",
|
||||
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES} then c.material else null end as material, (
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES} then c.material else null end as material,
|
||||
${nativeRunnerCheckpointRevisionSql(sql`c`)} as "headerRevision", (
|
||||
r.runtime_mode = 'native' and a.adapter_type = 'paperclip_runner'
|
||||
and r.runner_instance_id is not null and r.native_session_id is not null
|
||||
and r.process_pid is not null and r.process_started_at is not null
|
||||
@@ -89,7 +102,7 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
and coalesce(l.metadata->>'runnerTaskRetiring', 'false') != 'true'
|
||||
order by l.created_at desc limit 1) p on true
|
||||
where r.status = 'running'
|
||||
limit ${MAX_UPGRADE_RUNS + 1}`);
|
||||
limit ${MAX_UPGRADE_RUNS + 1}) q ${header.join}`);
|
||||
}, { accessMode: "read only" });
|
||||
const candidates: Candidate[] = [];
|
||||
for (const row of rows) {
|
||||
@@ -99,8 +112,9 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
const workspaceId = row.hasRemoteWorkspace
|
||||
? remote.success && !row.pluginLeaseId ? remote.data.workspaceId : null
|
||||
: row.pluginLeaseId ?? readNativeWorkspaceSyncReference(row.workspace)?.workspaceId;
|
||||
const identity = (row.recoverable || row.hasResult) && row.material
|
||||
? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.id }, row.material) : null;
|
||||
// A revision is present exactly when the header is within bounds and the run is recoverable or has a result.
|
||||
const identity = row.headerRevision
|
||||
? headers.inspect({ companyId: row.companyId, issueId: row.issueId, runId: row.id }, row.headerRevision, row.material) : null;
|
||||
const bound = Boolean(workspaceId && identity && identity.runId === row.id
|
||||
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
|
||||
&& identity.environmentLeaseId === workspaceId);
|
||||
@@ -112,10 +126,11 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
restartable: row.restartable === true, retainedUnowned: row.retainedUnowned });
|
||||
}
|
||||
// Additive evidence does not strengthen or redefine the existing ready flag.
|
||||
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
|
||||
const retained = await readNativeControllerRecoveryInventory(db, { headers }).catch(() => ({
|
||||
version: 1 as const, observedAt: new Date().toISOString(), coverage: "unknown" as const,
|
||||
reasons: { inventory_unavailable: 1 }, fullPayloadsValidated: false as const,
|
||||
}));
|
||||
headers.finish();
|
||||
const summary = summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
|
||||
activeRunIds: [...new Set([...beforeIds, ...input.activeRunIds])], controllerBootId: owner.bootId });
|
||||
return { ...summary, nativeControllerRecovery: { activeExecutionAdoption: {
|
||||
|
||||
@@ -1,40 +1,55 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "./native-runner-checkpoint.js";
|
||||
import { nativeRunnerCheckpointRevisionSql, nativeRunnerUpgradeHeaderSql, directNativeRunnerUpgradeHeaderRead, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES, type NativeRunnerUpgradeHeaderRead } from "./native-runner-checkpoint.js";
|
||||
|
||||
const RECOVERY_METADATA_SCHEMA = "paperclip.native-controller-recovery.v1";
|
||||
|
||||
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
||||
* This inventories selected retained task-session pointers, not all history.
|
||||
* Header evidence is preliminary: full snapshot/journal validation still occurs
|
||||
* only when ordinary admission selects its exact continuation source. */
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: {
|
||||
afterId?: string; limit?: number;
|
||||
/** The caller finishes this read. Without one, every inspected header is decrypted. */
|
||||
headers?: NativeRunnerUpgradeHeaderRead;
|
||||
} = {}) {
|
||||
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
|
||||
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
|
||||
const headers = input.headers ?? directNativeRunnerUpgradeHeaderRead();
|
||||
// The rows whose header the loop below inspects. The row after the page only shows that more exist.
|
||||
const header = nativeRunnerUpgradeHeaderSql(headers, { companyId: sql`q."companyId"`, issueId: sql`q."issueId"`,
|
||||
runId: sql`q."runId"`, revision: sql`q."headerRevision"`, material: sql`case when q.position <= ${limit} and not q.pending
|
||||
and q."runId" is not null and q."issueId" is not null and q."sessionId" is not null
|
||||
and q.metadata->>'schema' = ${RECOVERY_METADATA_SCHEMA} then q.material end` });
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<{
|
||||
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
|
||||
pending: boolean;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; headerRevision: string | null;
|
||||
material: Record<string, unknown> | null; pending: boolean;
|
||||
}>(sql`with selected as (
|
||||
select id, company_id, agent_id, last_run_id from agent_task_sessions
|
||||
select id, company_id, agent_id, last_run_id, row_number() over (order by id) as position from agent_task_sessions
|
||||
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
|
||||
order by id limit ${limit + 1}
|
||||
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
) select q.id, q."companyId", q."runId", q."issueId", q."sessionId", q."runnerId", q.metadata, q.pending,
|
||||
${header.revision} as "headerRevision", ${header.material} as material
|
||||
from (select s.id, s.position, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
||||
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
||||
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
||||
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES}
|
||||
then c.material else null end as material,
|
||||
${nativeRunnerCheckpointRevisionSql(sql`c`)} as "headerRevision",
|
||||
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
||||
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
||||
or f.phase not in ('committed', 'terminal_failure')) is true as pending
|
||||
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
|
||||
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header'
|
||||
order by s.id`);
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header') q ${header.join}
|
||||
order by q.id`);
|
||||
}, { accessMode: "read only" });
|
||||
const truncated = rows.length > limit;
|
||||
const page = rows.slice(0, limit);
|
||||
@@ -44,10 +59,11 @@ export async function readNativeControllerRecoveryInventory(db: Db, input: { aft
|
||||
let reason: string | null = null;
|
||||
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
|
||||
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
|
||||
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
|
||||
else if (row.metadata?.schema !== RECOVERY_METADATA_SCHEMA) reason = "metadata_unsupported_or_missing";
|
||||
else {
|
||||
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
|
||||
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
|
||||
const header = row.headerRevision ? headers.inspect({ companyId: row.companyId, issueId: row.issueId, runId: row.runId },
|
||||
row.headerRevision, row.material) : null;
|
||||
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|
||||
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|
||||
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { readNativeCheckpointRetirement } from "./native-runner-checkpoint-retirement.js";
|
||||
import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm";
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm";
|
||||
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js";
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js";
|
||||
import { runnerdStateHasNoProviderTurn, runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js";
|
||||
@@ -396,10 +397,137 @@ export async function listExpiredNativeRunnerAuthorities(db: Db, now: Date, runI
|
||||
return rows.map(row => row.runId);
|
||||
}
|
||||
|
||||
/** Only a header that decrypted has a repeatable answer: a key or material failure may not recur. */
|
||||
function inspectUpgradeHeaderMaterial(binding: Binding, material: Record<string, unknown>) {
|
||||
let contents: string;
|
||||
try { contents = decryptPart(createLocalEncryptedCipher("read"), binding, "header", material); }
|
||||
catch { return { decrypted: false as const, identity: null }; }
|
||||
return { decrypted: true as const, identity: inspectDurablePrpCheckpointHeader(contents) };
|
||||
}
|
||||
|
||||
/** Inspect only the bounded encrypted header. Protocol interpretation stays in Runner. */
|
||||
export function inspectNativeRunnerUpgradeHeader(binding: Binding, material: Record<string, unknown>) {
|
||||
return inspectUpgradeHeaderMaterial(binding, material).identity;
|
||||
}
|
||||
|
||||
/** The only header fields that rollout evidence compares, and that a memo keeps. */
|
||||
export type NativeRunnerUpgradeHeaderIdentity = Pick<DurableRecoveryIdentity,
|
||||
"runId" | "runnerInstanceId" | "normalizedSessionId" | "environmentLeaseId">;
|
||||
const upgradeHeaderIdentity = (identity: DurableRecoveryIdentity | null): NativeRunnerUpgradeHeaderIdentity | null => identity && {
|
||||
runId: identity.runId, runnerInstanceId: identity.runnerInstanceId,
|
||||
normalizedSessionId: identity.normalizedSessionId, environmentLeaseId: identity.environmentLeaseId };
|
||||
|
||||
/** One read of header evidence. `known` is a JSON object from binding to the row
|
||||
* revision this read answers without the material. Call `finish` after a completed read. */
|
||||
export type NativeRunnerUpgradeHeaderRead = {
|
||||
readonly known: string;
|
||||
inspect(binding: Binding, revision: string, material: Record<string, unknown> | null): NativeRunnerUpgradeHeaderIdentity | null;
|
||||
finish(): void;
|
||||
};
|
||||
|
||||
/** PostgreSQL writes a new row version, with a new xmin or ctid, for every change to
|
||||
* a row, so an unchanged revision means unchanged material. The write time and stored
|
||||
* size are included so that no single field carries that guarantee. */
|
||||
export function nativeRunnerCheckpointRevisionSql(checkpoint: SQL) {
|
||||
return sql`${checkpoint}.xmin::text || ':' || ${checkpoint}.ctid::text
|
||||
|| ':' || (extract(epoch from ${checkpoint}.updated_at) * 1000000)::bigint::text
|
||||
|| ':' || pg_column_size(${checkpoint}.material)::text`;
|
||||
}
|
||||
|
||||
/** `material` must already be null for a row the reader will not inspect. The statement
|
||||
* that reads a run sends its header's material only for a revision the read does not know. */
|
||||
export function nativeRunnerUpgradeHeaderSql(read: Pick<NativeRunnerUpgradeHeaderRead, "known">,
|
||||
row: { companyId: SQL; issueId: SQL; runId: SQL; revision: SQL; material: SQL }) {
|
||||
return {
|
||||
join: sql`left join jsonb_each_text(${read.known}::jsonb) as known_header(binding, revision)
|
||||
on known_header.binding = ${row.companyId}::text || '/' || ${row.issueId}::text || '/' || ${row.runId}::text`,
|
||||
revision: sql`case when ${row.material} is not null then ${row.revision} end`,
|
||||
material: sql`case when ${row.revision} is distinct from known_header.revision then ${row.material} end`,
|
||||
};
|
||||
}
|
||||
|
||||
/** A read that remembers nothing: every inspected header is sent and decrypted. */
|
||||
export function directNativeRunnerUpgradeHeaderRead(): NativeRunnerUpgradeHeaderRead {
|
||||
return { known: "{}", finish() {},
|
||||
inspect: (binding, _revision, material) => material ? upgradeHeaderIdentity(inspectNativeRunnerUpgradeHeader(binding, material)) : null };
|
||||
}
|
||||
|
||||
/** A digest of a value derived from the secrets key for this purpose alone. It opens nothing. */
|
||||
function upgradeHeaderKeyIdentity(): string | null {
|
||||
try {
|
||||
const contents = decryptPart(createLocalEncryptedCipher("read"), binding, "header", material);
|
||||
return inspectDurablePrpCheckpointHeader(contents);
|
||||
return createHash("sha256").update(deriveLocalEncryptedInstanceSecret("native-runner-upgrade-header-memo",
|
||||
process.env, { allowCreate: false })).digest("hex");
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Remembers the answer for a header that decrypted, so an unchanged header is not
|
||||
* sent or decrypted again. An answer is reused only for the same binding, row revision
|
||||
* and secrets key; a header that did not decrypt is read and tried on every read.
|
||||
* An entry holds four identifiers, entries are bounded in number and bytes, and a
|
||||
* read forgets every header it did not inspect, such as that of a run that ended. */
|
||||
export function createNativeRunnerUpgradeHeaderMemo(limits: { maxEntries?: number; maxBytes?: number } = {}) {
|
||||
const maxEntries = limits.maxEntries ?? 1_200, maxBytes = limits.maxBytes ?? 1024 * 1024;
|
||||
type Answer = { revision: string; identity: NativeRunnerUpgradeHeaderIdentity | null };
|
||||
const entries = new Map<string, Answer & { keyIdentity: string; bytes: number }>();
|
||||
let bytes = 0;
|
||||
const forget = (key: string) => {
|
||||
const entry = entries.get(key);
|
||||
if (entry && entries.delete(key)) bytes -= entry.bytes;
|
||||
};
|
||||
return {
|
||||
get size() { return entries.size; },
|
||||
get bytes() { return bytes; },
|
||||
read(): NativeRunnerUpgradeHeaderRead {
|
||||
let keyIdentity = entries.size ? upgradeHeaderKeyIdentity() : undefined;
|
||||
const usable = new Map([...entries].filter(([, entry]) => entry.keyIdentity === keyIdentity));
|
||||
const reused = new Set<string>(), decrypted = new Map<string, Answer>();
|
||||
let keyChanged = false;
|
||||
// A direct read loads the key for each header, after the query. An answer from memory
|
||||
// is as conservative only if the key is loaded and unchanged when the answer is used.
|
||||
const sameKey = () => {
|
||||
const now = upgradeHeaderKeyIdentity();
|
||||
if (keyIdentity === undefined) keyIdentity = now;
|
||||
const same = now !== null && now === keyIdentity;
|
||||
if (!same) keyChanged = true;
|
||||
return same;
|
||||
};
|
||||
return {
|
||||
known: JSON.stringify(Object.fromEntries([...usable].map(([key, entry]) => [key, entry.revision]))),
|
||||
inspect(binding, revision, material) {
|
||||
const key = `${binding.companyId}/${binding.issueId}/${binding.runId}`;
|
||||
if (!material) {
|
||||
// The query withholds material only for a revision this read offered.
|
||||
const entry = usable.get(key);
|
||||
if (entry?.revision !== revision || !sameKey()) return null;
|
||||
reused.add(key);
|
||||
return entry.identity;
|
||||
}
|
||||
const before = sameKey();
|
||||
const result = inspectUpgradeHeaderMaterial(binding, material);
|
||||
const identity = upgradeHeaderIdentity(result.identity);
|
||||
if (result.decrypted && before && sameKey()) decrypted.set(key, { revision, identity });
|
||||
else decrypted.delete(key);
|
||||
return identity;
|
||||
},
|
||||
finish() {
|
||||
// A read that saw the key missing or changed keeps and remembers nothing.
|
||||
const current = keyChanged || !keyIdentity ? null : keyIdentity;
|
||||
for (const [key, entry] of entries) {
|
||||
if (!reused.has(key) || entry.keyIdentity !== current) forget(key);
|
||||
}
|
||||
for (const [key, answer] of current ? decrypted : []) {
|
||||
const size = 2 * (key.length + answer.revision.length + current!.length
|
||||
+ Object.values(answer.identity ?? {}).reduce((sum, value) => sum + value.length, 0));
|
||||
forget(key);
|
||||
entries.set(key, { ...answer, keyIdentity: current!, bytes: size });
|
||||
bytes += size;
|
||||
}
|
||||
for (const key of entries.keys()) {
|
||||
if (entries.size <= maxEntries && bytes <= maxBytes) break;
|
||||
forget(key);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user