fix(runtime): await bounded journal proofs with ownership rechecks

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-27 20:32:03 -05:00
1 parent 7d8fd1aed6
commit e44ab40824
7 files changed
+310 -143

No files matched your search

+4 -4
View File
@@ -12269,7 +12269,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
await failedNativeRetryCoordinator(tx, run, source.issueId);
if (retiredOwner) {
if (
!nativePreProviderRetryAfterCleanupStateIsSafe({
!(await nativePreProviderRetryAfterCleanupStateIsSafe({
failedExecution: run.runnerProfileJson?.nativeExecutionInput,
retiredExecution:
retiredOwner.predecessor.runnerProfileJson?.nativeExecutionInput,
@@ -12285,7 +12285,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
processPid: run.processPid!,
processGroupId: run.processGroupId!,
receipt: retiredOwner.receipt,
})
}))
)
throw failedChatRetryDenied();
return;
@@ -12299,7 +12299,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
(checkpoint?.providerSessionId != null &&
(typeof checkpoint.providerSessionId !== "string" ||
checkpoint.providerSessionId.trim().length === 0)) ||
!nativeFailedRunRetryStateIsSafe({
!(await nativeFailedRunRetryStateIsSafe({
execution: run.runnerProfileJson?.nativeExecutionInput,
companyId,
issueId: source.issueId,
@@ -12321,7 +12321,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
| "bootstrap_retry"
| "exact_checkpoint_resume",
allowVerifiedBackup: leases.length > 0,
})
}))
)
throw failedChatRetryDenied();
}
@@ -379,7 +379,7 @@ const support = await getEmbeddedPostgresTestSupport();
"requires exact local cleanup for a new turn after worker loss (%s)", async mode => {
const f = await seed();
await db.update(heartbeatRuns).set({ errorCode: "native_session_cleanup_quarantined" }).where(eq(heartbeatRuns.id, f.sourceRunId));
const retire = vi.fn(() => mode !== "changed");
const retire = vi.fn(async () => mode !== "changed");
const verify = vi.spyOn(nativeExecutor, "verifyStoppedNativeSessionForContinuation").mockResolvedValue(
mode === "unproven" ? null : { evidence: { runId: f.sourceRunId, schema: "paperclip.stopped_native_conversation.v1" }, retire });
try {
@@ -147,7 +147,7 @@ export async function admitExplicitNativeContinuation(input: {
if (pendingInteraction || pendingApproval) return blocked("decision_pending", "A pending approval or question must be resolved before this message can start.");
const sources: Run[] = [];
const stoppedSessions: Array<{ evidence: Record<string, unknown>; retire: () => boolean }> = [];
const stoppedSessions: Array<{ evidence: Record<string, unknown>; retire: () => Promise<boolean> }> = [];
const cancelledStartupIds = new Set<string>();
for (const action of actions) {
const runId = action.evidence.runId ?? action.evidence.sourceRunId;
@@ -267,7 +267,7 @@ export async function admitExplicitNativeContinuation(input: {
summary: null, exposeLowTrustRaw: false });
if (input.dryRun) return { previousRunId: previous.id, commentId, ...(retry ? { failedRunId: input.failedRunId! } : {}) };
for (const stopped of stoppedSessions) {
if (!stopped.retire()) return blocked("local_cleanup", "The previous provider cleanup changed. Your message is saved.");
if (!await stopped.retire()) return blocked("local_cleanup", "The previous provider cleanup changed. Your message is saved.");
await appendHeartbeatRunEvent(db, {
companyId, runId: String(stopped.evidence.runId), agentId,
eventType: "native.stopped_conversation_verified", stream: "system", level: "info",
@@ -79,7 +79,7 @@ export async function hasHistoricalSuspendedNativeSession(db: Db, run: typeof he
if (checkpoint?.providerSessionId != null && (typeof checkpoint.providerSessionId !== "string" ||
!checkpoint.providerSessionId.trim())) return false;
const { nativeFailedRunRetryStateIsSafe } = await import("./native-runtime/native-session-executor.js");
return nativeFailedRunRetryStateIsSafe({
return (await nativeFailedRunRetryStateIsSafe({
execution: run.runnerProfileJson?.nativeExecutionInput,
companyId: run.companyId, issueId: run.nativeIssueId, agentId: run.agentId, runId: run.id,
nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId,
@@ -87,7 +87,7 @@ export async function hasHistoricalSuspendedNativeSession(db: Db, run: typeof he
providerSessionId: typeof checkpoint?.sessionId === "string" ? checkpoint.sessionId : null,
providerBackendSessionId: typeof checkpoint?.providerSessionId === "string" ? checkpoint.providerSessionId : null,
recoveryMode: "exact_checkpoint_resume", allowVerifiedBackup: false,
});
}));
}
/** Recover the exact stopped identity after the mutable run fields were cleared. */
@@ -41,7 +41,7 @@ export async function reconcileSafeNativeReplacements(
options: {
verifyStoppedSession?: (run: typeof heartbeatRuns.$inferSelect) => Promise<{
evidence: Record<string, unknown>;
retire: () => boolean;
retire: () => boolean | Promise<boolean>;
} | null>;
/** Test fault injection at durability boundaries; never exposed by an API. */
failpoint?: (phase: "successor_inserted" | "lineage_committed") => void;
@@ -358,7 +358,7 @@ export async function reconcileSafeNativeReplacements(
}
if (stoppedSession) {
// If the last ownership proof changes, roll back the status restoration.
if (!stoppedSession.retire()) throw new Error("native_replacement_stopped_session_changed");
if (!await stoppedSession.retire()) throw new Error("native_replacement_stopped_session_changed");
await appendHeartbeatRunEvent(tx as unknown as Db, {
companyId: run.companyId, runId: run.id, agentId: run.agentId,
eventType: "native.stopped_text_turn_verified", stream: "system", level: "info",
@@ -1,3 +1,4 @@
import * as nativeJournalProof from "./native-journal-projection-async.js";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
access,
@@ -935,7 +936,7 @@ describe("native incomplete-bootstrap evidence", () => {
};
try {
await writeFile(statePath, JSON.stringify(base));
expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(true);
expect((await runnerdStateProvesIncompleteBootstrap(root))).toBe(true);
for (const ambiguous of [
{ ...base, connectionCount: 1 },
@@ -950,7 +951,7 @@ describe("native incomplete-bootstrap evidence", () => {
},
]) {
await writeFile(statePath, JSON.stringify(ambiguous));
expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(false);
expect((await runnerdStateProvesIncompleteBootstrap(root))).toBe(false);
}
} finally {
await rm(root, { recursive: true, force: true });
@@ -3048,6 +3049,42 @@ describe("retained native cleanup activation", () => {
);
for (const [file, data] of source)
await writeFile(join(quarantine, file), JSON.stringify(data));
if (mode === "empty_root") {
const projectedHistory: Record<string, unknown> & { commands: Array<Record<string, unknown>> } = structuredClone(source[0][1]);
const output = "x".repeat(384 * 1024);
for (let index = 0; index < 24; index++) {
const ordinary = {
type: "semantic_tool.result", status: "completed",
payload: { callId: `read-${index}`, operationId: "paperclip_read", input: { output }, correlation,
sourceEventId: `read-input-${index}`, sourceEventType: "semantic_tool.input", isError: false },
result: { result: { callId: `read-${index}` } },
};
expect(Buffer.byteLength(JSON.stringify(ordinary))).toBeLessThan(1024 * 1024 - 4096);
projectedHistory.commands.push(ordinary);
}
const controlPath = join(quarantine, "control-plane/control-plane-state.json");
const bytes = JSON.stringify(projectedHistory);
expect(Buffer.byteLength(bytes)).toBeGreaterThan(8 * 1024 * 1024);
await writeFile(controlPath, bytes);
try {
const proof = await nativeJournalProof.readNativeJournalProjection(controlPath);
const matches = (control: Record<string, unknown>) => retainedNativeCleanupJournalMatches({
run, execution, accepted, control, providerSessionId: "exact-thread", providerAccountSessionId,
persistedEvents: [identityRow],
});
expect(matches(proof.value as Record<string, unknown>)).toBe(true);
for (const corrupt of ["input", "correlation"] as const) {
const changed = structuredClone(proof.value) as typeof projectedHistory;
const finish = changed.commands.find(command => (command.payload as Record<string, unknown> | undefined)?.operationId === "paperclip_finish")!;
const payload = finish.payload as { input: Record<string, unknown>; correlation: { runId: string } };
if (corrupt === "input") payload.input.summary = "changed";
else payload.correlation.runId = "foreign-run";
expect(matches(changed)).toBe(false);
}
} finally {
await writeFile(controlPath, JSON.stringify(source[0][1]));
}
}
await mkdir(join(quarantine, "codex-home/sessions"), {
recursive: true,
});
@@ -3996,7 +4033,7 @@ describe("stopped native conversation physical cleanup", () => {
expect(proof).not.toBeNull();
if (mode === "changed_state") await writeFile(runnerPath, "{}");
const kill = mode === "changed_pid" ? vi.spyOn(process, "kill").mockReturnValue(true) : null;
try { expect(proof!.retire()).toBe(["stopped", "replacement", "normalized_session_receipt"].includes(mode)); } finally { kill?.mockRestore(); }
try { expect(await proof!.retire()).toBe(["stopped", "replacement", "normalized_session_receipt"].includes(mode)); } finally { kill?.mockRestore(); }
expect(await readFile(join(root, `runner/${provider}-provider-state.json`), "utf8")).toBe(JSON.stringify(providerState));
} else expect(proof).toBeNull();
} finally {
@@ -4021,6 +4058,8 @@ describe("explicit failed native retry physical evidence", () => {
"active_provider",
"ambiguous_provider",
"live_pid",
"owner_started_during_journal_read",
"directory_replaced_during_journal_read",
"symlink",
"bootstrap",
"quarantined_bootstrap",
@@ -4079,6 +4118,8 @@ describe("explicit failed native retry physical evidence", () => {
"distinct_account",
"null_account",
].includes(kind);
let restoreRead: (() => void) | undefined;
let restoreKill: (() => void) | undefined;
try {
const identity = {
runId: execution.binding.runId,
@@ -4142,13 +4183,32 @@ describe("explicit failed native retry physical evidence", () => {
);
}
const before = await readdir(stateBase);
if (kind === "owner_started_during_journal_read" || kind === "directory_replaced_during_journal_read") {
const read = nativeJournalProof.readNativeJournalProjection;
let replacements = 0;
const spy = vi.spyOn(nativeJournalProof, "readNativeJournalProjection").mockImplementation(async (...args) => {
const proof = await read(...args);
if (kind === "owner_started_during_journal_read") {
const kill = vi.spyOn(process, "kill").mockReturnValue(true);
restoreKill = () => kill.mockRestore();
} else {
const control = join(root, "control-plane");
const archived = join(root, `changed-control-${replacements++}`);
await rename(control, archived);
await mkdir(control);
await writeFile(join(control, "control-plane-state.json"), await readFile(join(archived, "control-plane-state.json")));
}
return proof;
});
restoreRead = () => spy.mockRestore();
}
const retryInput = {
execution,
...execution.binding,
nativeSessionId: execution.session.normalizedSessionId!,
runnerInstanceId:
kind === "wrong_runner" ? "another-runner" : "runner-retry",
processPid: kind === "live_pid" ? process.pid : null,
processPid: kind === "live_pid" ? process.pid : kind === "owner_started_during_journal_read" ? 99_999_999 : null,
providerSessionId: expectedThread,
providerBackendSessionId: expectedAccount,
processGroupId: null,
@@ -4158,7 +4218,7 @@ describe("explicit failed native retry physical evidence", () => {
allowVerifiedBackup: false,
};
expect
.soft(nativeFailedRunRetryStateIsSafe(retryInput))
.soft((await nativeFailedRunRetryStateIsSafe(retryInput)))
.toBe(retryable || kind === "bootstrap");
if (!bootstrap && kind !== "symlink") {
const files = [
@@ -4207,26 +4267,26 @@ describe("explicit failed native retry physical evidence", () => {
processGroupId: 99_999_999,
receipt,
};
expect(nativePreProviderRetryAfterCleanupStateIsSafe(input)).toBe(
expect((await nativePreProviderRetryAfterCleanupStateIsSafe(input))).toBe(
retryable,
);
expect(
nativePreProviderRetryAfterCleanupStateIsSafe({
(await nativePreProviderRetryAfterCleanupStateIsSafe({
...input,
receipt: { ...receipt, settledFingerprint: "changed" },
}),
})),
).toBe(false);
expect(
nativePreProviderRetryAfterCleanupStateIsSafe({
(await nativePreProviderRetryAfterCleanupStateIsSafe({
...input,
receipt: { ...receipt, sourceFingerprint: undefined },
}),
})),
).toBe(false);
expect(
nativePreProviderRetryAfterCleanupStateIsSafe({
(await nativePreProviderRetryAfterCleanupStateIsSafe({
...input,
receipt: { ...receipt, requestId: "" },
}),
})),
).toBe(false);
}
expect(await readdir(stateBase)).toEqual(before);
@@ -4235,6 +4295,8 @@ describe("explicit failed native retry physical evidence", () => {
await access(join(root, "control-plane", "control-plane-state.json")),
).toBeUndefined();
} finally {
restoreRead?.();
restoreKill?.();
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous;
await rm(stateBase, { recursive: true, force: true });
@@ -8711,6 +8773,8 @@ describe("runnerd provider runtime wiring", () => {
{ commandCount: 24, fault: null },
{ commandCount: 4, fault: "oversized" },
{ commandCount: 4, fault: "wrong identity" },
{ commandCount: 4, fault: "native_journal_worker_busy" },
{ commandCount: 4, fault: "native_journal_worker_timeout" },
])(
"admits a valid large control-plane journal while preserving bounds and identity ($commandCount commands, $fault)",
async ({ commandCount, fault }) => {
@@ -8744,6 +8808,7 @@ describe("runnerd provider runtime wiring", () => {
status: "succeeded", runnerProfileJson: { nativeExecutionInput: prior },
}]) }) }) }),
} as unknown as Db;
let restoreRead: (() => void) | undefined;
try {
state.createBackend.mockClear();
state.createTransport.mockClear();
@@ -8781,10 +8846,26 @@ describe("runnerd provider runtime wiring", () => {
}
state.createBackend.mockClear();
state.createTransport.mockClear();
if (fault?.startsWith("native_journal_worker_")) {
const read = nativeJournalProof.readNativeJournalProjection;
const spy = vi.spyOn(nativeJournalProof, "readNativeJournalProjection").mockImplementation(async (path, purpose) => {
if (purpose === "identity") throw new Error(fault);
return read(path, purpose);
});
restoreRead = () => spy.mockRestore();
}
const continuation = createRunnerdBackend({
db: priorRunDb, execution: current, runnerInstanceId: "new-heartbeat-runner",
runnerExecutionTarget: remoteTarget,
});
if (fault?.startsWith("native_journal_worker_")) {
await expect(continuation).rejects.toThrow(fault);
expect(state.createBackend).not.toHaveBeenCalled();
expect(state.createTransport).not.toHaveBeenCalled();
expect((await readFile(controlPath)).equals(controlBytes)).toBe(true);
await expect(access(join(stateBase, "quarantine"))).rejects.toThrow();
return;
}
if (fault !== null) {
await expect(continuation).rejects.toThrow("runner_state_identity_mismatch");
expect(state.createBackend).not.toHaveBeenCalled();
@@ -8801,6 +8882,7 @@ describe("runnerd provider runtime wiring", () => {
expect((await readFile(controlPath)).equals(controlBytes)).toBe(true);
await expect(access(join(stateBase, "quarantine"))).rejects.toThrow();
} finally {
restoreRead?.();
if (previousStateDirectory === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
else process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory;
await rm(stateBase, { recursive: true, force: true });
@@ -9025,6 +9107,7 @@ describe("runnerd provider runtime wiring", () => {
"ambiguous turn start",
"state symlink",
"newer provider checkpoint",
"directory replaced during final hash",
])(
"automatically recovers only a proven settled local session: %s",
async (scenario) => {
@@ -9073,6 +9156,7 @@ describe("runnerd provider runtime wiring", () => {
});
}
const onLog = vi.fn(async () => {});
let finalHashSpy: ReturnType<typeof vi.spyOn> | undefined;
const profile = {
nativeExecutionInput:
scenario === "wrong scope"
@@ -9219,6 +9303,20 @@ describe("runnerd provider runtime wiring", () => {
);
};
await writeCandidate(candidate);
if (scenario === "directory replaced during final hash") {
const scan = nativeJournalProof.scanNativeStateFile;
let reads = 0;
finalHashSpy = vi.spyOn(nativeJournalProof, "scanNativeStateFile").mockImplementation(async (...args) => {
const proof = await scan(...args);
if (args[0] === join(candidate, "control-plane", "control-plane-state.json") && ++reads === 2) {
const bytes = await readFile(args[0]);
await rename(join(candidate, "control-plane"), join(candidate, "original-control-plane"));
await mkdir(join(candidate, "control-plane"));
await writeFile(args[0], bytes);
}
return proof;
});
}
if (scenario === "state symlink") {
await rename(
join(candidate, "runner", "runner-state.json"),
@@ -9306,6 +9404,7 @@ describe("runnerd provider runtime wiring", () => {
expect(state.createBackend).not.toHaveBeenCalled();
}
} finally {
finalHashSpy?.mockRestore();
processKill.mockRestore();
if (previousStateDirectory === undefined)
delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
@@ -1,4 +1,4 @@
import { readNativeJournalProjection, scanNativeStateFile } from "./native-journal-projection.js";
import { readNativeJournalProjection, scanNativeStateFile, nativeStateFileIncludes, nativeStateBase64Fingerprint } from "./native-journal-projection-async.js";
import {
isSupportedRemoteCodexVersion,
parseCodexCliVersion,
@@ -1620,17 +1620,19 @@ function legacyRunnerdStateRoot(execution: NativeExecutionInput): string {
);
}
function migrateLegacyRunnerdStateRoot(input: {
async function migrateLegacyRunnerdStateRoot(input: {
legacy: string;
scoped: string;
execution: NativeExecutionInput;
verifiedPriorRunId?: string;
}): string | null {
}): Promise<string | null> {
if (!existsSync(input.legacy)) return null;
if (!isSafeNativeStateDirectory(input.legacy)) {
throw new Error("runner_state_directory_unsafe");
}
const legacyIdentity = readRunnerdDurableIdentity(input.legacy);
const legacyDirectoryIdentity = nativeStateDirectoryIdentity(input.legacy);
const legacyIdentity = (await readRunnerdDurableIdentity(input.legacy));
assertNativeStateDirectoryIdentity(input.legacy, legacyDirectoryIdentity);
const exactRun = durableIdentityMatchesExecution(
legacyIdentity,
input.execution,
@@ -1666,7 +1668,7 @@ function migrateLegacyRunnerdStateRoot(input: {
// directory is authoritative once it exists; otherwise preserve the
// original migration failure rather than silently starting empty state.
if (!isSafeNativeStateDirectory(input.scoped)) throw error;
const scopedIdentity = readRunnerdDurableIdentity(input.scoped);
const scopedIdentity = (await readRunnerdDurableIdentity(input.scoped));
const exactScopedRun = durableIdentityMatchesExecution(
scopedIdentity,
input.execution,
@@ -1782,7 +1784,7 @@ function nativeStateFileMaxBytes(relativePath: string): number {
: NATIVE_RUNNER_STATE_MAX_BYTES;
}
function cleanupStateSnapshot(root: string, providerFile = "codex-provider-state.json") {
async function cleanupStateSnapshot(root: string, providerFile = "codex-provider-state.json") {
if (
![root, resolve(root, "runner"), resolve(root, "control-plane")].every(
isSafeNativeStateDirectory,
@@ -1790,8 +1792,11 @@ function cleanupStateSnapshot(root: string, providerFile = "codex-provider-state
) {
throw new Error("native_cleanup_maintenance_unproven");
}
const directoryIdentities = [root, resolve(root, "runner"), resolve(root, "control-plane")]
.map(path => ({ path, identity: nativeStateDirectoryIdentity(path) }));
const files = [...CLEANUP_CANONICAL_FILES.slice(0, 2), `runner/${providerFile}`];
const controlProof = readNativeJournalProjection(resolve(root, files[0]!));
const controlProof = (await readNativeJournalProjection(resolve(root, files[0]!)));
for (const directory of directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity);
const bytes = files.slice(1).map((file) =>
readBoundedNativeFile(resolve(root, file), NATIVE_RUNNER_STATE_MAX_BYTES, "native_cleanup_maintenance_unproven"),
);
@@ -2392,7 +2397,7 @@ export function rebaseRetainedNativeCleanupProviderHome(
export async function verifyStoppedNativeSessionForContinuation(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
): Promise<{ evidence: Record<string, unknown>; retire: () => boolean } | null> {
): Promise<{ evidence: Record<string, unknown>; retire: () => Promise<boolean> } | null> {
try {
if (run.runtimeMode !== "native" || !["failed", "cancelled", "interrupted", "timed_out"].includes(run.status) ||
!run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null;
@@ -2413,7 +2418,7 @@ export async function verifyStoppedNativeSessionForContinuation(
if (!stopped) return null;
const root = scopedRunnerdStateRoot(execution);
const providerFile = runnerProviderStateFilename(execution);
const snapshot = cleanupStateSnapshot(root, providerFile);
const snapshot = (await cleanupStateSnapshot(root, providerFile));
const identity = record(snapshot.control.identity);
if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId ||
!["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key =>
@@ -2479,14 +2484,14 @@ export async function verifyStoppedNativeSessionForContinuation(
if (!cleanupProcessAbsent(pid)) return null;
providerPids.add(pid);
}
const unchanged = () => idle() && cleanupProcessAbsent(stopped.processPid) &&
[...providerPids].every(cleanupProcessAbsent) && cleanupStateSnapshot(root, providerFile).fingerprint === snapshot.fingerprint;
const unchanged = async () => (await cleanupStateSnapshot(root, providerFile)).fingerprint === snapshot.fingerprint &&
idle() && cleanupProcessAbsent(stopped.processPid) && [...providerPids].every(cleanupProcessAbsent);
return {
evidence: { schema: "paperclip.stopped_native_conversation.v1", runId: run.id,
nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId,
processPid: stopped.processPid, providerPids: [...providerPids], stateFingerprint: snapshot.fingerprint },
retire: () => {
try { return unchanged() && completeTerminatedLocalNativeSessionCleanup({
retire: async () => {
try { return await unchanged() && completeTerminatedLocalNativeSessionCleanup({
companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId!,
}); } catch { return false; }
},
@@ -2500,7 +2505,7 @@ export async function verifyStoppedNativeSessionForContinuation(
export async function verifyStoppedNativeSessionForReplacement(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
): Promise<{ evidence: Record<string, unknown>; retire: () => boolean } | null> {
): Promise<{ evidence: Record<string, unknown>; retire: () => Promise<boolean> } | null> {
try {
if (run.runtimeMode !== "native" || run.status !== "failed" || !run.finishedAt ||
!run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null;
@@ -2520,7 +2525,7 @@ export async function verifyStoppedNativeSessionForReplacement(
const stopped = await readNativeLocalProcessStop(db, run.companyId, run.id);
if (!stopped) return null;
const root = scopedRunnerdStateRoot(execution);
const snapshot = cleanupStateSnapshot(root);
const snapshot = (await cleanupStateSnapshot(root));
const identity = record(snapshot.control.identity);
if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId ||
!["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key =>
@@ -2609,8 +2614,8 @@ export async function verifyStoppedNativeSessionForReplacement(
completedTaskControlCallIds: completedTaskControlCalls.map(call => call.callId) };
return {
evidence,
retire: () => idle() && cleanupProcessAbsent(stopped.processPid) && cleanupProcessAbsent(provider.processId) &&
cleanupStateSnapshot(root).fingerprint === snapshot.fingerprint &&
retire: async () => (await cleanupStateSnapshot(root)).fingerprint === snapshot.fingerprint &&
idle() && cleanupProcessAbsent(stopped.processPid) && cleanupProcessAbsent(provider.processId) &&
nativeSha256(readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven").toString("utf8")) === rolloutSha256 &&
completeTerminatedLocalNativeSessionCleanup({ companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId! }),
};
@@ -2647,7 +2652,7 @@ export async function reconcileRetainedNativeSessionCleanup(
quarantine: string;
root: string;
emptyRoot: ReturnType<typeof cleanupCanonicalVacancy>;
source: ReturnType<typeof cleanupStateSnapshot>;
source: Awaited<ReturnType<typeof cleanupStateSnapshot>>;
providerHome: ReturnType<typeof cleanupProviderHomeSnapshot>;
sourceArchive: {
intent: Record<string, unknown>;
@@ -2656,7 +2661,7 @@ export async function reconcileRetainedNativeSessionCleanup(
} | null;
copySource: {
directory: string;
snapshot: ReturnType<typeof cleanupStateSnapshot>;
snapshot: Awaited<ReturnType<typeof cleanupStateSnapshot>>;
requestId: string;
} | null;
providerPid: number;
@@ -2887,7 +2892,7 @@ export async function reconcileRetainedNativeSessionCleanup(
)
return null;
}
const source = cleanupStateSnapshot(sourceDirectory);
const source = (await cleanupStateSnapshot(sourceDirectory));
const identity = record(source.control.identity);
if (
!durableIdentityMatchesExecution(identity, execution) ||
@@ -2975,7 +2980,7 @@ export async function reconcileRetainedNativeSessionCleanup(
return null;
let copySource: {
directory: string;
snapshot: ReturnType<typeof cleanupStateSnapshot>;
snapshot: Awaited<ReturnType<typeof cleanupStateSnapshot>>;
requestId: string;
} | null = null;
const providerHome = cleanupProviderHomeSnapshot(
@@ -3032,7 +3037,7 @@ export async function reconcileRetainedNativeSessionCleanup(
})
)
return null;
const attempted = cleanupStateSnapshot(directory);
const attempted = (await cleanupStateSnapshot(directory));
const receipts = await tx
.select()
.from(heartbeatRunEvents)
@@ -3173,7 +3178,7 @@ export async function reconcileRetainedNativeSessionCleanup(
const authorize = async () => {
await assertLease();
if (
cleanupStateSnapshot(owned.quarantine).fingerprint !==
(await cleanupStateSnapshot(owned.quarantine)).fingerprint !==
owned.source.fingerprint ||
cleanupProviderHomeSnapshot(
resolve(owned.quarantine, "codex-home"),
@@ -3181,7 +3186,7 @@ export async function reconcileRetainedNativeSessionCleanup(
).metadataFingerprint !== owned.providerHome.metadataFingerprint ||
(owned.copySource &&
(!retainedRunnerdMaintenanceIsIdle(owned.copySource.directory) ||
cleanupStateSnapshot(owned.copySource.directory).fingerprint !==
(await cleanupStateSnapshot(owned.copySource.directory)).fingerprint !==
owned.copySource.snapshot.fingerprint ||
lstatSync(
resolve(owned.copySource.directory, CLEANUP_ACTIVATION_FILE),
@@ -3191,6 +3196,7 @@ export async function reconcileRetainedNativeSessionCleanup(
canonicalJson(owned.emptyRoot)
)
throw denied();
await assertLease();
};
const appendMaintenanceHistory = async (entry: Record<string, unknown>) => {
const history = await db.transaction(async (tx) => {
@@ -3237,7 +3243,7 @@ export async function reconcileRetainedNativeSessionCleanup(
if (
canonicalJson(cleanupArchiveRootIdentity(sourceDirectory)) !==
canonicalJson(archive.intent.rootIdentity) ||
cleanupStateSnapshot(sourceDirectory).fingerprint !==
(await cleanupStateSnapshot(sourceDirectory)).fingerprint !==
owned.source.fingerprint ||
cleanupProviderHomeSnapshot(
resolve(sourceDirectory, "codex-home"),
@@ -3248,6 +3254,7 @@ export async function reconcileRetainedNativeSessionCleanup(
: existsSync(owned.root))
)
throw denied();
await assertLease();
if (archive.fromCanonical) {
const parent = dirname(owned.quarantine);
mkdirSync(parent, { recursive: true, mode: 0o700 });
@@ -3262,7 +3269,7 @@ export async function reconcileRetainedNativeSessionCleanup(
true,
);
if (
cleanupStateSnapshot(owned.quarantine).fingerprint !==
(await cleanupStateSnapshot(owned.quarantine)).fingerprint !==
owned.source.fingerprint ||
archivedHome.fingerprint !== owned.providerHome.fingerprint ||
canonicalJson(cleanupArchiveRootIdentity(owned.quarantine)) !==
@@ -3685,7 +3692,7 @@ async function assertCleanupActivationCommitted(
entry.kind === "native_cleanup_maintenance" &&
entry.requestId === marker.requestId,
);
const snapshot = cleanupStateSnapshot(root);
const snapshot = (await cleanupStateSnapshot(root));
if (
marker.settledProviderHomeFingerprint !== undefined ||
receipt?.settledProviderHomeFingerprint !== undefined
@@ -3725,10 +3732,34 @@ async function assertCleanupActivationCommitted(
rmSync(path);
}
function failedNativeRetryOwnersStopped(execution: NativeExecutionInput, processPid: number | null, processGroupId: number | null): boolean {
const scope = nativeSessionScopeKey(execution);
if (
executingRunnerdSessionScopes.has(scope) ||
initializingSessionToolAuthorities.has(scope) ||
warmNativeSessions.has(scope)
)
return false;
for (const [id, group] of [
[processPid, false],
[processGroupId, true],
] as const) {
if (id === null) continue;
if (!Number.isSafeInteger(id) || id <= 0) return false;
try {
process.kill(group ? -id : id, 0);
return false;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") return false;
}
}
return true;
}
/** Read-only admission evidence for an explicit retry of a terminal failed
* run. Never migrate/archive state, release an owner, or contact a provider.
* Normal executor admission independently verifies the state again. */
export function nativeFailedRunRetryStateIsSafe(input: {
export async function nativeFailedRunRetryStateIsSafe(input: {
execution: unknown;
companyId: string;
issueId: string;
@@ -3742,7 +3773,7 @@ export function nativeFailedRunRetryStateIsSafe(input: {
processGroupId: number | null;
recoveryMode: "bootstrap_retry" | "exact_checkpoint_resume";
allowVerifiedBackup: boolean;
}): boolean {
}): Promise<boolean> {
try {
const execution = parseNativeExecutionInput(input.execution);
if (
@@ -3754,26 +3785,7 @@ export function nativeFailedRunRetryStateIsSafe(input: {
!input.runnerInstanceId
)
return false;
const scope = nativeSessionScopeKey(execution);
if (
executingRunnerdSessionScopes.has(scope) ||
initializingSessionToolAuthorities.has(scope) ||
warmNativeSessions.has(scope)
)
return false;
for (const [id, group] of [
[input.processPid, false],
[input.processGroupId, true],
] as const) {
if (id === null) continue;
if (!Number.isSafeInteger(id) || id <= 0) return false;
try {
process.kill(group ? -id : id, 0);
return false;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") return false;
}
}
if (!failedNativeRetryOwnersStopped(execution, input.processPid, input.processGroupId)) return false;
const root = scopedRunnerdStateRoot(execution);
if (!lstatSync(root, { throwIfNoEntry: false })) {
if (
@@ -3812,7 +3824,9 @@ export function nativeFailedRunRetryStateIsSafe(input: {
}
return true;
}
const identity = readRunnerdDurableIdentity(root);
const identity = (await readRunnerdDurableIdentity(root));
// Worker reads yield: a new in-process owner or recycled PID invalidates the earlier stop proof.
if (!failedNativeRetryOwnersStopped(execution, input.processPid, input.processGroupId)) return false;
if (
!durableIdentityMatchesSession(identity, execution) ||
!durableIdentityMatchesExecution(identity, execution) ||
@@ -3895,7 +3909,7 @@ export function nativeFailedRunRetryStateIsSafe(input: {
/** Physical half of retrying a request rejected before provider admission.
* The caller separately proves the failed receipt/coordinator has no native
* events or result and selects exactly one committed cleanup owner. */
export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
export async function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
failedExecution: unknown;
retiredExecution: unknown;
companyId: string;
@@ -3910,7 +3924,7 @@ export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
processPid: number;
processGroupId: number;
receipt: Record<string, unknown>;
}): boolean {
}): Promise<boolean> {
try {
const failed = parseNativeExecutionInput(input.failedExecution);
const retired = parseNativeExecutionInput(input.retiredExecution);
@@ -3934,7 +3948,7 @@ export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
!/^[a-f0-9]{64}$/.test(input.receipt.sourceFingerprint) ||
typeof input.receipt.settledFingerprint !== "string" ||
!/^[a-f0-9]{64}$/.test(input.receipt.settledFingerprint) ||
!nativeFailedRunRetryStateIsSafe({
!(await nativeFailedRunRetryStateIsSafe({
execution: retired,
companyId: input.companyId,
issueId: input.issueId,
@@ -3948,7 +3962,7 @@ export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
providerBackendSessionId: input.providerBackendSessionId,
recoveryMode: "exact_checkpoint_resume",
allowVerifiedBackup: false,
})
}))
)
return false;
const root = scopedRunnerdStateRoot(retired);
@@ -3978,8 +3992,9 @@ export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
return false;
}
return (
cleanupStateSnapshot(root).fingerprint ===
input.receipt.settledFingerprint
(await cleanupStateSnapshot(root)).fingerprint ===
input.receipt.settledFingerprint &&
failedNativeRetryOwnersStopped(retired, input.processPid, input.processGroupId)
);
} catch {
return false;
@@ -4056,7 +4071,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: {
}
}
function hasRetainedWarmTransitionEvidence(root: string): boolean {
async function hasRetainedWarmTransitionEvidence(root: string): Promise<boolean> {
for (const [directory, filename, maximum] of [
["control-plane", "control-plane-state.json", DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES],
["runner", "runner-state.json", NATIVE_RUNNER_STATE_MAX_BYTES],
@@ -4065,23 +4080,21 @@ function hasRetainedWarmTransitionEvidence(root: string): boolean {
if (!lstatSync(path, { throwIfNoEntry: false })) continue;
try {
const state = record(directory === "control-plane"
? readNativeJournalProjection(path).value
? (await readNativeJournalProjection(path)).value
: JSON.parse(readBoundedNativeFile(path, maximum, "runner_state_too_large").toString("utf8")));
if (Object.prototype.hasOwnProperty.call(state, "warmTransition")
|| state.schema === "paperclip.runner.durable.control-plane-state.warm-transition.v1"
|| state.schema === "paperclip.runner.durable.state.warm-transition.v1") return true;
} catch {
} catch (error) {
if (nativeJournalProofUnavailable(error)) throw error;
// Detection is never admission. Preserve the malformed-forward-receipt
// fence without materializing its potentially large discarded history.
let tail = "", found = false;
try {
scanNativeStateFile(path, maximum, chunk => {
const text = tail + chunk.toString("utf8");
if (text.includes("warm-transition.v1") || text.includes('"warmTransition"')) found = true;
tail = text.slice(-32);
});
if (found) return true;
} catch { /* The ordinary verifier owns unreadable legacy state. */ }
if (await nativeStateFileIncludes(path, maximum, ["warm-transition.v1", '\"warmTransition\"'])) return true;
} catch (error) {
if (nativeJournalProofUnavailable(error)) throw error;
// The ordinary verifier owns unreadable legacy state.
}
}
}
return false;
@@ -4094,7 +4107,7 @@ type VerifiedWarmTransitionBinding = {
stateFingerprint: string;
};
function readWarmTransitionSnapshot(root: string) {
async function readWarmTransitionSnapshot(root: string) {
if (
![root, resolve(root, "control-plane"), resolve(root, "runner")].every(
isSafeNativeStateDirectory,
@@ -4102,20 +4115,23 @@ function readWarmTransitionSnapshot(root: string) {
) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
const directoryIdentities = [root, resolve(root, "runner"), resolve(root, "control-plane")]
.map(path => ({ path, identity: nativeStateDirectoryIdentity(path) }));
try {
const corePath = resolve(root, "control-plane", "control-plane-state.json");
const runnerPath = resolve(root, "runner", "runner-state.json");
const core = readNativeJournalProjection(corePath);
const core = (await readNativeJournalProjection(corePath));
const runner = readBoundedNativeFile(runnerPath, NATIVE_RUNNER_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven");
// Preserve the original canonical JSON [base64(core), base64(runner)] hash.
const digest = createHash("sha256").update('["');
const rawCore = scanNativeStateFile(corePath, DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES, chunk => { digest.update(chunk.toString("base64")); });
digest.update('","');
const rawRunner = scanNativeStateFile(runnerPath, NATIVE_RUNNER_STATE_MAX_BYTES, chunk => { digest.update(chunk.toString("base64")); });
digest.update('"]');
if (rawCore.sha256 !== core.sha256 || rawRunner.sha256 !== createHash("sha256").update(runner).digest("hex")) throw new Error("native_state_file_changed");
return { controlPlaneState: core.value, runnerState: JSON.parse(runner.toString("utf8")) as unknown, stateFingerprint: digest.digest("hex") };
} catch {
// The worker preserves the original canonical JSON [base64(core), base64(runner)] hash.
const fingerprint = await nativeStateBase64Fingerprint([
{ path: corePath, maximum: DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES },
{ path: runnerPath, maximum: NATIVE_RUNNER_STATE_MAX_BYTES },
]);
for (const directory of directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity);
if (fingerprint.fileSha256[0] !== core.sha256 || fingerprint.fileSha256[1] !== createHash("sha256").update(runner).digest("hex")) throw new Error("native_state_file_changed");
return { controlPlaneState: core.value, runnerState: JSON.parse(runner.toString("utf8")) as unknown, stateFingerprint: fingerprint.sha256 };
} catch (error) {
if (nativeJournalProofUnavailable(error)) throw error;
throw new Error("native_runner_warm_transition_recovery_unproven");
}
}
@@ -4143,11 +4159,11 @@ async function verifyWarmTransitionRestart(input: {
)
return deny();
const root = scopedRunnerdStateRoot(input.execution);
const snapshot = readWarmTransitionSnapshot(root);
const snapshot = (await readWarmTransitionSnapshot(root));
const artifact = readRunnerdArtifactBinding(resolvePaperclipRunnerBinary());
const controller = await currentNativeControllerIdentity();
const binding = input.execution.binding;
let finalAuthorityCheck: (() => boolean) | undefined;
let finalAuthorityCheck: (() => Promise<boolean>) | undefined;
const verified = await input.db.transaction(async (tx) => {
const current = await tx
.select()
@@ -4439,9 +4455,9 @@ async function verifyWarmTransitionRestart(input: {
root,
input.execution,
);
finalAuthorityCheck = () => {
finalAuthorityCheck = async () => {
if (
readWarmTransitionSnapshot(root).stateFingerprint !==
(await readWarmTransitionSnapshot(root)).stateFingerprint !==
snapshot.stateFingerprint
)
return false;
@@ -4463,7 +4479,7 @@ async function verifyWarmTransitionRestart(input: {
finalProof.receipt.leaseExpiresAtUnixMs > Date.now()
);
};
if (!finalAuthorityCheck()) return deny();
if (!await finalAuthorityCheck()) return deny();
return {
runnerInstanceId: current.runnerInstanceId,
environmentLeaseId: expectedEnvironmentLeaseId,
@@ -4473,7 +4489,7 @@ async function verifyWarmTransitionRestart(input: {
});
// A lock wait or the transaction completion itself can outlive a lease.
// Never carry a timestamp sampled before those awaits into admission.
if (!finalAuthorityCheck?.()) return deny();
if (!await finalAuthorityCheck?.()) return deny();
return verified;
}
@@ -4510,7 +4526,10 @@ async function migrateRunnerdStateRootForExecution(input: {
if (!isSafeNativeStateDirectory(scoped)) {
throw new Error("runner_state_directory_unsafe");
}
if (hasRetainedWarmTransitionEvidence(scoped)) {
const scopedDirectoryIdentity = nativeStateDirectoryIdentity(scoped);
const retainedWarmTransition = await hasRetainedWarmTransitionEvidence(scoped);
assertNativeStateDirectoryIdentity(scoped, scopedDirectoryIdentity);
if (retainedWarmTransition) {
// A copied receipt is not restart/process authority. Keep both valid
// unsupported and invalid forward evidence out of destructive legacy
// migration until the exact transition admission path proves its owner.
@@ -4518,7 +4537,8 @@ async function migrateRunnerdStateRootForExecution(input: {
return verified;
}
await assertCleanupActivationCommitted(input.db, scoped, input.execution);
const identity = readRunnerdDurableIdentity(scoped);
const identity = (await readRunnerdDurableIdentity(scoped));
assertNativeStateDirectoryIdentity(scoped, scopedDirectoryIdentity);
if (!identity) {
if (input.restartRecovery?.kind !== "reattach_existing_runner" &&
input.restartRecovery?.kind !== "reattach_remote_runner") {
@@ -4534,7 +4554,9 @@ async function migrateRunnerdStateRootForExecution(input: {
throw new Error("runner_state_identity_mismatch");
}
if (input.restartRecovery?.kind === "bootstrap_incomplete") {
if (runnerdStateProvesIncompleteBootstrap(scoped)) {
const incompleteBootstrap = await runnerdStateProvesIncompleteBootstrap(scoped);
assertNativeStateDirectoryIdentity(scoped, scopedDirectoryIdentity);
if (incompleteBootstrap) {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
return;
}
@@ -4573,6 +4595,7 @@ async function migrateRunnerdStateRootForExecution(input: {
allowVerifiedBackup: input.allowVerifiedBackup,
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
});
assertNativeStateDirectoryIdentity(scoped, scopedDirectoryIdentity);
if (
verification !== "verified" &&
verification !== "retained_warm_runner"
@@ -4595,7 +4618,9 @@ async function migrateRunnerdStateRootForExecution(input: {
legacyRunnerdStateRoot(input.execution),
]) {
if (!existsSync(legacy)) continue;
const identity = readRunnerdDurableIdentity(legacy);
const legacyDirectoryIdentity = nativeStateDirectoryIdentity(legacy);
const identity = (await readRunnerdDurableIdentity(legacy));
assertNativeStateDirectoryIdentity(legacy, legacyDirectoryIdentity);
if (
!identity ||
!durableIdentityMatchesSession(identity, input.execution)
@@ -4615,6 +4640,7 @@ async function migrateRunnerdStateRootForExecution(input: {
allowVerifiedBackup: input.allowVerifiedBackup,
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
});
assertNativeStateDirectoryIdentity(legacy, legacyDirectoryIdentity);
if (
verification !== "verified" &&
verification !== "retained_warm_runner"
@@ -4630,12 +4656,12 @@ async function migrateRunnerdStateRootForExecution(input: {
}
verifiedPriorRunId = identity.runId;
}
migrateLegacyRunnerdStateRoot({
(await migrateLegacyRunnerdStateRoot({
legacy,
scoped,
execution: input.execution,
...(verifiedPriorRunId ? { verifiedPriorRunId } : {}),
});
}));
return;
}
}
@@ -4687,13 +4713,17 @@ async function recoverQuiescentRunnerdState(input: {
runId: string;
processPid: number;
processGroupId: number;
providerProcessIds: unknown[];
directoryIdentities: Array<{ path: string; identity: NativeStateDirectoryIdentity }>;
}> = [];
for (const root of candidates) {
if (!isSafeNativeStateDirectory(root)) continue;
const identity = readRunnerdDurableIdentity(root);
const identity = (await readRunnerdDurableIdentity(root));
if (!durableIdentityMatchesSession(identity, input.execution)) continue;
if (identity.runId === input.execution.binding.runId) continue;
try {
const directoryIdentities = [root, resolve(root, "runner"), resolve(root, "control-plane")]
.map(path => ({ path, identity: nativeStateDirectoryIdentity(path) }));
const readState = (directory: string, name: string) => {
if (!isSafeNativeStateDirectory(resolve(root, directory)))
throw new Error("unsafe_recovery_state");
@@ -4704,7 +4734,7 @@ async function recoverQuiescentRunnerdState(input: {
).toString("utf8");
};
const runnerBytes = readState("runner", "runner-state.json");
const controlProof = readNativeJournalProjection(resolve(root, "control-plane", "control-plane-state.json"));
const controlProof = (await readNativeJournalProjection(resolve(root, "control-plane", "control-plane-state.json")));
const controlSha256 = controlProof.sha256;
const providerBytes = readState("runner", "codex-provider-state.json");
const runner = record(JSON.parse(runnerBytes));
@@ -4830,10 +4860,12 @@ async function recoverQuiescentRunnerdState(input: {
// No mutation if any evidence changed while the database was read.
if (
runnerBytes !== readState("runner", "runner-state.json") ||
controlSha256 !== scanNativeStateFile(resolve(root, "control-plane", "control-plane-state.json"), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES).sha256 ||
controlSha256 !== (await scanNativeStateFile(resolve(root, "control-plane", "control-plane-state.json"), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES)).sha256 ||
providerBytes !== readState("runner", "codex-provider-state.json")
)
continue;
for (const directory of directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity);
if (processIds.some(pid => !localProcessDefinitelyGone(pid))) continue;
verified.push({
root,
runnerBytes,
@@ -4843,8 +4875,11 @@ async function recoverQuiescentRunnerdState(input: {
runId: identity.runId,
processPid: prior.processPid!,
processGroupId: prior.processGroupId!,
providerProcessIds: processIds,
directoryIdentities,
});
} catch {
} catch (error) {
if (nativeJournalProofUnavailable(error)) throw error;
// Unreadable, unsafe, unscoped, or unprovable state stays quarantined.
}
}
@@ -4872,15 +4907,17 @@ async function recoverQuiescentRunnerdState(input: {
]) {
if (
(relativePath === "control-plane/control-plane-state.json"
? scanNativeStateFile(resolve(candidate.root, relativePath), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES).sha256
? (await scanNativeStateFile(resolve(candidate.root, relativePath), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES)).sha256
: readBoundedNativeFile(resolve(candidate.root, relativePath!), NATIVE_RUNNER_STATE_MAX_BYTES, "recovery_state_too_large").toString("utf8")) !== expected
) {
throw new Error("runner_state_identity_mismatch");
}
}
for (const directory of candidate.directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity);
if (
!localProcessDefinitelyGone(candidate.processPid) ||
!localProcessDefinitelyGone(candidate.processGroupId, true)
!localProcessDefinitelyGone(candidate.processGroupId, true) ||
candidate.providerProcessIds.some(pid => !localProcessDefinitelyGone(pid))
) {
throw new Error("runner_state_identity_mismatch");
}
@@ -4923,14 +4960,14 @@ function localProcessDefinitelyGone(value: unknown, group = false): boolean {
}
}
export function runnerdStateProvesIncompleteBootstrap(root: string): boolean {
export async function runnerdStateProvesIncompleteBootstrap(root: string): Promise<boolean> {
try {
const statePath = resolve(
root,
"control-plane",
"control-plane-state.json",
);
const state = record(readNativeJournalProjection(statePath).value);
const state = record((await readNativeJournalProjection(statePath)).value);
const commands = Array.isArray(state.commands)
? state.commands.map(record)
: [];
@@ -4948,11 +4985,29 @@ export function runnerdStateProvesIncompleteBootstrap(root: string): boolean {
committedEvents.length === 0 &&
onlyUnconsumedBootstrapCommands
);
} catch {
} catch (error) {
if (nativeJournalProofUnavailable(error)) throw error;
return false;
}
}
function nativeJournalProofUnavailable(error: unknown): boolean {
return error instanceof Error && (error.message.startsWith("native_journal_worker_")
|| error.message === "native_state_file_changed"
|| error.message === "native_journal_projection_budget_exceeded");
}
type NativeStateDirectoryIdentity = { dev: number; ino: number };
function nativeStateDirectoryIdentity(path: string): NativeStateDirectoryIdentity {
const stat = lstatSync(path);
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("runner_state_directory_unsafe");
return { dev: stat.dev, ino: stat.ino };
}
function assertNativeStateDirectoryIdentity(path: string, expected: NativeStateDirectoryIdentity): void {
const current = nativeStateDirectoryIdentity(path);
if (current.dev !== expected.dev || current.ino !== expected.ino) throw new Error("runner_state_directory_unsafe");
}
function isSafeNativeStateDirectory(path: string): boolean {
if (!existsSync(path)) return false;
const stats = lstatSync(path);
@@ -4966,20 +5021,28 @@ type RunnerdDurableIdentity = Record<string, unknown> & {
environmentLeaseId: string;
};
function readRunnerdDurableIdentity(
async function readRunnerdDurableIdentity(
root: string,
): Record<string, unknown> | null {
): Promise<Record<string, unknown> | null> {
if (!isSafeNativeStateDirectory(root)) return null;
const controlPlaneRoot = resolve(root, "control-plane");
if (!isSafeNativeStateDirectory(controlPlaneRoot)) return null;
const statePath = resolve(controlPlaneRoot, "control-plane-state.json");
if (!existsSync(statePath)) return null;
const rootIdentity = nativeStateDirectoryIdentity(root);
const controlIdentity = nativeStateDirectoryIdentity(controlPlaneRoot);
try {
const state = record(readNativeJournalProjection(statePath, "identity").value);
const state = record((await readNativeJournalProjection(statePath, "identity")).value);
if (state.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA) return null;
return record(state.identity);
} catch {
} catch (error) {
// Resource pressure or a changing snapshot is not corrupt durable identity.
if (nativeJournalProofUnavailable(error)) throw error;
return null;
} finally {
// A replaced directory is unsafe, not merely missing identity that can be quarantined.
assertNativeStateDirectoryIdentity(root, rootIdentity);
assertNativeStateDirectoryIdentity(controlPlaneRoot, controlIdentity);
}
}
@@ -5016,7 +5079,7 @@ function durableIdentityMatchesSession(
* session scope. Ambiguous company/session-only state can never be claimed by
* another agent, workspace, or provider profile.
*/
function runnerdStateRoot(execution: NativeExecutionInput): string {
async function runnerdStateRoot(execution: NativeExecutionInput): Promise<string> {
const scoped = scopedRunnerdStateRoot(execution);
if (existsSync(scoped)) {
if (!isSafeNativeStateDirectory(scoped)) {
@@ -5028,33 +5091,36 @@ function runnerdStateRoot(execution: NativeExecutionInput): string {
legacyCompanyRunnerdStateRoot(execution),
legacyRunnerdStateRoot(execution),
]) {
const migrated = migrateLegacyRunnerdStateRoot({
const migrated = (await migrateLegacyRunnerdStateRoot({
legacy,
scoped,
execution,
});
if (migrated) return migrated;
}));
if (migrated) {
nativeStateDirectoryIdentity(migrated);
return migrated;
}
}
return scoped;
}
function loadRunnerdDurableBinding(
async function loadRunnerdDurableBinding(
execution: NativeExecutionInput,
transition?: VerifiedWarmTransitionBinding,
): {
): Promise<{
runnerInstanceId: string;
environmentLeaseId: string;
} | null {
} | null> {
if (transition) {
if (
readWarmTransitionSnapshot(runnerdStateRoot(execution))
(await readWarmTransitionSnapshot((await runnerdStateRoot(execution))))
.stateFingerprint !== transition.stateFingerprint
) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
return transition;
}
const identity = readRunnerdDurableIdentity(runnerdStateRoot(execution));
const identity = (await readRunnerdDurableIdentity((await runnerdStateRoot(execution))));
// The run id is intentionally different during a continuation. Reuse only
// the verified runner/lease binding from the same company-scoped durable
// session root; rotateLocalAuthorityEpoch then requires that exact binding
@@ -7318,7 +7384,7 @@ async function executePaperclipNativeSessionWithinScope(
});
}
const durableRunnerBinding = input.useRunnerd
? loadRunnerdDurableBinding(input.execution, retainedTransition)
? (await loadRunnerdDurableBinding(input.execution, retainedTransition))
: null;
const effectiveRunnerInstanceId =
durableRunnerBinding?.runnerInstanceId ?? input.runnerInstanceId;
@@ -10385,7 +10451,7 @@ export async function createRunnerdBackend(input: {
if (
executingRunnerdSessionScopes.get(sessionScopeId) !==
input.execution.binding.runId ||
hasRetainedWarmTransitionEvidence(scopedRunnerdStateRoot(input.execution))
(await hasRetainedWarmTransitionEvidence(scopedRunnerdStateRoot(input.execution)))
) {
retainedTransition = await migrateRunnerdStateRootForExecution({
db: input.db,
@@ -10510,10 +10576,10 @@ async function createRunnerdBackendWithinSessionClaim(
authorityEpoch.revoke();
throw error;
}
const root = runnerdStateRoot(input.execution);
const durableIdentity = readRunnerdDurableIdentity(root);
const root = (await runnerdStateRoot(input.execution));
const durableIdentity = (await readRunnerdDurableIdentity(root));
const durableBinding = retainedTransition
? loadRunnerdDurableBinding(input.execution, retainedTransition)
? (await loadRunnerdDurableBinding(input.execution, retainedTransition))
: durableIdentityMatchesSession(durableIdentity, input.execution)
? durableIdentity
: null;
@@ -11307,11 +11373,11 @@ async function createRunnerdBackendWithinSessionClaim(
const claimUntouchedSessionInResumedLease = async (): Promise<boolean> => {
if (!remoteCommandRunner || !remoteRuntimeRoot || !remoteSessionRoot) return false;
const identity = readRunnerdDurableIdentity(root);
const identity = (await readRunnerdDurableIdentity(root));
if (!durableIdentityMatchesExecution(identity, input.execution) ||
identity?.runnerInstanceId !== effectiveRunnerInstanceId ||
identity?.environmentLeaseId !== effectiveEnvironmentLeaseId ||
!runnerdStateProvesIncompleteBootstrap(root)) return false;
!(await runnerdStateProvesIncompleteBootstrap(root))) return false;
// A reusable workspace may have failed before any harness was created.
// Claim this exact new session atomically under readable real directories.
// Missing files inside an existing session never authorize a fresh start.
@@ -12064,9 +12130,11 @@ async function createRunnerdBackendWithinSessionClaim(
PAPERCLIP_WORKSPACE_CWD: input.execution.workspace.cwd,
};
const archiveContinuityState = async () => {
if (hasRetainedWarmTransitionEvidence(root)) {
const rootIdentity = nativeStateDirectoryIdentity(root);
if ((await hasRetainedWarmTransitionEvidence(root))) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
assertNativeStateDirectoryIdentity(root, rootIdentity);
const archiveToken = `${Date.now()}-${randomUUID()}`;
const archiveRoot = resolve(root, "continuity-breaks", archiveToken);
mkdirSync(archiveRoot, { recursive: true, mode: 0o700 });