feat(runner): persist controller authority and bounded delivery

Part 3 of the opt-in indexed history draft series. Validated at the integrated tip; activation remains disabled by default.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-29 16:20:49 -05:00
1 parent a4720aa56f
commit f2f631dad2
84 files changed
+7225 -669

No files matched your search

@@ -11,6 +11,7 @@
"session": "Provider-neutral session open, resume, reconciliation, close, and failure.",
"turn": "Model turn submission through terminal turn disposition.",
"item": "Provider-neutral model/tool item lifecycle.",
"output": "Negotiated, immutable output-body chunks with exact byte offsets and content digests.",
"mcp_app": "MCP App discovery, initialization, tool, action, host-context, and teardown evidence.",
"runtime_request": "Runtime permission/input request lifecycle.",
"interaction": "Issue-thread interaction proposal, materialization, response, delivery, and rejection.",
@@ -6,7 +6,7 @@ Status: canonical explanatory contract for the Paperclip runner V1 surface.
This document keeps three independent meanings of **group** separate. PRP families describe wire evidence and controller commands; capability placement decides who owns an operation; behavioral eval groups organize the 106 scenario corpus. None of the three axes can be used as a substitute for another.
The generated totals are **105 PRP events in 31 event families**, **18 controller commands in 7 command families**, **10 control-plane operations**, **48 reconciled semantic operations** (14 always, 34 optional), and **106 scenarios in 16 behavior groups**.
The generated totals are **106 PRP events in 32 event families**, **18 controller commands in 7 command families**, **10 control-plane operations**, **48 reconciled semantic operations** (14 always, 34 optional), and **106 scenarios in 16 behavior groups**.
## Axis 1: PRP v1 event and command families
@@ -37,6 +37,7 @@ PRP records ordered, replayable execution evidence. It is not the model's Paperc
| `provider` | Redacted provider notices and actionable warnings. | `provider.notice.recorded` | 1 |
| `session` | Provider-neutral session open, resume, reconciliation, close, and failure. | `session.starting`<br>`session.started`<br>`session.resuming`<br>`session.resumed`<br>`session.reconciled`<br>`session.updated`<br>`session.closed`<br>`session.failed` | 8 |
| `turn` | Model turn submission through terminal turn disposition. | `turn.submitted`<br>`turn.accepted`<br>`turn.started`<br>`turn.completed`<br>`turn.failed`<br>`turn.interrupted`<br>`turn.cancelled` | 7 |
| `output` | Negotiated, immutable output-body chunks with exact byte offsets and content digests. | `output.body.chunk` | 1 |
| `item` | Provider-neutral model/tool item lifecycle. | `item.started`<br>`item.delta`<br>`item.completed`<br>`item.failed` | 4 |
| `usage` | Provider/model-attributed usage and accounting boundaries. | `usage.reported` | 1 |
| `semantic_tool` | Canonical authorized Paperclip tool input and result evidence. | `semantic_tool.input`<br>`semantic_tool.result`<br>`semantic_tool.reconciled` | 3 |
@@ -4,6 +4,7 @@ import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../
import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js";
import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js";
import { HarnessDriverBackend } from "./harness-driver-backend.js";
import { authorityJson } from "../control-plane/durable-authority-store.js";
const result: PrpStructuredRunResult = {
schema: "paperclip.run_result.v1",
@@ -97,6 +98,26 @@ const driver: HarnessDriver = {
};
describe("HarnessDriverBackend", () => {
it.each([false, true])("recovers a committed terminal after delivery ACK and validates its binding (conflict: %s)", async (conflict) => {
class DurableTerminalSession extends FakeHarnessSession {
override async *events(): AsyncIterable<PrpEvent> { /* Already acknowledged before the crash. */ }
override async snapshot(): Promise<PersistedHarnessSession> {
return {
...(await super.snapshot()), activeTurnId: null,
semanticResult: { result, fingerprint: authorityJson(result), turnId: "turn-1" },
durableTerminal: { event: { ...prpEvent(2, "turn.completed", { status: "completed" }), ...(conflict ? { normalizedSessionId: "other-session" } : {}) }, semanticFingerprint: authorityJson(result) },
};
}
}
const backend = new HarnessDriverBackend({ ...driver, openSession: async () => new DurableTerminalSession() });
const session = await backend.openSession({ identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" } });
if (conflict) await expect(session.snapshot()).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError);
else {
expect((await session.snapshot()).terminal).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" });
expect(await session.result()).toMatchObject({ result, turnId: "turn-1" });
}
});
it.each([false, true])("honors driver steering support even when the transport exposes a steer method (%s)", async supported => {
const steer = vi.fn(async () => { throw new Error("provider does not support steering"); });
const session = Object.assign(new FakeHarnessSession(), { steer });
@@ -1,3 +1,4 @@
import { decodeSourceCursor, encodeSourceCursor, normalizedEventId } from "../control-plane/event-epochs.js";
import type {
HarnessDriver,
HarnessSession,
@@ -17,6 +18,7 @@ import type {
PrpEvent,
PrpTerminalState,
} from "../protocol/replay-contract.js";
import { validatePrpEvent } from "../protocol/replay-contract.js";
const MAX_RECOVERY_TERMINAL_TURNS = 4_096;
const MAX_RECOVERY_TERMINAL_BYTES = 8 * 1024 * 1024;
@@ -133,6 +135,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
normalizedSessionId: snapshot.identity.sessionId,
activeTurnId,
lastSourceSequence: parseCursor(snapshot.cursor),
...(decodeSourceCursor(snapshot.cursor).sourceEpoch ? { lastSourceEpoch: decodeSourceCursor(snapshot.cursor).sourceEpoch } : {}),
...(snapshot.semanticResult === undefined ||
snapshot.semanticResult === null
? {}
@@ -393,6 +396,7 @@ class HarnessNativeSession implements NativeSession {
readonly #session: HarnessSession;
#terminal: PrpTerminalState | null = null;
#explicitlyCancelled = false;
#lastDriverEventId: string | null = null;
#protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null;
#assertProtocolIntegrity(): void {
@@ -412,6 +416,24 @@ class HarnessNativeSession implements NativeSession {
try {
const snapshot = await this.#session.snapshot();
this.#assertProtocolIntegrity();
// The indexed driver has an explicit terminal fact even if the previous
// process died after its run-log ACK but before orchestration checkpointed.
const durable = snapshot.durableTerminal;
if (!this.#explicitlyCancelled && this.#terminal === null && durable && snapshot.activeTurnId === null && snapshot.semanticResult) {
const event = durable.event;
if (!validatePrpEvent(event).ok || event.runId !== this.#input.identity.runId || event.normalizedSessionId !== this.#input.identity.sessionId
|| event.turnId !== snapshot.semanticResult.turnId || durable.semanticFingerprint !== snapshot.semanticResult.fingerprint
|| durable.semanticFingerprint !== canonicalJson(snapshot.semanticResult.result)
|| !["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) {
throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict");
}
this.#terminal = {
schema: "paperclip.prp.terminal.v1",
turnTerminalState: event.eventType === "turn.completed" ? "completed" : event.eventType === "turn.failed" ? "failed" : event.eventType === "turn.interrupted" ? "interrupted" : "cancelled",
runTerminalState: event.eventType === "turn.completed" ? "succeeded" : event.eventType === "turn.failed" ? "failed" : "cancelled",
reportedWorkDisposition: snapshot.semanticResult.result.reportedWorkDisposition,
};
}
return snapshot;
} catch (error) {
this.#rethrowProtocolIntegrity(error);
@@ -504,6 +526,7 @@ class HarnessNativeSession implements NativeSession {
this.#assertProtocolIntegrity();
let sourceInstanceId: string | null = null;
let lastSourceSequence = 0;
let lastSourceEpoch: string | undefined;
let sawTerminal = false;
let synthesizedDurableWait = false;
let streamFailure: unknown = null;
@@ -517,7 +540,8 @@ class HarnessNativeSession implements NativeSession {
event.payload.kind === "interrupt_acknowledgement");
if (this.#explicitlyCancelled && !isCancellationEvent) continue;
sourceInstanceId = event.sourceInstanceId;
lastSourceSequence = Math.max(lastSourceSequence, event.sourceSeq);
lastSourceSequence = event.sourceSeq;
lastSourceEpoch = event.sourceEpoch;
if (event.eventType === "runtime_request.created") {
const request = plainRecord(event.payload.request);
if (
@@ -577,6 +601,7 @@ class HarnessNativeSession implements NativeSession {
reportedWorkDisposition: disposition,
};
}
this.#lastDriverEventId = event.sourceEventId;
yield structuredClone(event);
}
} catch (error) {
@@ -596,7 +621,8 @@ class HarnessNativeSession implements NativeSession {
let governedWaitTurnId: string | undefined;
for (const request of observedPendingInputs.values()) {
const sourceSeq =
Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1;
Math.max(lastSourceSequence, snapshot?.lastSourceEpoch === lastSourceEpoch ? snapshot?.lastSourceSequence ?? 0 : 0) + 1;
if (!Number.isSafeInteger(sourceSeq)) throw new Error("native_synthetic_event_epoch_required");
lastSourceSequence = sourceSeq;
const requestId = String(request.requestId);
const turnId =
@@ -606,7 +632,8 @@ class HarnessNativeSession implements NativeSession {
typeof request.itemId === "string" ? request.itemId : requestId;
yield {
schema: "paperclip.prp.event.v1",
sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`,
sourceEventId: normalizedEventId(sourceInstanceId, this.#input.identity.runId, sourceSeq, lastSourceEpoch),
...(lastSourceEpoch ? { sourceEpoch: lastSourceEpoch } : {}),
sourceSeq,
sourceInstanceId,
sourceKind: "runner",
@@ -635,7 +662,8 @@ class HarnessNativeSession implements NativeSession {
}
if (governedWaitTurnId) {
const sourceSeq =
Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1;
Math.max(lastSourceSequence, snapshot?.lastSourceEpoch === lastSourceEpoch ? snapshot?.lastSourceSequence ?? 0 : 0) + 1;
if (!Number.isSafeInteger(sourceSeq)) throw new Error("native_synthetic_event_epoch_required");
lastSourceSequence = sourceSeq;
synthesizedDurableWait = true;
sawTerminal = true;
@@ -647,7 +675,8 @@ class HarnessNativeSession implements NativeSession {
};
yield {
schema: "paperclip.prp.event.v1",
sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`,
sourceEventId: normalizedEventId(sourceInstanceId, this.#input.identity.runId, sourceSeq, lastSourceEpoch),
...(lastSourceEpoch ? { sourceEpoch: lastSourceEpoch } : {}),
sourceSeq,
sourceInstanceId,
sourceKind: "runner",
@@ -665,6 +694,22 @@ class HarnessNativeSession implements NativeSession {
if (streamFailure && !synthesizedDurableWait) throw streamFailure;
}
async acknowledgeEvent(event: PrpEvent): Promise<void> {
// Backend-synthesized provider-loss facts have their own durable writer;
// they were never entries in the driver's normalized outbox.
if (this.#lastDriverEventId !== event.sourceEventId) return;
await this.#session.acknowledgeEvent?.(event);
this.#lastDriverEventId = null;
}
setEventCommitter(commit: (event: PrpEvent) => Promise<void>): void {
this.#session.setEventCommitter?.(commit);
}
async flushEventDelivery(): Promise<void> {
await this.#session.flushEventDelivery?.();
}
async startTurn(input: Parameters<HarnessSession["startTurn"]>[0]) {
this.#assertProtocolIntegrity();
// Stop can arrive after session publication but before the first turn.
@@ -801,7 +846,7 @@ class HarnessNativeSession implements NativeSession {
cursor:
snapshot.lastSourceSequence === undefined
? null
: String(snapshot.lastSourceSequence),
: encodeSourceCursor({ sourceSeq: snapshot.lastSourceSequence, sourceEpoch: snapshot.lastSourceEpoch }),
semanticResult: this.#explicitlyCancelled
? null
: (snapshot.semanticResult?.result ?? null),
@@ -849,7 +894,7 @@ export function createHarnessDriverBackend(
function parseCursor(cursor: string | null | undefined): number | undefined {
if (cursor === undefined || cursor === null || cursor === "")
return undefined;
const parsed = Number(cursor);
const parsed = decodeSourceCursor(cursor).sourceSeq;
return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined;
}
@@ -68,7 +68,7 @@ describe("canonical operation-groups specification", () => {
expect(input.source.behaviorGroups).toHaveLength(16);
expect(input.scenarios).toHaveLength(106);
expect(new Set(input.scenarios.map((scenario) => scenario.id)).size).toBe(106);
expect(input.eventTypes).toHaveLength(105);
expect(input.eventTypes).toHaveLength(106);
expect(input.commandTypes).toHaveLength(18);
});
@@ -99,7 +99,7 @@ export interface OperationGroupsInput {
catalogIndexSource: string;
}
const EXPECTED_EVENT_COUNT = 105;
const EXPECTED_EVENT_COUNT = 106;
const EXPECTED_COMMAND_COUNT = 18;
const EXPECTED_SCENARIO_COUNT = 106;
const EXPECTED_BEHAVIOR_GROUP_COUNT = 16;
@@ -17,8 +17,9 @@ export interface OpenControlPlaneRunInput {
}
export interface AppendedEventReceipt {
cursor: number;
cursor: number | string;
highestContiguousSourceSeq: number;
highestContiguousSourceEpoch?: string;
disposition: "committed" | "duplicate";
}
@@ -42,12 +43,14 @@ export interface ReplayControlPlaneEventsInput {
runId: string;
sourceInstanceId: string;
afterSourceSeq: number;
sourceEpoch?: string;
limit: number;
}
export interface ReplayedControlPlaneEvents {
events: PrpEvent[];
highestContiguousSourceSeq: number;
highestContiguousSourceEpoch?: string;
}
export interface FinalizeControlPlaneOperationOptions {
@@ -92,6 +92,7 @@ export interface DurableRecoveryCoreCommand {
schema: "paperclip.prp.command.v1" | "paperclip.prp.command.v2";
commandId: string;
controllerSeq: number;
controllerEpoch?: string;
type: string;
issuedAt: string;
payload: Record<string, unknown>;
@@ -106,6 +107,7 @@ export interface DurableRecoveryCoreCommand {
}
export interface DurableRecoveryCommittedEvent {
sourceEpoch?: string;
sourceSeq: number;
sourceEventId: string;
eventType: string;
@@ -470,6 +470,8 @@ export interface PersistedHarnessSession {
activeTurnId?: string | null;
semanticResult?: PersistedHarnessSemanticResult | null;
terminalTurns?: PersistedHarnessTurnTerminal[];
/** Indexed reducer fact committed with its normalized delivery outbox. */
durableTerminal?: { event: PrpEvent; semanticFingerprint: string | null };
codexUsageBaseline?: { baseline: Record<string, number>; latest: Record<string, number> };
/** A result-less terminal task may spend this fail-closed one-shot recovery allowance. */
dispositionOnlyRecoveryConsumed?: boolean;
@@ -479,6 +481,7 @@ export interface PersistedHarnessSession {
goal?: HarnessThreadGoal | null;
lineage?: HarnessThreadLineageEntry[];
lastSourceSequence?: number;
lastSourceEpoch?: string;
/** Tagged provider identity used to reject cross-profile recovery. */
providerIdentity?: PersistedHarnessProviderIdentity;
/** Narrow escape hatch for a durable response-wake when a provider cannot reload its prior native session. */
@@ -501,6 +504,10 @@ export interface HarnessSession {
displayId?: string | null;
};
events(): AsyncIterable<PrpEvent>;
/** Called only after the exact normalized event is durable in the run log. */
acknowledgeEvent?(event: PrpEvent): Promise<void>;
setEventCommitter?(commit: (event: PrpEvent) => Promise<void>): void;
flushEventDelivery?(): Promise<void>;
attachRun?(input: { runId: string }): Promise<void> | void;
startTurn(input: {
message: NativeUserMessage;
@@ -232,6 +232,7 @@ export interface NativeSessionExecutionResult {
driverVersion: string;
nativeEventCount: number;
highestContiguousSourceSeq: number;
highestContiguousSourceEpoch?: string;
usage: Record<string, unknown> | null;
/** The active durable goal reached a safe turn boundary for run rollover. */
goalRolloverRequired?: boolean;
@@ -144,6 +144,9 @@ export interface NativeSession {
/** Relinquish controller authority without suspending provider execution. */
detachControllerForRestart?(): Promise<void>;
events(input?: { afterCursor?: string | null }): AsyncIterable<PrpEvent>;
acknowledgeEvent?(event: PrpEvent): Promise<void>;
setEventCommitter?(commit: (event: PrpEvent) => Promise<void>): void;
flushEventDelivery?(): Promise<void>;
startTurn(input: {
message: NativeUserMessage;
/** Set by orchestration only after successful provider-session recovery. */
@@ -0,0 +1,47 @@
import { describe, expect, it } from "vitest";
import { authorityDirectlyFollows, authorityGeneration, authorityInteger, validateAuthorityCommit, validateAuthorityWorkPage } from "./durable-authority-store.js";
const first = "r:00000000-0000-4000-8000-000000000001";
const second = "r:00000000-0000-4000-8000-000000000002";
const maximum = "9223372036854775807";
describe("exact authority revisions", () => {
it("accepts legacy exact boundaries and opaque revisions without converting them to numbers", () => {
for (const value of ["0", "9007199254740993", maximum, first]) expect(authorityGeneration(value)).toBe(value);
expect(authorityDirectlyFollows({ generation: first, committedFrom: maximum, state: {} }, maximum)).toBe(true);
expect(authorityDirectlyFollows({ generation: second, committedFrom: first, state: {} }, first)).toBe(true);
validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [] });
validateAuthorityWorkPage("process-owner", "", 128, first);
});
it("requires exact immediate predecessor evidence for migration activation", () => {
const latest = { generation: second, committedFrom: first, state: {} };
expect(authorityDirectlyFollows(latest, maximum)).toBe(false);
expect(authorityDirectlyFollows(latest, second)).toBe(false);
expect(authorityDirectlyFollows({ generation: first, state: {} }, maximum)).toBe(false);
expect(authorityDirectlyFollows({ generation: "2", state: {} }, first)).toBe(false);
expect(authorityDirectlyFollows({ generation: first, committedFrom: first, state: {} }, first)).toBe(false);
expect(authorityDirectlyFollows({ generation: "9007199254740993", state: {} }, "9007199254740992")).toBe(true);
expect(authorityDirectlyFollows({ generation: maximum, state: {} }, maximum)).toBe(false);
});
it.each(["-1", "01", "9223372036854775808", "r:missing", first.toUpperCase(), first.replace("-4000-", "-1000-")])("rejects invalid revision %s", value => {
expect(() => authorityGeneration(value)).toThrow("invalid_authority");
});
it("keeps event ordering numeric and separate from opaque state revisions", () => {
expect(authorityInteger(maximum)).toBe(9_223_372_036_854_775_807n);
expect(() => authorityInteger(first)).toThrow("invalid_authority");
expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [
{ epoch: "epoch-1", kind: "event", id: "event-1", sequence: second, body: {} },
] })).toThrow("invalid_authority");
});
it("reserves zero for effects which have exact identities without ordering", () => {
const receipt = { epoch: "run-1", kind: "effect" as const, id: "effect", sequence: "0", body: {} };
expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [receipt] })).not.toThrow();
for (const kind of ["command", "event"] as const) {
expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [{ ...receipt, kind }] })).toThrow("invalid_authority");
}
});
});
@@ -0,0 +1,28 @@
import { closeSync, mkdtempSync, openSync, rmSync, truncateSync, writeSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { expect, it } from "vitest";
import { readDurableControlPlaneState } from "./authority-locator.js";
import { DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES } from "./durable-prp-control-plane.js";
it("preserves the existing legacy reader window when adding indexed locators", async () => {
const root = mkdtempSync(resolve(tmpdir(), "legacy-authority-window-"));
const path = resolve(root, "control-plane-state.json");
try {
const fd = openSync(path, "wx", 0o600);
try {
writeSync(fd, '{"schema":"paperclip.runner.durable.control-plane-state.v1","history":"');
const chunk = Buffer.alloc(1024 * 1024, "x");
for (let index = 0; index < 65; index++) writeSync(fd, chunk);
writeSync(fd, '"}');
} finally { closeSync(fd); }
// This valid compatibility file exceeds the accidental 64 MiB locator
// reader cap, but is still inside the controller's original 192 MiB bound.
const state = await readDurableControlPlaneState(root);
expect((state.history as string).length).toBe(65 * 1024 * 1024);
// A sparse oversized file is only a metadata-rejection fixture, not growth
// qualification: the reader must reject it before allocating its body.
truncateSync(path, DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES + 1);
await expect(readDurableControlPlaneState(root)).rejects.toThrow("state_unsafe");
} finally { rmSync(root, { recursive: true, force: true }); }
}, 15_000);
@@ -0,0 +1,80 @@
import { constants, lstatSync, readSync, openSync, fstatSync, closeSync } from "node:fs";
import { basename, resolve } from "node:path";
import { SqliteAuthorityStore } from "./sqlite-authority-store.js";
import { assertNoPendingLegacyMigration } from "./legacy-migration-gate.js";
import { DurableAuthorityStoreError, MAX_LEGACY_AUTHORITY_STATE_BYTES, type AuthoritySnapshot } from "./durable-authority-store.js";
import { materializeCurrentAuthority } from "./current-authority-evidence.js";
import { authorityJson } from "./durable-authority-store.js";
import { createHash } from "node:crypto";
export interface AuthorityLocation {
kind: "sqlite" | "postgres";
binding: string;
file?: string;
epoch?: string;
}
export interface AuthorityLocator {
schema: "paperclip.runner.authority-locator.v1";
location: AuthorityLocation;
}
export type ExternalAuthorityReader = (location: AuthorityLocation) => Promise<AuthoritySnapshot>;
export function parseAuthorityLocator(value: unknown): AuthorityLocator | null {
const record = value as Partial<AuthorityLocator> | null;
if (!record || record.schema !== "paperclip.runner.authority-locator.v1") return null;
const location = record.location;
if (!location || !["sqlite", "postgres"].includes(location.kind) || typeof location.binding !== "string" || location.binding.length > 4096 ||
(location.kind === "sqlite" && (typeof location.file !== "string" || location.file !== basename(location.file) || !/^[a-zA-Z0-9_.-]+\.sqlite$/.test(location.file))) ||
(location.kind === "postgres" && (typeof location.epoch !== "string" || !/^[a-zA-Z0-9_.:-]{1,240}$/.test(location.epoch)))) {
throw new DurableAuthorityStoreError("invalid_authority", "invalid authority locator");
}
return record as AuthorityLocator;
}
/** Legacy files are bounded compatibility input; indexed locators load one row. */
export async function readDurableControlPlaneState(directory: string, readExternal?: ExternalAuthorityReader): Promise<Record<string, unknown>> {
return (await readDurableControlPlaneSnapshot(directory, readExternal)).state;
}
/** The materialized view and its stored digest are different representations.
* Archive fences compare the latter, after checking the former still matches
* the authority the caller admitted. External stores own their own fences. */
export async function readDurableControlPlaneSnapshot(directory: string, readExternal?: ExternalAuthorityReader): Promise<{
state: Record<string, unknown>; storedSha256: string | null;
}> {
assertNoPendingLegacyMigration(directory);
const path = resolve(directory, "control-plane-state.json");
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
let value: unknown;
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > MAX_LEGACY_AUTHORITY_STATE_BYTES) throw new Error("native_runner_control_plane_state_unsafe");
const bytes = Buffer.alloc(stat.size);
let offset = 0;
while (offset < bytes.length) {
const count = readSync(fd, bytes, offset, bytes.length - offset, offset);
if (!count) throw new Error("native_runner_control_plane_state_changed");
offset += count;
}
const after = fstatSync(fd);
if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("native_runner_control_plane_state_changed");
value = JSON.parse(bytes.toString("utf8"));
if (parseAuthorityLocator(value) && process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.())) throw new Error("native_runner_control_plane_state_unsafe");
} finally { closeSync(fd); }
const locator = parseAuthorityLocator(value);
if (!locator) return { state: value as Record<string, unknown>, storedSha256: null };
if (locator.location.kind === "postgres") {
if (!readExternal) throw new DurableAuthorityStoreError("storage_unavailable", "Postgres authority reader is required");
return { state: (await readExternal(locator.location)).state, storedSha256: null };
}
const metadata = lstatSync(directory);
if (metadata.isSymbolicLink() || !metadata.isDirectory()) throw new Error("native_runner_control_plane_state_unsafe");
const store = await SqliteAuthorityStore.open({ binding: locator.location.binding, path: resolve(directory, locator.location.file!), create: false, readOnly: true });
try {
const snapshot = await store.load();
if (!snapshot) throw new DurableAuthorityStoreError("storage_unavailable", "activated authority snapshot is missing");
const state = await materializeCurrentAuthority(snapshot.state, store.getRecord.bind(store));
if ((await store.load())?.generation !== snapshot.generation) throw new DurableAuthorityStoreError("stale_authority", "authority changed while resolving current evidence");
return { state, storedSha256: createHash("sha256").update(authorityJson(snapshot.state)).digest("hex") };
} finally { await store.close(); }
}
@@ -0,0 +1,40 @@
import { createHash } from "node:crypto";
/** Ordering is local to a negotiated delivery namespace, never the run lifetime. */
export const COMMAND_EPOCH_CAPABILITY = "transport.command_epochs.v1";
export const COMMAND_EPOCH_LIMIT = 1_048_576;
const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
export interface CommandEpochTransition {
schema: "paperclip.prp.command-epoch.v1";
runId: string;
transitionId: string;
fromEpoch: string | null;
nextEpoch: string;
finalOrdinal: number;
}
export function isCommandEpoch(value: unknown): value is string {
return typeof value === "string" && uuid.test(value);
}
export function isCommandEpochTransition(value: unknown): value is CommandEpochTransition {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const v = value as CommandEpochTransition;
return Object.keys(v).length === 6 && v.schema === "paperclip.prp.command-epoch.v1"
&& typeof v.runId === "string" && v.runId.length > 0 && v.runId.length <= 240
&& isCommandEpoch(v.transitionId) && (v.fromEpoch === null || isCommandEpoch(v.fromEpoch))
&& isCommandEpoch(v.nextEpoch) && v.nextEpoch !== v.fromEpoch
&& Number.isSafeInteger(v.finalOrdinal) && v.finalOrdinal > 0;
}
/** Current proof slots from older epochs precede this epoch's ordered suffix.
* There is at most one old settled receipt per slot; no history traversal. */
export function compareCurrentCommands(
a: { controllerEpoch?: string; controllerSeq: number },
b: { controllerEpoch?: string; controllerSeq: number },
epoch: string | undefined,
): number {
if (a.controllerEpoch === b.controllerEpoch) return a.controllerSeq - b.controllerSeq;
return Number(a.controllerEpoch === epoch) - Number(b.controllerEpoch === epoch);
}
export function commandEpochCloseId(transition: Pick<CommandEpochTransition, "runId" | "fromEpoch">): string {
return `command-epoch-from-${createHash("sha256").update(JSON.stringify([transition.runId, transition.fromEpoch])).digest("hex")}`;
}
@@ -0,0 +1,29 @@
import { expect, it, vi } from "vitest";
import { materializeCurrentAuthority, referenceCurrentAuthority } from "./current-authority-evidence.js";
import type { StoredCoreState } from "./durable-prp-control-plane.js";
it("resolves only exact current references and rejects missing, conflicting or rebound authority", async () => {
const command = { commandId: "finish-result", controllerSeq: 12, status: "completed", type: "semantic_tool.result",
payload: { operationId: "paperclip_finish", input: { summary: "body".repeat(10000) } }, result: { accepted: true } };
const active = { commandId: "pending", controllerSeq: 13, status: "pending", type: "turn.start" };
const state = { identity: { runId: "run-1" }, commands: [command, active], committedEvents: [],
indexedState: { schema: "paperclip.runner.current-authority.v1", pendingSemanticInputIds: [] } } as unknown as StoredCoreState;
const stored = referenceCurrentAuthority(state);
expect(stored.commands).toEqual([active]);
expect(JSON.stringify(stored).length).toBeLessThan(1024);
const receipt = { epoch: "run-1", kind: "command" as const, id: command.commandId, sequence: "12", body: command };
const read = vi.fn(async () => receipt);
expect(await materializeCurrentAuthority(stored as unknown as Record<string, unknown>, read)).toEqual(state);
expect(read).toHaveBeenCalledExactlyOnceWith("run-1", "command", "finish-result");
for (const bad of [null, { ...receipt, epoch: "foreign" }, { ...receipt, id: "other" }, { ...receipt, sequence: "11" }, { ...receipt, body: { ...command, result: {} } }]) {
await expect(materializeCurrentAuthority(stored as unknown as Record<string, unknown>, async () => bad)).rejects.toThrow("exact receipt");
}
const absent = structuredClone(stored);
delete absent.indexedState!.recoveryEvidence;
await expect(materializeCurrentAuthority(absent as unknown as Record<string, unknown>, read)).rejects.toThrow("exact receipt");
const rebound = structuredClone(stored);
const refs = rebound.indexedState!.recoveryEvidence!.records;
refs["command:other"] = refs["command:terminal:paperclip_finish"]!;
delete refs["command:terminal:paperclip_finish"];
await expect(materializeCurrentAuthority(rebound as unknown as Record<string, unknown>, read)).rejects.toThrow("exact receipt");
});
@@ -0,0 +1,139 @@
import { normalizedEpochCloseId } from "./event-epochs.js";
import { compareCurrentEvents, isEventEpochTransition, eventEpochCloseId } from "./event-epochs.js";
import { compareCurrentCommands, isCommandEpochTransition, commandEpochCloseId } from "./command-epochs.js";
import { createHash } from "node:crypto";
import { authorityJson, DurableAuthorityStoreError, type AuthorityRecord, type DurableAuthorityStore } from "./durable-authority-store.js";
import type { StoredCoreState } from "./durable-prp-control-plane.js";
export interface CurrentAuthorityReference {
epoch: string;
kind: "command" | "event";
id: string;
sequence: string;
sha256: string;
}
export interface CurrentAuthorityEvidence {
schema: "paperclip.current-evidence.v1";
records: Record<string, CurrentAuthorityReference>;
commandOrder?: string[];
eventOrder?: string[];
}
const digest = (body: unknown) => createHash("sha256").update(authorityJson(body)).digest("hex");
const lifecycleEvents = new Set(["session.started", "session.resumed", "session.reconciled", "harness.ready", "turn.accepted", "turn.started", "turn.completed", "turn.failed", "run.terminal"]);
const terminalOperations = new Set(["paperclip_finish", "paperclip_block"]);
const object = (v: unknown): Record<string, unknown> => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record<string, unknown> : {};
function invalid(): never { throw new DurableAuthorityStoreError("invalid_authority", "current recovery evidence differs from its exact receipt"); }
function slot(kind: CurrentAuthorityReference["kind"], body: Record<string, unknown>): string | null {
if (kind === "command") {
if (body.status === "pending" || body.status === "indeterminate") return null;
if (typeof body.type !== "string" || body.type.length > 80) invalid();
if (body.type !== "semantic_tool.result") return `command:${body.type}`;
const operation = String(object(body.payload).operationId);
return terminalOperations.has(operation) ? `command:terminal:${operation}` : null;
}
const operation = String(object(object(object(object(body.envelope).payload).payload).semantic_tool).operationId);
if (terminalOperations.has(operation)) return `event:${body.eventType}:${operation}`;
return lifecycleEvents.has(String(body.eventType)) ? `event:${body.eventType}` : null;
}
/** The on-disk current row contains active work and a fixed set of typed slots.
* Settled payloads live once in the immutable receipt index. This does not
* certify a finish by itself: server acceptance still validates its structured
* input, operation result, completion contract and business result receipt. */
export function referenceCurrentAuthority(state: StoredCoreState): StoredCoreState {
const next = structuredClone(state);
const current = next.indexedState!;
const evidence: CurrentAuthorityEvidence = { schema: "paperclip.current-evidence.v1", records: {}, commandOrder: next.commands.map(c => c.commandId), eventOrder: next.committedEvents.map(e => e.sourceEventId) };
const remember = (kind: CurrentAuthorityReference["kind"], body: Record<string, unknown>, id: string, sequence: number) => {
const key = slot(kind, body);
if (!key) return false;
evidence.records[key] = { epoch: next.identity.runId, kind, id, sequence: String(sequence), sha256: digest(body) };
return true;
};
next.commands = next.commands.filter(command => !remember("command", command as unknown as Record<string, unknown>, command.commandId, command.controllerSeq));
next.committedEvents = next.committedEvents.filter(event => current.pendingSemanticInputIds.includes(event.sourceEventId) ||
!remember("event", { ...event, deliveryCount: 1 }, event.sourceEventId, event.sourceSeq));
if (Object.keys(evidence.records).length > 40) invalid();
current.recoveryEvidence = evidence;
current.schema = "paperclip.runner.current-authority.v2";
return next;
}
/** Resolves only named current receipts. There is no history scan or fallback
* to a nearby event if a receipt is missing. Callers fence snapshot generation
* before and after this bounded read when opening an active authority. */
export async function materializeCurrentAuthority(state: Record<string, unknown>, read: DurableAuthorityStore["getRecord"]): Promise<Record<string, unknown>> {
const current = object(state.indexedState);
if (state.indexedState !== undefined && !["paperclip.runner.current-authority.v1", "paperclip.runner.current-authority.v2"].includes(String(current.schema))) invalid();
if (current.recoveryEvidence === undefined) {
if (current.schema === "paperclip.runner.current-authority.v2") invalid();
return state; // Retained v1 current rows and already materialized views.
}
if (current.controllerEpoch !== undefined || current.lastCommandEpochTransition !== undefined) {
const transition = current.lastCommandEpochTransition;
if (!isCommandEpochTransition(transition) || transition.nextEpoch !== current.controllerEpoch || transition.runId !== object(state.identity).runId) invalid();
for (const id of [`command-epoch-${transition.transitionId}`, commandEpochCloseId(transition)]) {
const receipt = await read(transition.runId, "effect", id);
if (!receipt || receipt.epoch !== transition.runId || receipt.kind !== "effect" || receipt.id !== id || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(transition)) invalid();
}
}
if (current.sourceEpoch !== undefined || current.lastEventEpochTransition !== undefined) {
const transition = current.lastEventEpochTransition;
if (!isEventEpochTransition(transition) || transition.nextEpoch !== current.sourceEpoch || transition.runId !== object(state.identity).runId) invalid();
for (const id of [`event-epoch-${transition.transitionId}`, eventEpochCloseId(transition)]) {
const receipt = await read(transition.runId, "effect", id);
if (!receipt || receipt.epoch !== transition.runId || receipt.kind !== "effect" || receipt.id !== id || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(transition)) invalid();
}
}
const delivery = object(current.normalizedDelivery);
if (delivery.producedEpoch !== undefined || delivery.lastSourceEpochTransition !== undefined) {
const t = delivery.lastSourceEpochTransition;
const identity = object(state.identity);
if (!isEventEpochTransition(t) || t.nextEpoch !== delivery.producedEpoch || t.runId !== identity.runId) invalid();
for (const id of [`normalized-epoch-${t.transitionId}`, normalizedEpochCloseId(t.runId, String(identity.runnerInstanceId), t.fromEpoch)]) {
const receipt = await read(t.runId, "effect", id);
if (!receipt || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(t)) invalid();
}
}
const raw = object(object(current.normalizedDelivery).raw);
if (current.normalizedDelivery && raw.sourceEpoch !== current.sourceEpoch) {
const receipt = await read(String(object(state.identity).runId), "effect", eventEpochCloseId({runId: String(object(state.identity).runId), fromEpoch: (raw.sourceEpoch as string | undefined) ?? null}));
if (!receipt || !isEventEpochTransition(receipt.body) || receipt.body.fromEpoch !== (raw.sourceEpoch ?? null) || Number(raw.sourceSeq) > receipt.body.finalOrdinal) invalid();
}
const evidence = object(current.recoveryEvidence), entries = Object.entries(object(evidence.records));
if (current.schema !== "paperclip.runner.current-authority.v2" || evidence.schema !== "paperclip.current-evidence.v1" || !evidence.records || Array.isArray(evidence.records) || entries.length > 40 ||
!Array.isArray(state.commands) || !Array.isArray(state.committedEvents)) invalid();
const result = structuredClone(state) as unknown as StoredCoreState;
const ids = new Set<string>();
for (const [key, value] of entries) {
const ref = value as CurrentAuthorityReference;
if (!ref || ref.epoch !== result.identity.runId || !["command", "event"].includes(ref.kind) ||
typeof ref.id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(ref.id) ||
!/^[1-9][0-9]{0,15}$/.test(ref.sequence) || !Number.isSafeInteger(Number(ref.sequence)) || !/^[a-f0-9]{64}$/.test(ref.sha256)) invalid();
const receipt: AuthorityRecord | null = await read(ref.epoch, ref.kind, ref.id);
if (!receipt || receipt.epoch !== ref.epoch || receipt.kind !== ref.kind || receipt.id !== ref.id ||
receipt.sequence !== ref.sequence || digest(receipt.body) !== ref.sha256 || slot(ref.kind, receipt.body) !== key || ids.has(`${ref.kind}:${ref.id}`)) invalid();
ids.add(`${ref.kind}:${ref.id}`);
if (ref.kind === "command") {
if (receipt.body.commandId !== ref.id || String(receipt.body.controllerSeq) !== ref.sequence || result.commands.some(c => c.commandId === ref.id)) invalid();
result.commands.push(receipt.body as unknown as StoredCoreState["commands"][number]);
} else {
if (receipt.body.sourceEventId !== ref.id || String(receipt.body.sourceSeq) !== ref.sequence || result.committedEvents.some(e => e.sourceEventId === ref.id)) invalid();
result.committedEvents.push(receipt.body as unknown as StoredCoreState["committedEvents"][number]);
}
}
result.commands.sort((a, b) => compareCurrentCommands(a, b, result.indexedState?.controllerEpoch));
result.committedEvents.sort((a, b) => compareCurrentEvents(a, b, result.indexedState?.sourceEpoch));
for (const [order, records, id] of [
[evidence.commandOrder, result.commands, "commandId"], [evidence.eventOrder, result.committedEvents, "sourceEventId"],
] as const) {
if (order === undefined) continue; // retained current-evidence v1
if (!Array.isArray(order) || order.length !== records.length || new Set(order).size !== order.length) invalid();
const position = new Map(order.map((key, index) => [key, index]));
if (records.some(record => !position.has((record as unknown as Record<string, unknown>)[id]))) invalid();
records.sort((a, b) => position.get((a as unknown as Record<string, unknown>)[id])! - position.get((b as unknown as Record<string, unknown>)[id])!);
}
delete result.indexedState!.recoveryEvidence;
result.indexedState!.schema = "paperclip.runner.current-authority.v1";
return result as unknown as Record<string, unknown>;
}
@@ -0,0 +1,181 @@
import type { AuthorityLocation } from "./authority-locator.js";
/** The execution store is independent of the server and its database driver.
* History is queried by exact identity or cursor, never returned by load(). */
export const INDEXED_DURABILITY_CAPABILITY = "durability.indexed_state.v1";
/** Compatibility readers and writers must admit the same legacy journal size. */
export const MAX_LEGACY_AUTHORITY_STATE_BYTES = 192 * 1024 * 1024;
export const MAX_AUTHORITY_STATE_BYTES = 16 * 1024 * 1024;
export const MAX_AUTHORITY_RECORD_BYTES = 1024 * 1024;
export const MAX_AUTHORITY_COMMIT_RECORDS = 512;
export interface AuthoritySnapshot {
/** Exact store revision. Compare equality; never order or increment it. */
generation: string;
/** Exact predecessor when the store uses opaque commit identities. */
committedFrom?: string;
state: Record<string, unknown>;
}
export interface AuthorityRecord {
epoch: string;
kind: "command" | "event" | "effect";
id: string;
/** Command/event order belongs to the authenticated run epoch. Effects are
* addressed by exact identity and use zero when the store supports it. */
sequence: string;
/** Absent is the retained numeric namespace; epochs require store capability. */
sequenceEpoch?: string;
body: Record<string, unknown>;
}
export interface AuthorityCommit {
expectedGeneration: string;
state: Record<string, unknown>;
/** Newly settled receipts/events only. Existing records are immutable. */
records: readonly AuthorityRecord[];
work?: readonly AuthorityWorkChange[];
}
export interface AuthorityWorkRecord {
collection: "process-owner";
id: string;
body: Record<string, unknown>;
sha256: string;
}
export interface AuthorityWorkChange {
collection: AuthorityWorkRecord["collection"];
id: string;
expectedSha256: string | null;
/** Null retires current work; its exact retirement proof belongs in records. */
body: Record<string, unknown> | null;
}
export interface AuthorityWorkPage { records: AuthorityWorkRecord[]; nextAfter: string | null }
export interface AuthorityPage {
records: AuthorityRecord[];
nextAfter: string | null;
}
export interface DurableAuthorityStore {
readonly binding: string;
readonly location: AuthorityLocation;
/** Retained v1 stores require a positive ordered effect sequence. */
readonly unorderedEffectReceipts?: boolean;
readonly commandEpochs?: boolean;
readonly eventEpochs?: boolean;
load(): Promise<AuthoritySnapshot | null>;
commit(input: AuthorityCommit): Promise<string>;
getRecord(epoch: string, kind: AuthorityRecord["kind"], id: string): Promise<AuthorityRecord | null>;
/** Session-scoped exact effect lookup survives run/authority rotation. */
getSessionEffect(id: string): Promise<AuthorityRecord | null>;
getWork(collection: AuthorityWorkRecord["collection"], id: string): Promise<AuthorityWorkRecord | null>;
readWorkPage(collection: AuthorityWorkRecord["collection"], after: string, limit: number, expectedGeneration: string): Promise<AuthorityWorkPage>;
readEvents(epoch: string, after: string, limit: number, byteBudget: number, sequenceEpoch?: string): Promise<AuthorityPage>;
close(): Promise<void>;
}
export class DurableAuthorityStoreError extends Error {
readonly code: "storage_pressure" | "storage_unavailable" | "stale_authority" | "receipt_conflict" | "invalid_authority";
constructor(code: DurableAuthorityStoreError["code"], message: string) {
super(`${code}: ${message}`);
this.name = "DurableAuthorityStoreError";
this.code = code;
}
}
export function authorityInteger(value: string): bigint {
if (!/^(0|[1-9][0-9]{0,18})$/.test(value) || BigInt(value) > 9_223_372_036_854_775_807n) {
throw new DurableAuthorityStoreError("invalid_authority", "invalid exact sequence/generation");
}
return BigInt(value);
}
/** Legacy stores expose exact integer revisions. Stores with opaque revisions
* retain predecessor evidence for operations which must prove one transition. */
export function authorityGeneration(value: string): string {
if (/^r:[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(value)) return value;
authorityInteger(value);
return value;
}
export function authorityDirectlyFollows(snapshot: AuthoritySnapshot, previous: string): boolean {
authorityGeneration(previous);
authorityGeneration(snapshot.generation);
if (snapshot.committedFrom !== undefined) {
authorityGeneration(snapshot.committedFrom);
return snapshot.committedFrom === previous && snapshot.generation !== previous;
}
// Numeric v1 stores have no revision wraparound and prove their predecessor
// by exact arithmetic. An opaque revision without a commit stamp cannot.
if (snapshot.generation.startsWith("r:") || previous.startsWith("r:")) return false;
return authorityInteger(snapshot.generation) === authorityInteger(previous) + 1n;
}
/** Stable encoding for receipt comparison across SQLite and Postgres. */
export function authorityJson(value: unknown): string {
const ancestors = new Set<object>();
function encode(value: unknown, depth: number): string {
if (depth > 128) throw new DurableAuthorityStoreError("invalid_authority", "authority JSON nesting exceeds 128");
if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
if (typeof value === "number" && Number.isFinite(value)) return JSON.stringify(value);
if (typeof value !== "object" || value === null || (!Array.isArray(value) && ![Object.prototype, null].includes(Object.getPrototypeOf(value)))) {
throw new DurableAuthorityStoreError("invalid_authority", "authority requires JSON values");
}
if (ancestors.has(value)) throw new DurableAuthorityStoreError("invalid_authority", "cyclic authority JSON");
ancestors.add(value);
try {
if (Array.isArray(value)) return `[${value.map((item) => encode(item, depth + 1)).join(",")}]`;
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${encode((value as Record<string, unknown>)[key], depth + 1)}`).join(",")}}`;
} finally {
ancestors.delete(value);
}
}
return encode(value, 0);
}
export function validateAuthorityCommit(input: AuthorityCommit): void {
authorityGeneration(input.expectedGeneration);
if (Buffer.byteLength(authorityJson(input.state)) > MAX_AUTHORITY_STATE_BYTES || input.records.length > MAX_AUTHORITY_COMMIT_RECORDS) {
throw new DurableAuthorityStoreError("storage_pressure", "current state or commit exceeds admission capacity");
}
let bytes = 0;
for (const record of input.records) {
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(record.epoch) || !["command", "event", "effect"].includes(record.kind) || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(record.id) || (authorityInteger(record.sequence) === 0n && record.kind !== "effect")) {
throw new DurableAuthorityStoreError("invalid_authority", "invalid receipt identity");
}
if (record.sequenceEpoch !== undefined && (record.kind === "effect" || !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(record.sequenceEpoch))) {
throw new DurableAuthorityStoreError("invalid_authority", "invalid receipt sequence epoch");
}
const size = Buffer.byteLength(authorityJson(record.body));
bytes += size;
if (size > MAX_AUTHORITY_RECORD_BYTES || bytes > MAX_AUTHORITY_STATE_BYTES) {
throw new DurableAuthorityStoreError("storage_pressure", "receipt transaction exceeds admission capacity");
}
}
const changes = input.work ?? [];
if (changes.length > 128) throw new DurableAuthorityStoreError("storage_pressure", "outstanding-work transaction exceeds capacity");
const keys = new Set<string>();
for (const change of changes) {
validateAuthorityWorkKey(change.collection, change.id);
const key = `${change.collection}:${change.id}`;
if (keys.has(key) || (change.expectedSha256 !== null && !/^[a-f0-9]{64}$/.test(change.expectedSha256)) ||
(change.body !== null && Buffer.byteLength(authorityJson(change.body)) > 16384)) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work mutation");
keys.add(key);
}
}
export function validateAuthorityWorkKey(collection: string, id: string): void {
if (collection !== "process-owner" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(id)) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work identity");
}
export function validateAuthorityWorkPage(collection: string, after: string, limit: number, generation: string): void {
validateAuthorityWorkKey(collection, after || "first"); authorityGeneration(generation);
if (!Number.isInteger(limit) || limit < 1 || limit > 128) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work page");
}
export function validateAuthorityPage(after: string, limit: number, byteBudget: number, sequenceEpoch?: string): void {
authorityInteger(after);
if (sequenceEpoch !== undefined && !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(sequenceEpoch)) throw new DurableAuthorityStoreError("invalid_authority", "invalid event page namespace");
if (!Number.isInteger(limit) || limit < 1 || limit > 128 || !Number.isInteger(byteBudget) || byteBudget < MAX_AUTHORITY_RECORD_BYTES || byteBudget > MAX_AUTHORITY_STATE_BYTES) {
throw new DurableAuthorityStoreError("invalid_authority", "invalid event page capacity");
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,83 @@
import { createHash } from "node:crypto";
import { isCommandEpoch } from "./command-epochs.js";
export const EVENT_EPOCH_CAPABILITY = "transport.event_epochs.v1";
export const EVENT_EPOCH_LIMIT = 1_048_576;
/** The sender drains its durable outbox before installing this successor. */
export interface EventEpochTransition {
schema: "paperclip.prp.event-epoch.v1";
runId: string;
transitionId: string;
fromEpoch: string | null;
nextEpoch: string;
finalOrdinal: number;
}
export interface EventResume {
sourceEpoch: string | null;
nextSourceEventSeq: number;
ackedSourceSeq: number;
transition: EventEpochTransition | null;
}
export function isEventEpochTransition(value: unknown): value is EventEpochTransition {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const v = value as EventEpochTransition;
return Object.keys(v).length === 6 && v.schema === "paperclip.prp.event-epoch.v1"
&& typeof v.runId === "string" && v.runId.length > 0 && v.runId.length <= 240
&& isCommandEpoch(v.transitionId) && (v.fromEpoch === null || isCommandEpoch(v.fromEpoch))
&& isCommandEpoch(v.nextEpoch) && v.nextEpoch !== v.fromEpoch
&& Number.isSafeInteger(v.finalOrdinal) && v.finalOrdinal > 0;
}
export function isEventResume(value: unknown): value is EventResume {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const v = value as EventResume;
return Object.keys(v).length === 4 && (v.sourceEpoch === null || isCommandEpoch(v.sourceEpoch))
&& Number.isSafeInteger(v.nextSourceEventSeq) && v.nextSourceEventSeq > 0
&& Number.isSafeInteger(v.ackedSourceSeq) && v.ackedSourceSeq >= 0 && v.ackedSourceSeq < v.nextSourceEventSeq
&& (v.transition === null || (isEventEpochTransition(v.transition) && v.transition.fromEpoch === v.sourceEpoch && v.transition.finalOrdinal === v.nextSourceEventSeq - 1));
}
export function eventEpochCloseId(transition: Pick<EventEpochTransition, "runId" | "fromEpoch">): string {
return `event-epoch-from-${createHash("sha256").update(JSON.stringify([transition.runId, transition.fromEpoch])).digest("hex")}`;
}
export function compareCurrentEvents(a: { sourceEpoch?: string; sourceSeq: number }, b: { sourceEpoch?: string; sourceSeq: number }, epoch: string | undefined): number {
if (a.sourceEpoch === b.sourceEpoch) return a.sourceSeq - b.sourceSeq;
return Number(a.sourceEpoch === epoch) - Number(b.sourceEpoch === epoch);
}
/** The normalized outbox commits this boundary with its first successor event.
* These identities never depend on a lifetime ordinal. */
export function normalizedEpochCloseId(runId: string, sourceInstanceId: string, fromEpoch: string | null): string {
return `normalized-epoch-from-${createHash("sha256").update(JSON.stringify([runId, sourceInstanceId, fromEpoch])).digest("hex")}`;
}
export function normalizedEventId(sourceInstanceId: string, runId: string, sourceSeq: number, sourceEpoch?: string): string {
return sourceEpoch === undefined ? `${sourceInstanceId}:${runId}:${sourceSeq}`
: `normalized-${createHash("sha256").update(JSON.stringify([sourceInstanceId, runId, sourceEpoch, sourceSeq])).digest("hex")}`;
}
export interface SourceCursor { sourceEpoch?: string; sourceSeq: number }
/** Only a verified predecessor and an explicit first-event receipt can cross
* namespaces. Numeric ordering is meaningful inside one namespace only. */
export function advanceSourceCursor(previous: SourceCursor, event: SourceCursor & { runId: string; sourceEpochTransition?: EventEpochTransition }): SourceCursor {
if (!Number.isSafeInteger(previous.sourceSeq) || previous.sourceSeq < 0 || !Number.isSafeInteger(event.sourceSeq) || event.sourceSeq < 1
|| (previous.sourceEpoch !== undefined && !isCommandEpoch(previous.sourceEpoch))
|| (event.sourceEpoch !== undefined && !isCommandEpoch(event.sourceEpoch))) throw new Error("invalid source cursor");
if (previous.sourceEpoch === event.sourceEpoch) {
if (event.sourceEpochTransition !== undefined || previous.sourceSeq === Number.MAX_SAFE_INTEGER || event.sourceSeq !== previous.sourceSeq + 1) throw new Error("source cursor is not contiguous");
} else {
const t = event.sourceEpochTransition;
if (!isEventEpochTransition(t) || t.runId !== event.runId || t.fromEpoch !== (previous.sourceEpoch ?? null)
|| t.nextEpoch !== event.sourceEpoch || t.finalOrdinal !== previous.sourceSeq || event.sourceSeq !== 1) throw new Error("source epoch has no exact predecessor");
}
return { ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), sourceSeq: event.sourceSeq };
}
export function encodeSourceCursor(cursor: SourceCursor): string {
if (!Number.isSafeInteger(cursor.sourceSeq) || cursor.sourceSeq < 0 || (cursor.sourceEpoch !== undefined && !isCommandEpoch(cursor.sourceEpoch))) throw new Error("invalid source cursor");
return cursor.sourceEpoch ? `e:${cursor.sourceEpoch}:${cursor.sourceSeq}` : String(cursor.sourceSeq);
}
export function decodeSourceCursor(value: string | null | undefined): SourceCursor {
if (value?.startsWith("e:")) {
const match = /^e:([^:]+):(0|[1-9][0-9]*)$/.exec(value);
if (!match || !isCommandEpoch(match[1]) || !Number.isSafeInteger(Number(match[2]))) throw new Error("invalid source cursor");
return { sourceEpoch: match[1], sourceSeq: Number(match[2]) };
}
const sourceSeq = Number(value ?? 0);
return { sourceSeq: Number.isSafeInteger(sourceSeq) && sourceSeq >= 0 ? sourceSeq : 0 };
}
@@ -0,0 +1,75 @@
import { createHash } from "node:crypto";
import type { IndexedLocalSnapshot } from "./indexed-local-state-reader.js";
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
const { authorityGeneration } = await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js");
const CHUNK_BYTES = 192 * 1024;
const MAX_SNAPSHOT_BYTES = 40 * 1024 * 1024;
const MAX_FRAME_BYTES = CHUNK_BYTES * 4 / 3 + 128;
const object = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value);
const invalid = () => new Error("native_indexed_inspection_invalid_or_incomplete");
const digest = (value: unknown): value is string => typeof value === "string" && value.length === 64 && /^[a-f0-9]+$/.test(value);
/** Decoder shared by local worker reads and owned remote inspection channels.
* Limits apply to one bounded current snapshot, never accumulated history. */
export class IndexedInspectionDecoder {
private pending: Buffer = Buffer.alloc(0);
private header?: { byteLength: number; sha256: string };
private chunks: Buffer[] = [];
private hash = createHash("sha256");
private length = 0;
private complete = false;
private finished = false;
push(chunk: Uint8Array): void {
if (this.finished) throw invalid();
try {
const bytes = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
let start = 0;
while (start < bytes.length) {
const end = bytes.indexOf(10, start);
const part = bytes.subarray(start, end < 0 ? bytes.length : end);
if (this.pending.length + part.length > MAX_FRAME_BYTES) throw invalid();
this.pending = Buffer.concat([this.pending, part]);
if (end < 0) break;
this.line(this.pending); this.pending = Buffer.alloc(0); start = end + 1;
}
} catch { throw invalid(); }
}
private line(bytes: Buffer): void {
if (this.complete) throw invalid();
const value: unknown = JSON.parse(bytes.toString("utf8"));
if (!object(value)) throw invalid();
if (!this.header) {
if (value.schema !== "paperclip.indexed-inspection.v1" || !Number.isSafeInteger(value.byteLength) ||
Number(value.byteLength) < 1 || Number(value.byteLength) > MAX_SNAPSHOT_BYTES || !digest(value.sha256)) throw invalid();
this.header = { byteLength: Number(value.byteLength), sha256: value.sha256 };
} else if (value.complete === true) {
if (Object.keys(value).length !== 1 || this.length !== this.header.byteLength) throw invalid();
this.complete = true;
} else {
if (value.index !== this.chunks.length || typeof value.bytes !== "string" || value.bytes.length > CHUNK_BYTES * 4 / 3) throw invalid();
const chunk = Buffer.from(value.bytes, "base64");
if (chunk.toString("base64") !== value.bytes || chunk.length !== Math.min(CHUNK_BYTES, this.header.byteLength - this.length) || chunk.length === 0) throw invalid();
this.hash.update(chunk); this.chunks.push(chunk); this.length += chunk.length;
}
}
/** Call only after a clean native process exit and the complete stdout stream. */
finish(): IndexedLocalSnapshot {
if (this.finished) throw invalid();
this.finished = true;
try {
if (this.pending.length || !this.complete || !this.header || this.hash.digest("hex") !== this.header.sha256) throw invalid();
const value: unknown = JSON.parse(Buffer.concat(this.chunks, this.length).toString("utf8"));
if (!object(value) || !object(value.state) || typeof value.generation !== "string" || value.generation !== value.generation.trim() ||
value.generation === "0" || !digest(value.stateDigest) ||
(value.preparation !== null && !object(value.preparation))) throw invalid();
authorityGeneration(value.generation);
return value as unknown as IndexedLocalSnapshot;
} catch { throw invalid(); }
finally { this.chunks = []; }
}
}
@@ -0,0 +1,97 @@
import { createHash } from "node:crypto";
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { expect, it } from "vitest";
import { readIndexedLocalState, suspendIndexedRunnerState } from "./indexed-local-state-reader.js";
it.each([1, 2])("schema %s seals only the verified empty runner generation without replacing its locator or receipts", async (schema) => {
const root = mkdtempSync(join(tmpdir(), "runner-indexed-proof-"));
const path = join(root, "runner-state.json");
const database = join(root, "runner-state.sqlite");
const locator = JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/one/session" });
writeFileSync(path, locator, { mode: 0o600 });
const db = new DatabaseSync(database);
chmodSync(database, 0o600);
writeFileSync(`${database}.lifetime`, "", { mode: 0o600 });
try {
db.exec("PRAGMA journal_mode=WAL; CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB); CREATE TABLE receipts(key TEXT PRIMARY KEY,bytes BLOB)");
db.prepare("INSERT INTO store_binding VALUES (1,?,?)").run(schema, "runner/one/session");
if (schema === 2) db.exec("CREATE TABLE receipt_prepare(singleton INTEGER PRIMARY KEY,state_key TEXT,generation INTEGER,body BLOB,digest BLOB)");
db.prepare("INSERT INTO receipts VALUES ('ancient',?)").run(Buffer.from("original outcome"));
const activeInput = schema === 2 ? "界".repeat(500_000) : "small";
const write = (generation: number | bigint | string, ackedSourceSeq: number) => {
const body = Buffer.from(JSON.stringify({ lifecycle: "ready", nextSourceSeq: 101, ackedSourceSeq, pendingTerminalDelivery: null, activeInput }));
db.prepare("INSERT OR REPLACE INTO current_state VALUES ('runner',?,?,?)").run(generation, body, createHash("sha256").update(body).digest());
};
write(1, 100);
expect(await readIndexedLocalState(path)).toMatchObject({ generation: "1", state: { outbox: [], lifecycle: "ready", activeInput } });
rmSync(`${database}.lifetime`);
await expect(readIndexedLocalState(path)).rejects.toThrow();
expect(existsSync(`${database}.lifetime`)).toBe(false);
const foreign = join(root, "foreign-lifetime");
writeFileSync(foreign, "foreign", { mode: 0o600 });
symlinkSync(foreign, `${database}.lifetime`);
await expect(readIndexedLocalState(path)).rejects.toThrow();
expect(readFileSync(foreign, "utf8")).toBe("foreign");
rmSync(`${database}.lifetime`);
writeFileSync(`${database}.lifetime`, "", { mode: 0o600 });
if (schema === 2) {
db.exec("CREATE TABLE backup_manifest(singleton INTEGER PRIMARY KEY,id TEXT,complete INTEGER); INSERT INTO backup_manifest VALUES(1,'backup',0)");
await expect(readIndexedLocalState(path)).rejects.toThrow(/incomplete backup/);
await expect(suspendIndexedRunnerState(path, "1")).rejects.toThrow("backup_incomplete");
expect(db.prepare("SELECT generation FROM current_state").get()!.generation).toBe(1);
db.exec("UPDATE backup_manifest SET complete=1");
db.exec("INSERT INTO receipt_prepare SELECT 1,key,generation,bytes,digest FROM current_state");
expect(await readIndexedLocalState(path)).toMatchObject({ generation: "1", state: { lifecycle: "ready" } });
await suspendIndexedRunnerState(path, "1");
expect(db.prepare("SELECT generation FROM receipt_prepare").get()!.generation).toBe(2);
expect(await readIndexedLocalState(path)).toMatchObject({ generation: "2", state: { lifecycle: "suspended" } });
db.exec("UPDATE receipt_prepare SET generation=1");
await expect(readIndexedLocalState(path)).rejects.toThrow(/conflicting prepared receipts/);
db.exec("DELETE FROM receipt_prepare");
}
write(2, 100);
await expect(suspendIndexedRunnerState(path, "1")).rejects.toThrow("stale_or_unsettled");
await suspendIndexedRunnerState(path, "2");
expect(await readIndexedLocalState(path)).toMatchObject({ generation: "3", state: { outbox: [], lifecycle: "suspended" } });
expect(readFileSync(path, "utf8")).toBe(locator);
expect(Buffer.from(db.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome");
write(9223372036854775807n, 100);
expect((await readIndexedLocalState(path)).generation).toBe("9223372036854775807");
await suspendIndexedRunnerState(path, "9223372036854775807");
const opaque = (await readIndexedLocalState(path)).generation;
expect(opaque).toMatch(/^r:[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/);
await expect(suspendIndexedRunnerState(path, "9223372036854775807")).rejects.toThrow("stale_or_unsettled");
await suspendIndexedRunnerState(path, opaque);
expect((await readIndexedLocalState(path)).generation).not.toBe(opaque);
expect(Buffer.from(db.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome");
write(4, 99);
expect((await readIndexedLocalState(path)).state.outbox).toEqual([{ indexedPending: true }]);
await expect(suspendIndexedRunnerState(path, "4")).rejects.toThrow("stale_or_unsettled");
db.exec("UPDATE current_state SET digest=zeroblob(32)");
await expect(readIndexedLocalState(path)).rejects.toThrow(/digest.*mismatch/);
} finally {
db.close();
rmSync(root, { recursive: true, force: true });
}
}, 20_000);
it("reaps its native reader before releasing a failed or timed-out read", async () => {
const root = mkdtempSync(join(tmpdir(), "runner-native-reader-owner-"));
try {
const binary = join(root, "reader");
writeFileSync(binary, `#!${process.execPath}\nconst fs = require('node:fs');\nconst path = process.argv.at(-1);\nfs.writeFileSync(path + '.pid', String(process.pid));\nprocess.stdout.write(path.endsWith('invalid') ? 'invalid\\n' + 'x'.repeat(2 * 1024 * 1024) : JSON.stringify({schema:'paperclip.indexed-inspection.v1',byteLength:1,sha256:'a'.repeat(64)})+'\\n');\nsetInterval(() => {}, 1000);\n`, { mode: 0o700 });
for (const kind of ["invalid", "stalled"]) {
const path = join(root, kind);
await expect(readIndexedLocalState(path, { runnerBinary: binary, timeoutMs: 2000 })).rejects.toThrow(kind === "invalid" ? "invalid_or_incomplete" : "read_timeout");
const pid = Number(readFileSync(`${path}.pid`, "utf8"));
expect(() => process.kill(pid, 0)).toThrow();
}
await expect(readIndexedLocalState(join(root, "absent"), { runnerBinary: join(root, "missing-binary") })).rejects.toThrow("ENOENT");
// All ownership/admission slots are available after failures.
const failed = await Promise.allSettled(Array.from({ length: 4 }, () => readIndexedLocalState(join(root, "invalid"), { runnerBinary: binary })));
expect(failed.every(result => result.status === "rejected" && String(result.reason).includes("invalid_or_incomplete"))).toBe(true);
} finally { rmSync(root, { recursive: true, force: true }); }
}, 15_000);
@@ -0,0 +1,83 @@
import { spawn } from "node:child_process";
import { Worker } from "node:worker_threads";
import { basename, dirname } from "node:path";
import { defaultStorageRunnerBinary, nativeStorageCommand } from "./native-storage-command.js";
import { authorityGeneration } from "./durable-authority-store.js";
let readers = 0;
export interface IndexedLocalSnapshot {
state: Record<string, unknown>;
generation: string;
stateDigest: string;
preparation: Record<string, unknown> | null;
}
/** Reads one runner-owned current-state row off the JavaScript event loop. */
export function readIndexedLocalState(path: string, options: { runnerBinary?: string; timeoutMs?: number } = {}): Promise<IndexedLocalSnapshot> {
return localStateJob(path, options);
}
/** Caller must independently fence and prove the runner/process group stopped.
* Only the exact verified, empty checkpoint can be sealed for cold continuation. */
export async function suspendIndexedRunnerState(path: string, expectedGeneration: string, runnerBinary?: string): Promise<void> {
if (basename(path) !== "runner-state.json" || expectedGeneration === "0") throw new Error("indexed_state_seal_invalid");
try { authorityGeneration(expectedGeneration); } catch { throw new Error("indexed_state_seal_invalid"); }
await nativeStorageCommand(["seal", "--directory", dirname(path), "--generation", expectedGeneration], runnerBinary);
}
async function localStateJob(path: string, options: { runnerBinary?: string; timeoutMs?: number }): Promise<IndexedLocalSnapshot> {
if (readers >= 4) throw new Error("indexed_state_reader_busy");
const timeoutMs = options.timeoutMs ?? 30_000;
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30_000) throw new Error("indexed_state_read_timeout_invalid");
readers += 1;
let worker: Worker | undefined;
let child: ReturnType<typeof spawn> | undefined;
let closed: Promise<void> | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
worker = new Worker(new URL(`./indexed-native-state-worker.${extension}`, import.meta.url), { execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 128 } });
child = spawn(options.runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "inspect-state", "--path", path], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, env: {} });
closed = new Promise<void>(resolve => child!.once("close", () => resolve()));
return await new Promise<IndexedLocalSnapshot>((resolve, reject) => {
let settled = false, stderr = "";
const fail = (error: Error) => { if (!settled) { settled = true; reject(error); } };
timer = setTimeout(() => fail(new Error("indexed_state_read_timeout")), timeoutMs);
worker!.on("message", (message: { consumed?: boolean; value?: IndexedLocalSnapshot; error?: string }) => {
if (settled) return;
if (message.error) fail(new Error(message.error));
else if (message.value) { settled = true; resolve(message.value); }
else if (message.consumed) child!.stdout!.resume();
else fail(new Error("indexed_state_reader_invalid_message"));
});
worker!.once("error", fail);
worker!.once("exit", () => { if (!settled) fail(new Error("indexed_state_reader_exited")); });
child!.once("error", fail);
child!.stderr!.on("data", (chunk: Buffer) => { if (stderr.length < 8192) stderr += chunk.toString("utf8").slice(0, 8192 - stderr.length); });
child!.stdout!.on("data", (chunk: Buffer) => {
// One unconsumed transport chunk at a time; worker message queues must
// not become a second current-state/history buffer.
child!.stdout!.pause();
if (!settled) worker!.postMessage({ bytes: chunk });
});
child!.once("close", code => {
if (settled) return;
if (code !== 0) fail(new Error(stderr.trim() || "indexed_state_native_reader_failed"));
else worker!.postMessage({ complete: true });
});
});
} finally {
if (timer) clearTimeout(timer);
// A decoder failure or deadline cannot leak a native SQLite reader/lifetime
// fence. Keep the admission credit until the exact owned child is reaped.
try {
if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
// Failure may leave stdout paused with additional bytes in the pipe.
// Destroy owned streams so unread output cannot delay the close/reap.
child?.stdout?.destroy(); child?.stderr?.destroy();
await closed;
} finally {
try { await worker?.terminate(); } finally { readers -= 1; }
}
}
}
@@ -0,0 +1,82 @@
import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from "node:fs";
import { dirname, resolve, basename } from "node:path";
import { createHash } from "node:crypto";
import { DatabaseSync } from "node:sqlite";
import { parentPort, workerData } from "node:worker_threads";
function privateFile(path: string, maximum: number): Buffer {
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > maximum || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("indexed_state_file_unsafe");
const bytes = Buffer.alloc(stat.size);
let offset = 0;
while (offset < bytes.length) { const count = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!count) throw new Error("indexed_state_file_changed"); offset += count; }
const after = fstatSync(fd);
if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("indexed_state_file_changed");
return bytes;
} finally { closeSync(fd); }
}
try {
const path = String(workerData.path);
const directory = dirname(path);
const dir = lstatSync(directory);
if (!dir.isDirectory() || dir.isSymbolicLink() || (process.platform !== "win32" && ((dir.mode & 0o077) !== 0 || dir.uid !== process.getuid?.()))) throw new Error("indexed_state_directory_unsafe");
const locator = JSON.parse(privateFile(path, 4096).toString("utf8")) as { schema?: string; binding?: string };
const type = locator.schema === "paperclip.runner.durable.state.indexed.v1" ? "runner"
: locator.schema === "paperclip.runner.codex-provider-state.indexed.v1" ? "codex-provider" : null;
if (!type || typeof locator.binding !== "string" || locator.binding.length > 1024) throw new Error("indexed_state_locator_invalid");
const databasePath = resolve(realpathSync(directory), `${basename(path, ".json")}.sqlite`);
for (const suffix of ["", "-wal", "-shm"]) {
const stat = lstatSync(databasePath + suffix, { throwIfNoEntry: false });
if (!stat && suffix) continue;
if (!stat || !stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("indexed_state_database_unsafe");
}
const db = new DatabaseSync(databasePath, { readOnly: true });
let result: Record<string, unknown>;
try {
db.exec("PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000; PRAGMA query_only=ON;");
const binding = db.prepare("SELECT schema_version,binding FROM store_binding WHERE singleton=1").get();
if (![1, 2].includes(Number(binding?.schema_version)) || binding?.binding !== locator.binding) throw new Error("indexed_state_binding_mismatch");
db.exec("BEGIN");
const backup = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_manifest'").get();
if (backup && db.prepare("SELECT complete FROM backup_manifest WHERE singleton=1").get()?.complete !== 1) throw new Error("indexed_state_backup_incomplete");
const importing = db.prepare("SELECT CASE WHEN length(bytes)<=16384 THEN bytes END AS bytes,digest FROM current_state WHERE key='legacy-import'").get();
let preparation: Record<string, unknown> | null = null;
if (importing) {
if (!(importing.bytes instanceof Uint8Array) || !(importing.digest instanceof Uint8Array)
|| !createHash("sha256").update(importing.bytes).digest().equals(Buffer.from(importing.digest))) throw new Error("indexed_state_import_invalid");
const progress = JSON.parse(Buffer.from(importing.bytes).toString()) as Record<string, unknown>;
if (progress.schema !== "paperclip.runner.local-import.v1" || progress.binding !== locator.binding || !progress.prepared) throw new Error("indexed_state_import_incomplete");
preparation = progress.prepared as Record<string, unknown>;
}
const metadata = db.prepare("SELECT length(bytes) AS size FROM current_state WHERE key=?").get(type);
if (!metadata || typeof metadata.size !== "number" || metadata.size > 32 * 1024 * 1024) throw new Error("indexed_state_snapshot_invalid");
const statement = db.prepare("SELECT generation,bytes,CASE WHEN length(digest)=32 THEN digest END AS digest,length(bytes) AS size FROM current_state WHERE key=?");
statement.setReadBigInts(true);
const row = statement.get(type);
if (!row || typeof row.generation !== "bigint" || row.generation <= 0n || Number(row.size) > 32 * 1024 * 1024 || !(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array)) throw new Error("indexed_state_snapshot_invalid");
const bytes = Buffer.from(row.bytes);
if (!createHash("sha256").update(bytes).digest().equals(Buffer.from(row.digest))) throw new Error("indexed_state_digest_mismatch");
const state = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
const stateDigest = Buffer.from(row.digest).toString("hex");
const generation = row.generation;
if (binding!.schema_version === 2) {
const pendingQuery = db.prepare("SELECT generation,CASE WHEN length(body)<=33554432 THEN body END AS body,digest FROM receipt_prepare WHERE state_key=?");
pendingQuery.setReadBigInts(true);
const pending = pendingQuery.get(type);
if (pending && (pending.generation !== generation || !(pending.body instanceof Uint8Array) || !(pending.digest instanceof Uint8Array) || !bytes.equals(Buffer.from(pending.body)) || !Buffer.from(row.digest).equals(Buffer.from(pending.digest)))) throw new Error("indexed_state_prepared_commit_mismatch");
}
if (type === "runner") {
const next = state.nextSourceSeq, acked = state.ackedSourceSeq;
if (!Number.isSafeInteger(next) || !Number.isSafeInteger(acked) || Number(next) <= Number(acked)) throw new Error("indexed_state_cursor_invalid");
// This is a proof view, not an outbox restore. A non-empty outbox remains
// a negative fence without loading the pending payloads into the server.
state.outbox = Number(next) === Number(acked) + 1 ? [] : [{ indexedPending: true }];
}
result = { state, generation: String(generation), stateDigest, preparation };
} finally { db.close(); }
parentPort!.postMessage(result);
} catch (error) {
parentPort!.postMessage({ error: error instanceof Error ? error.message : "indexed_state_reader_failed" });
}
@@ -0,0 +1,22 @@
import { parentPort } from "node:worker_threads";
// Source-mode workers run through Node's type stripping, compiled workers use
// the adjacent JS module. The worker owns only bounded decoding, never SQLite
// or a child process that could outlive worker termination.
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
const { IndexedInspectionDecoder } = await import(new URL(`./indexed-inspection-codec.${extension}`, import.meta.url).href);
const decoder = new IndexedInspectionDecoder();
parentPort!.on("message", (message: { bytes?: Uint8Array; complete?: boolean }) => {
try {
if (message.bytes) {
decoder.push(message.bytes);
parentPort!.postMessage({ consumed: true });
} else if (message.complete) {
parentPort!.postMessage({ value: decoder.finish() });
parentPort!.close();
} else throw new Error("indexed_state_reader_invalid_message");
} catch (error) {
parentPort!.postMessage({ error: error instanceof Error ? error.message : "indexed_state_reader_failed" });
parentPort!.close();
}
});
@@ -0,0 +1,205 @@
import { createHash, randomUUID } from "node:crypto";
import { constants } from "node:fs";
import { link, lstat, mkdir, open, rename } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { authorityDirectlyFollows, authorityJson, type DurableAuthorityStore } from "./durable-authority-store.js";
import { legacyControllerActivation, stageLegacyControlPlaneAuthority, type StoredCoreState } from "./durable-prp-control-plane.js";
import { readIndexedLocalState } from "./indexed-local-state-reader.js";
import type { DurableRecoveryIdentity } from "./prp-transport-types.js";
export interface PreparedLocalAuthority {
schema: "paperclip.runner.prepared-legacy-authority.v1";
binding: string;
fenceId: string;
sourceDigest: string;
currentDigest: string;
currentKey: "runner" | "codex-provider";
generation: 1;
receipts: number;
}
export interface PreparedLocalSession { runner: PreparedLocalAuthority; provider: PreparedLocalAuthority }
type SourceIdentity = { dev: string; ino: string; size: string; mtime: string; ctime: string };
type Preparation = PreparedLocalSession & { controllerGeneration: string; controllerDigest: string; files: string[]; sources: Record<string, SourceIdentity> };
interface Marker {
schema: "paperclip.runner.legacy-activation.v1";
identity: DurableRecoveryIdentity;
fenceId: string;
phase: "copying" | "activating" | "active";
preparation: Preparation | null;
preparationDigest: string | null;
}
export interface LegacySessionMigrationOptions {
root: string;
identity: DurableRecoveryIdentity;
authority: DurableAuthorityStore;
fenceId: string;
/** Use the same qualified native artifact for staging and current reads. */
runnerBinary?: string;
/** Must prove exclusive controller ownership AND every old runner/provider
* writer stopped. An expired lease or a missing PID alone is insufficient. */
assertExclusiveFence(): Promise<void>;
/** Run the qualified runnerd `storage stage-legacy` command with the exact
* source invocation. It writes only this private destination. */
prepareLocal(destination: string): Promise<PreparedLocalSession>;
onBoundary?(name: string): Promise<void>;
}
const digest = (value: unknown) => createHash("sha256").update(authorityJson(value)).digest("hex");
const prefixes = ["runner-state", "codex-provider-state"];
const dataFiles = prefixes.flatMap(prefix => [".sqlite", ".sqlite-wal", ".sqlite-shm", ".sqlite.receipts", ".sqlite.routing", ".sqlite.lifetime", ".sqlite.lock"].map(suffix => prefix + suffix));
const locators = prefixes.map(prefix => `${prefix}.json`);
const isStoreDirectory = (file: string) => file.endsWith(".receipts") || file.endsWith(".routing");
async function stat(path: string) { try { return await lstat(path); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; } }
async function privatePath(path: string, directory = false): Promise<void> {
const value = await lstat(path);
if (value.isSymbolicLink() || (directory ? !value.isDirectory() : !value.isFile()) ||
(process.platform !== "win32" && ((value.mode & 0o077) !== 0 || value.uid !== process.getuid?.()))) throw new Error("native_legacy_migration_unsafe");
}
async function sourceIdentity(path: string): Promise<SourceIdentity> {
await privatePath(path);
const value = await lstat(path, { bigint: true });
return { dev: String(value.dev), ino: String(value.ino), size: String(value.size), mtime: String(value.mtimeNs), ctime: String(value.ctimeNs) };
}
function sameSource(a: SourceIdentity, b: SourceIdentity, linked = false) {
return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtime === b.mtime && (linked || a.ctime === b.ctime);
}
async function syncDirectory(path: string) {
if (process.platform === "win32") return;
const fd = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
try { await fd.sync(); } finally { await fd.close(); }
}
async function publish(path: string, value: unknown) {
if (await stat(path)) await privatePath(path);
const temporary = `${path}.${randomUUID()}.tmp`;
const file = await open(temporary, "wx", 0o600);
try { await file.writeFile(JSON.stringify(value)); await file.sync(); } finally { await file.close(); }
await rename(temporary, path);
await syncDirectory(dirname(path));
}
async function readMarker(path: string): Promise<Marker | null> {
if (!await stat(path)) return null;
await privatePath(path);
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const before = await file.stat();
if (before.size > 16 * 1024) throw new Error("native_legacy_migration_marker_oversized");
const bytes = Buffer.alloc(before.size);
let offset = 0;
while (offset < bytes.length) { const { bytesRead } = await file.read(bytes, offset, bytes.length - offset, offset); if (!bytesRead) throw new Error("native_legacy_migration_changed"); offset += bytesRead; }
const after = await file.stat();
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("native_legacy_migration_changed");
return JSON.parse(bytes.toString()) as Marker;
} finally { await file.close(); }
}
function validateLocal(proof: PreparedLocalAuthority, kind: "runner" | "codex-provider", identity: DurableRecoveryIdentity, fenceId: string) {
const binding = `${kind === "runner" ? "runner" : "provider"}/${identity.runnerInstanceId}/${identity.normalizedSessionId}`;
if (proof.schema !== "paperclip.runner.prepared-legacy-authority.v1" || proof.binding !== binding || proof.fenceId !== fenceId || proof.currentKey !== kind || proof.generation !== 1 ||
!Number.isSafeInteger(proof.receipts) || proof.receipts < 0 || !/^[a-f0-9]{64}$/.test(proof.sourceDigest) || !/^[a-f0-9]{64}$/.test(proof.currentDigest)) throw new Error("native_legacy_preparation_invalid");
}
async function verifyLocal(directory: string, prepared: PreparedLocalSession, identity: DurableRecoveryIdentity, fenceId: string, runnerBinary?: string) {
for (const [name, proof] of [["runner-state", prepared.runner], ["codex-provider-state", prepared.provider]] as const) {
validateLocal(proof, name === "runner-state" ? "runner" : "codex-provider", identity, fenceId);
const snapshot = await readIndexedLocalState(resolve(directory, `${name}.json`), { runnerBinary });
if (snapshot.generation !== "1" || snapshot.stateDigest !== proof.currentDigest || authorityJson(snapshot.preparation) !== authorityJson(proof)) throw new Error("native_legacy_preparation_changed");
if (name === "runner-state" && Object.entries(identity).some(([key, value]) => snapshot.state[key] !== value)) throw new Error("native_legacy_runner_binding_changed");
}
}
/** Roll forward a three-store migration. The durable intent gates BOTH formats
* until the exact controller commit and all local stores have been published.
* Originals remain private and inspectable; no rollback guesses from history. */
export async function migrateLegacySessionAuthority(options: LegacySessionMigrationOptions): Promise<void> {
const { identity, fenceId, authority } = options;
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(fenceId)) throw new Error("native_legacy_migration_fence_invalid");
const root = resolve(options.root), controller = resolve(root, "control-plane"), runner = resolve(root, "runner");
const staging = resolve(root, "indexed-migration"), local = resolve(staging, "runner"), originals = resolve(staging, "legacy");
const markerPath = resolve(root, "indexed-migration.json"), sourcePath = resolve(controller, "control-plane-state.json");
await options.assertExclusiveFence();
for (const path of [root, controller, runner]) await privatePath(path, true);
let marker = await readMarker(markerPath);
if (marker && (marker.schema !== "paperclip.runner.legacy-activation.v1" || marker.fenceId !== fenceId || authorityJson(marker.identity) !== authorityJson(identity)
|| !["copying", "activating", "active"].includes(marker.phase))) throw new Error("native_legacy_migration_conflict");
if (!marker) {
marker = { schema: "paperclip.runner.legacy-activation.v1", identity, fenceId, phase: "copying", preparation: null, preparationDigest: null };
await publish(markerPath, marker);
await options.onBoundary?.("intent");
}
// Completed execution may have advanced since migration. Never rewrite it.
if (marker.phase === "active") return;
for (const path of [staging, originals]) {
try { await mkdir(path, { mode: 0o700 }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
await privatePath(path, true); await syncDirectory(dirname(path));
}
if (marker.phase === "copying") {
const sources: Record<string, SourceIdentity> = {};
for (const name of ["control-plane-state.json", ...locators]) sources[name] = await sourceIdentity(name === "control-plane-state.json" ? sourcePath : resolve(runner, name));
const prepared = await stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId, assertExclusiveFence: options.assertExclusiveFence });
const peers = await options.prepareLocal(local);
await verifyLocal(local, peers, identity, fenceId, options.runnerBinary);
const runnerState = (await readIndexedLocalState(resolve(local, "runner-state.json"), { runnerBinary: options.runnerBinary })).state;
const projection = prepared.state.projection as unknown as StoredCoreState;
if (projection.warmTransition || Number(runnerState.ackedSourceSeq) > projection.ackedSourceSeq || Number(runnerState.nextSourceSeq) <= projection.ackedSourceSeq ||
Number(runnerState.lastControllerCommandSeq) >= projection.indexedState!.nextControllerSeq) throw new Error("native_legacy_peer_cursors_conflict");
await options.assertExclusiveFence();
// Preparation of local peers can take time. Revalidate the original
// controller identity after it, without scanning its copied prefix again.
await stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId, assertExclusiveFence: options.assertExclusiveFence });
for (const name of Object.keys(sources)) if (!sameSource(sources[name]!, await sourceIdentity(name === "control-plane-state.json" ? sourcePath : resolve(runner, name)))) throw new Error("native_legacy_source_changed");
const files: string[] = [];
for (const file of [...dataFiles, ...locators]) if (await stat(resolve(local, file))) { await privatePath(resolve(local, file), isStoreDirectory(file)); files.push(file); }
if (![...prefixes.flatMap(prefix => [`${prefix}.sqlite`, `${prefix}.sqlite.routing`]), ...locators].every(file => files.includes(file))) throw new Error("native_legacy_preparation_incomplete");
marker.preparation = { ...peers, controllerGeneration: prepared.generation, controllerDigest: digest(prepared.state), files, sources };
marker.preparationDigest = digest(marker.preparation);
marker.phase = "activating";
await publish(markerPath, marker);
await options.onBoundary?.("prepared");
}
const prepared = marker.preparation;
if (!prepared || marker.preparationDigest !== digest(prepared) || !Array.isArray(prepared.files) || new Set(prepared.files).size !== prepared.files.length
|| prepared.files.some(file => ![...dataFiles, ...locators].includes(file))) throw new Error("native_legacy_migration_preparation_invalid");
validateLocal(prepared.runner, "runner", identity, fenceId); validateLocal(prepared.provider, "codex-provider", identity, fenceId);
for (const name of ["control-plane-state.json", ...locators]) {
const backup = resolve(originals, name), linked = !!await stat(backup);
const source = linked ? backup : name === "control-plane-state.json" ? sourcePath : resolve(runner, name);
// Our hard link changes ctime but preserves the original inode, bytes and
// modification time. Before that link, require the full source identity.
if (!prepared.sources?.[name] || !sameSource(prepared.sources[name]!, await sourceIdentity(source), linked)) throw new Error("native_legacy_source_changed");
}
await options.assertExclusiveFence();
const current = await authority.load();
if (!current) throw new Error("native_legacy_controller_missing");
const receipt = (current.state.indexedState as StoredCoreState["indexedState"])?.legacyActivation;
if (receipt?.fenceId === fenceId && receipt.preparationDigest === marker.preparationDigest && !authorityDirectlyFollows(current, prepared.controllerGeneration)) throw new Error("native_legacy_controller_advanced_before_publication");
if (receipt?.fenceId !== fenceId || receipt.preparationDigest !== marker.preparationDigest) {
if (current.generation !== prepared.controllerGeneration || digest(current.state) !== prepared.controllerDigest) throw new Error("native_legacy_controller_changed");
const projection = legacyControllerActivation(current, identity, fenceId, marker.preparationDigest!);
await authority.commit({ expectedGeneration: current.generation, state: projection as unknown as Record<string, unknown>, records: [] });
await options.onBoundary?.("controller-commit");
}
// Preserve exact original bytes with no-clobber hard links before replacing
// any locator. All peers are stopped; the external fence is rechecked below.
for (const [name, path] of [["control-plane-state.json", sourcePath], ...locators.map(file => [file, resolve(runner, file)])] as const) {
const backup = resolve(originals, name!);
if (!await stat(backup)) { await privatePath(path!); await link(path!, backup); await syncDirectory(originals); }
else await privatePath(backup);
}
for (const file of prepared.files) {
await options.assertExclusiveFence();
const source = resolve(local, file), target = resolve(runner, file);
const sourceStat = await stat(source), targetStat = await stat(target);
if (sourceStat) {
await privatePath(source, isStoreDirectory(file));
if (targetStat && !locators.includes(file)) throw new Error("native_legacy_destination_conflict");
if (targetStat) await privatePath(target, isStoreDirectory(file));
await rename(source, target); await syncDirectory(local); await syncDirectory(runner);
await options.onBoundary?.(`publish:${file}`);
} else await privatePath(target, isStoreDirectory(file));
}
await verifyLocal(runner, prepared, identity, fenceId, options.runnerBinary);
await options.assertExclusiveFence();
await publish(sourcePath, { schema: "paperclip.runner.authority-locator.v1", location: authority.location });
await options.onBoundary?.("controller-locator");
marker.phase = "active";
await publish(markerPath, marker);
await options.onBoundary?.("active");
}
@@ -0,0 +1,92 @@
import { createHash } from "node:crypto";
import { mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { DurablePrpControlPlane, stageLegacyControlPlaneAuthority, type StoredCoreState } from "./durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "./sqlite-authority-store.js";
import type { DurableRecoveryCoreCommand, DurableRecoveryIdentity } from "./prp-transport-types.js";
const identity: DurableRecoveryIdentity = { runnerInstanceId: "migration-runner", environmentLeaseId: "migration-environment", normalizedSessionId: "migration-session", runId: "migration-run", turnId: "migration-turn", itemId: "migration-item" };
const command = (seq: number): DurableRecoveryCoreCommand => ({ schema: "paperclip.prp.command.v1",commandId:`command-${seq}`,controllerSeq:seq,type:"run.prepare",issuedAt:"2026-09-28T00:00:00.000Z",payload:{},status:"completed",result:{status:"completed",commandId:`command-${seq}`,controllerSeq:seq,commandType:"run.prepare"} });
const coreOptions = (directory: string) => ({ stateDirectory: directory, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
function event(seq: number, eventType: string, payload: Record<string, unknown>) {
return { sourceSeq:seq,sourceEventId:`event-${seq}`,eventType,priority:1,deliveryCount:1,logicalEffectCount:1,envelope:{protocol:"paperclip.runner",version:1,kind:"event",...identity,payload:{schema:"paperclip.prp.event.v1",schemaVersion:1,...identity,sourceSeq:seq,sourceEventId:`event-${seq}`,sourceInstanceId:identity.runnerInstanceId,sourceKind:"runner",eventType,priority:1,emittedAt:"2026-09-28T00:00:00.000Z",payload}} };
}
it("stages a >192 MiB controller journal with bounded current state, exact old receipts, and restartable checkpoints", async () => {
const directory = await mkdtemp(join(tmpdir(), "legacy-authority-"));
const sourcePath = join(directory, "control-plane-state.json");
const storage = { path: join(directory, "authority.sqlite"), binding: "legacy-import-test", create: true };
let authority = await SqliteAuthorityStore.open(storage);
try {
const legacy = await DurablePrpControlPlane.open(coreOptions(directory));
await legacy.stop();
const initial = JSON.parse(await readFile(sourcePath, "utf8"));
const fd = await open(sourcePath, "w", 0o600);
await fd.write('{"committedEvents":['); // history precedes commands deliberately
const payload = { code: "history", text: "x".repeat(100 * 1024) };
for (let seq = 1; seq <= 2000; seq++) await fd.write(`${seq > 1 ? "," : ""}${JSON.stringify(event(seq,"harness.diagnostic",payload))}`);
const tail = { ...initial, commands:[command(1),command(2)],ackedSourceSeq:2000,commandDeliveryCounts:{"command-1":1,"command-2":3} };
delete tail.committedEvents;
await fd.write(`],${JSON.stringify(tail).slice(1)}`);
await fd.close();
let checkpoints=0;
const input={sourcePath,identity,authority,fenceId:"exclusive-migration-1",assertExclusiveFence:async()=>{}};
await expect(stageLegacyControlPlaneAuthority({...input,onCheckpoint:async()=>{if(++checkpoints===3)throw new Error("simulated controller death");}})).rejects.toThrow("simulated controller death");
const interrupted=await authority.load();
expect((interrupted!.state.cursor as {offset:number}).offset).toBeGreaterThan(16*1024*1024);
await authority.close();
authority=await SqliteAuthorityStore.open({...storage,create:false});
const prepared=await stageLegacyControlPlaneAuthority({...input,authority});
expect(prepared.state.phase).toBe("prepared");
expect((prepared.state.cursor as {offset:number}).offset).toBeGreaterThan(192*1024*1024);
expect(JSON.stringify(prepared.state).length).toBeLessThan(16*1024);
const current=prepared.state.projection as unknown as StoredCoreState;
expect(current.commands.map(command=>command.commandId)).toEqual(["command-2"]);
expect(current.commandDeliveryCounts).toEqual({"command-2":3});
expect(current.indexedState!.nextControllerSeq).toBe(3);
expect((await authority.getRecord(identity.runId,"command","command-1"))!.body).toEqual(command(1));
expect((await authority.getRecord(identity.runId,"event","event-1"))!.body).toEqual(event(1,"harness.diagnostic",payload));
// Preparation must not replace the legacy locator or activate staging.
const beginning=await open(sourcePath,"r");
const bytes=Buffer.alloc(24);await beginning.read(bytes,0,bytes.length,0);await beginning.close();
expect(bytes.toString()).toContain('"committedEvents"');
await expect(DurablePrpControlPlane.open({...coreOptions(directory),authorityStore:authority})).rejects.toThrow();
await expect(stageLegacyControlPlaneAuthority({...input,authority,fenceId:"wrong-owner"})).rejects.toThrow("different authority or fence");
} finally {await authority.close();await rm(directory,{recursive:true,force:true});}
},180_000);
it("reconciles semantic inputs after all commands and refuses a lost ownership fence",async()=>{
const directory=await mkdtemp(join(tmpdir(),"legacy-semantic-import-"));
const sourcePath=join(directory,"control-plane-state.json");
const authority=await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"semantic-import",create:true});
try{
const legacy=await DurablePrpControlPlane.open(coreOptions(directory));await legacy.stop();
const initial=JSON.parse(await readFile(sourcePath,"utf8"));
const correlation={runId:identity.runId,normalizedSessionId:identity.normalizedSessionId,turnId:identity.turnId,itemId:identity.itemId};
const semantic=(callId:string)=>({semantic_tool:{callId,operationId:"get_task_context",correlation,input:{}}});
const commandId=`command_tool_${createHash("sha256").update(`${identity.runId}\0settled-call`).digest("hex").slice(0,32)}`;
const settled={...command(1),commandId,type:"semantic_tool.result",payload:{callId:"settled-call",operationId:"get_task_context",sourceEventId:"event-1",sourceEventType:"semantic_tool.input",correlation,input:{}},result:{commandId,controllerSeq:1,commandType:"semantic_tool.result",status:"completed"}};
const {committedEvents:_events,commands:_commands,...fields}=initial;
await writeFile(sourcePath,JSON.stringify({committedEvents:[event(1,"semantic_tool.input",semantic("settled-call")),event(2,"semantic_tool.input",semantic("pending-call"))],...fields,commands:[settled],ackedSourceSeq:2}),{mode:0o600});
const input={sourcePath,identity,authority,fenceId:"semantic-fence",assertExclusiveFence:async()=>{}};
await expect(stageLegacyControlPlaneAuthority({...input,assertExclusiveFence:async()=>{throw new Error("lease lost");}})).rejects.toThrow("lease lost");
expect(await authority.load()).toBeNull();
const prepared=await stageLegacyControlPlaneAuthority(input);
const current=prepared.state.projection as unknown as StoredCoreState;
expect(current.indexedState!.pendingSemanticInputIds).toEqual(["event-2"]);
expect(current.committedEvents.map(event=>event.sourceEventId)).toEqual(["event-2"]);
await writeFile(sourcePath,'{"changed":true}',{mode:0o600});
await expect(stageLegacyControlPlaneAuthority(input)).rejects.toThrow("source changed");
}finally{await authority.close();await rm(directory,{recursive:true,force:true});}
});
it("does not invent execution authority from missing legacy fields", async () => {
const directory = await mkdtemp(join(tmpdir(), "legacy-import-missing-"));
const sourcePath = join(directory, "control-plane-state.json");
const authority = await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "missing-fields", create: true });
try {
await writeFile(sourcePath, JSON.stringify({ identity }), { mode: 0o600 });
await expect(stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId: "missing-fence", assertExclusiveFence: async () => {} })).rejects.toThrow("missing required source fields");
expect(await authority.load()).toBeNull();
} finally { await authority.close(); await rm(directory, { recursive: true, force: true }); }
});
@@ -0,0 +1,58 @@
import { mkdtemp, open, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { LegacyJsonReader, type LegacyJsonCursor } from "./legacy-json-reader.js";
it("streams more than 192 MiB losslessly and resumes from a durable page cursor", async () => {
const directory = await mkdtemp(join(tmpdir(), "legacy-import-"));
const path = join(directory, "legacy.json");
const fd = await open(path, "wx", 0o600);
const payload = "A".repeat(128 * 1024) + ' escaped: \\" ] } λ';
await fd.write('{"schema":"legacy-test","commands":[],"committedEvents":[');
for (let index = 0; index < 1600; index++) await fd.write(`${index ? "," : ""}${JSON.stringify({ index, payload })}`);
await fd.write('],"commandDeliveryCounts":{"first":2},"identity":{"session":"same"}}');
await fd.close();
let reader = new LegacyJsonReader(path);
let cursor: LegacyJsonCursor | null = null;
let source: string | null = null;
let seen = 0, pages = 0;
const fields: Record<string, unknown> = {};
try {
for (;;) {
const page = await reader.page(source, cursor);
expect(page.entries.length).toBeLessThanOrEqual(128);
for (const entry of page.entries) {
if (entry.field === "committedEvents") { expect(entry.key).toBe(String(seen)); expect(entry.value).toEqual({ index: seen++, payload }); }
else fields[entry.field] = entry.value;
}
const before = cursor?.offset ?? 0;
cursor = page.cursor; source = page.source; pages++;
expect(cursor.offset).toBeGreaterThan(before);
if (page.done) break;
if (pages === 5) { await reader.close(); reader = new LegacyJsonReader(path); }
}
expect(seen).toBe(1600);
expect(cursor!.offset).toBeGreaterThan(192 * 1024 * 1024);
expect(fields).toEqual({ schema: "legacy-test", commandDeliveryCounts: 2, identity: { session: "same" } });
} finally { await reader.close(); await rm(directory, { recursive: true, force: true }); }
}, 60_000);
it("rejects a changed source on resume and malformed boundaries", async () => {
const directory = await mkdtemp(join(tmpdir(), "legacy-import-invalid-"));
const path = join(directory, "legacy.json");
try {
for (const content of ['{"commands":[1,]}', '{"a":1,"a":2}', '{"commands":[{"a":]}', '{"a":"unterminated}', '{"a":1} trailing', '{"a":1,}']) {
await writeFile(path, content, { mode: 0o600 });
const reader = new LegacyJsonReader(path);
try { await expect(reader.page(null, null)).rejects.toThrow(); } finally { await reader.close(); }
}
await writeFile(path, JSON.stringify({ committedEvents: Array.from({ length: 500 }, (_, i) => ({ i })) }), { mode: 0o600 });
const reader = new LegacyJsonReader(path);
try {
const page = await reader.page(null, null);
await writeFile(path, '{"changed":true}', { mode: 0o600 });
await expect(reader.page(page.source, page.cursor)).rejects.toThrow("source changed");
} finally { await reader.close(); }
} finally { await rm(directory, { recursive: true, force: true }); }
});
@@ -0,0 +1,30 @@
import { Worker } from "node:worker_threads";
import type { LegacyJsonCursor, LegacyJsonPage } from "./legacy-json-scanner.js";
export type { LegacyJsonCursor, LegacyJsonPage, LegacyJsonEntry } from "./legacy-json-scanner.js";
/** One bounded page at a time. Import has no whole-job deadline; an interrupted
* caller resumes at its last transactionally committed source cursor. */
export class LegacyJsonReader {
private readonly worker: Worker;
private pending: { resolve(page: LegacyJsonPage): void; reject(error: Error): void } | null = null;
private failure: Error | null = null;
constructor(private readonly path: string) {
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
this.worker = new Worker(new URL(`./legacy-json-worker.${extension}`, import.meta.url), { execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 128 } });
this.worker.on("message", (result: { page?: LegacyJsonPage; error?: string }) => {
const pending = this.pending; this.pending = null;
if (!pending) return;
if (result.error || !result.page) pending.reject(new Error(result.error ?? "missing legacy import page"));
else pending.resolve(result.page);
});
this.worker.on("error", (error) => this.fail(error));
this.worker.on("exit", () => this.fail(new Error("legacy import reader exited")));
}
private fail(error: Error): void { this.failure ??= error; this.pending?.reject(this.failure); this.pending = null; }
page(source: string | null, cursor: LegacyJsonCursor | null): Promise<LegacyJsonPage> {
if (this.failure) return Promise.reject(this.failure);
if (this.pending) return Promise.reject(new Error("legacy import already has a pending page"));
return new Promise((resolve, reject) => { this.pending = { resolve, reject }; this.worker.postMessage({ path: this.path, source, cursor }); });
}
async close(): Promise<void> { await this.worker.terminate(); }
}
@@ -0,0 +1,144 @@
import { createHash } from "node:crypto";
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
export interface LegacyJsonCursor {
offset: number;
phase: "start" | "field" | "entry" | "entry_separator" | "field_separator" | "done";
fields: string[];
collection: { field: string; kind: "array" | "object"; index: string } | null;
prefixDigest: string;
}
export interface LegacyJsonEntry { field: string; key: string | null; value: unknown }
export interface LegacyJsonPage { source: string; cursor: LegacyJsonCursor; entries: LegacyJsonEntry[]; done: boolean }
const CHUNK = 64 * 1024;
const MAX_VALUE = 16 * 1024 * 1024;
const MAX_PAGE = 8 * 1024 * 1024;
const collections: Record<string, "array" | "object"> = { commands: "array", committedEvents: "array", commandDeliveryCounts: "object" };
const invalid = () => new Error("invalid legacy migration JSON or cursor");
class Scanner {
offset: number;
private buffer = Buffer.allocUnsafe(CHUNK);
private begin = -1;
private length = 0;
private fd: number;
private size: number;
constructor(fd: number, size: number, offset: number) { this.fd = fd; this.size = size; this.offset = offset; }
peek(): number {
if (this.offset >= this.size) return -1;
if (this.offset < this.begin || this.offset >= this.begin + this.length) {
this.begin = this.offset;
this.length = readSync(this.fd, this.buffer, 0, Math.min(CHUNK, this.size - this.offset), this.offset);
if (!this.length) throw new Error("legacy migration source changed");
}
return this.buffer[this.offset - this.begin]!;
}
space(): void { while ([32, 9, 10, 13].includes(this.peek())) this.offset++; }
expect(byte: number): void { this.space(); if (this.peek() !== byte) throw invalid(); this.offset++; }
/** Locate one JSON value with bounded memory, then let the JSON parser
* validate its complete grammar. Delimiters inside strings remain data. */
value(maximum = MAX_VALUE): { value: unknown; bytes: number } {
this.space();
const start = this.offset;
const first = this.peek();
if (first < 0 || [44, 93, 125].includes(first)) throw invalid();
let depth = 0, quoted = false, escaped = false;
const container = first === 91 || first === 123;
for (;;) {
const byte = this.peek();
if (byte < 0) {
if (quoted || depth > 0 || container) throw invalid();
break;
}
if (!quoted && depth === 0 && this.offset > start && [32, 9, 10, 13, 44, 93, 125].includes(byte)) break;
this.offset++;
if (this.offset - start > maximum) throw new Error("legacy migration record exceeds bounded admission capacity");
if (quoted) {
if (escaped) escaped = false;
else if (byte === 92) escaped = true;
else if (byte === 34) { quoted = false; if (depth === 0) break; }
} else if (byte === 34) quoted = true;
else if (byte === 91 || byte === 123) { if (++depth > 128) throw invalid(); }
else if (byte === 93 || byte === 125) { if (--depth === 0) break; if (depth < 0) throw invalid(); }
}
const length = this.offset - start;
const bytes = Buffer.allocUnsafe(length);
let offset = 0;
while (offset < length) {
const count = readSync(this.fd, bytes, offset, length - offset, start + offset);
if (!count) throw new Error("legacy migration source changed");
offset += count;
}
return { value: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), bytes: length };
}
}
/** Worker-only lossless input. No total file limit; each page has a durable,
* seekable cursor. Callers must hold the migration's exclusive ownership fence
* and persist the page's records and cursor in the same staging transaction. */
export function scanLegacyJsonPage(path: string, expectedSource: string | null, previous: LegacyJsonCursor | null): LegacyJsonPage {
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = fstatSync(fd, { bigint: true });
if (!stat.isFile() || (process.platform !== "win32" && ((stat.mode & 0o077n) !== 0n || stat.uid !== BigInt(process.getuid!())))) throw new Error("unsafe legacy migration source");
const identity = (value: typeof stat) => JSON.stringify([value.dev, value.ino, value.size, value.mtimeNs, value.ctimeNs].map(String));
const source = identity(stat);
if (expectedSource !== null && expectedSource !== source) throw new Error("legacy migration source changed");
const size = Number(stat.size);
if (!Number.isSafeInteger(size)) throw new Error("legacy migration source exceeds filesystem seek range");
const cursor: LegacyJsonCursor = structuredClone(previous ?? { offset: 0, phase: "start", fields: [], collection: null, prefixDigest: "0".repeat(64) });
if (!Number.isSafeInteger(cursor.offset) || cursor.offset < 0 || cursor.offset > size || !Array.isArray(cursor.fields) || cursor.fields.length > 128 || !/^[0-9a-f]{64}$/.test(cursor.prefixDigest)) throw invalid();
const scanner = new Scanner(fd, size, cursor.offset);
const begin = cursor.offset;
const entries: LegacyJsonEntry[] = [];
let bytes = 0;
while (entries.length < 128 && bytes < MAX_PAGE && cursor.phase !== "done") {
if (cursor.phase === "start") { scanner.expect(123); cursor.phase = "field"; }
else if (cursor.phase === "field") {
scanner.space();
if (scanner.peek() === 125) { scanner.offset++; cursor.phase = "done"; continue; }
const key = scanner.value(4096).value;
if (typeof key !== "string" || cursor.fields.includes(key) || cursor.fields.length >= 128) throw invalid();
cursor.fields.push(key); scanner.expect(58);
const kind = Object.hasOwn(collections, key) ? collections[key]! : null;
if (kind) { scanner.expect(kind === "array" ? 91 : 123); cursor.collection = { field: key, kind, index: "0" }; cursor.phase = "entry"; }
else { const value = scanner.value(); entries.push({ field: key, key: null, value: value.value }); bytes += value.bytes; cursor.phase = "field_separator"; }
} else if (cursor.phase === "entry") {
const collection = cursor.collection;
if (!collection || collections[collection.field] !== collection.kind || !/^(0|[1-9][0-9]*)$/.test(collection.index) || collection.index.length > 32) throw invalid();
scanner.space();
if (scanner.peek() === (collection.kind === "array" ? 93 : 125)) { scanner.offset++; cursor.collection = null; cursor.phase = "field_separator"; continue; }
let key = collection.index;
if (collection.kind === "object") {
const parsed = scanner.value(4096).value;
if (typeof parsed !== "string") throw invalid();
key = parsed; scanner.expect(58);
}
const value = scanner.value();
entries.push({ field: collection.field, key, value: value.value }); bytes += value.bytes;
collection.index = String(BigInt(collection.index) + 1n); cursor.phase = "entry_separator";
} else if (cursor.phase === "entry_separator") {
const collection = cursor.collection;
if (!collection) throw invalid();
scanner.space();
if (scanner.peek() === (collection.kind === "array" ? 93 : 125)) { scanner.offset++; cursor.collection = null; cursor.phase = "field_separator"; }
else { scanner.expect(44); scanner.space(); if ([93, 125].includes(scanner.peek())) throw invalid(); cursor.phase = "entry"; }
} else if (cursor.phase === "field_separator") {
scanner.space();
if (scanner.peek() === 125) { scanner.offset++; cursor.phase = "done"; }
else { scanner.expect(44); scanner.space(); if (scanner.peek() === 125) throw invalid(); cursor.phase = "field"; }
} else throw invalid();
}
if (cursor.phase === "done") { scanner.space(); if (scanner.offset !== size) throw invalid(); }
const digest = createHash("sha256").update(Buffer.from(cursor.prefixDigest, "hex"));
const buffer = Buffer.allocUnsafe(CHUNK);
for (let offset = begin; offset < scanner.offset;) {
const count = readSync(fd, buffer, 0, Math.min(CHUNK, scanner.offset - offset), offset);
if (!count) throw new Error("legacy migration source changed");
digest.update(buffer.subarray(0, count)); offset += count;
}
if (source !== identity(fstatSync(fd, { bigint: true }))) throw new Error("legacy migration source changed");
cursor.offset = scanner.offset; cursor.prefixDigest = digest.digest("hex");
return { source, cursor, entries, done: cursor.phase === "done" };
} finally { closeSync(fd); }
}
@@ -0,0 +1,8 @@
import { parentPort } from "node:worker_threads";
import type { LegacyJsonCursor } from "./legacy-json-scanner.js";
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
const { scanLegacyJsonPage } = await import(new URL(`./legacy-json-scanner.${extension}`, import.meta.url).href) as typeof import("./legacy-json-scanner.js");
parentPort!.on("message", (input: { path: string; source: string | null; cursor: LegacyJsonCursor | null }) => {
try { parentPort!.postMessage({ page: scanLegacyJsonPage(input.path, input.source, input.cursor) }); }
catch (error) { parentPort!.postMessage({ error: error instanceof Error ? error.message : "legacy migration read failed" }); }
});
@@ -0,0 +1,24 @@
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { expect, it } from "vitest";
import { assertNoPendingLegacyMigration } from "./legacy-migration-gate.js";
import { readDurableControlPlaneState } from "./authority-locator.js";
it("holds absent/legacy/indexed admission while migration is partial and refuses unsafe markers", async () => {
const root = await mkdtemp(join(tmpdir(), "migration-gate-"));
const controller = join(root, "control-plane"), marker = join(root, "indexed-migration.json");
try {
await mkdir(controller, { mode: 0o700 });
expect(() => assertNoPendingLegacyMigration(controller)).not.toThrow();
for (const phase of ["copying", "activating", "unknown"]) {
await writeFile(marker, JSON.stringify({ schema: "paperclip.runner.legacy-activation.v1", phase }), { mode: 0o600 });
expect(() => assertNoPendingLegacyMigration(controller)).toThrow("migration_pending");
await expect(readDurableControlPlaneState(controller)).rejects.toThrow("migration_pending");
}
await writeFile(marker, JSON.stringify({ schema: "paperclip.runner.legacy-activation.v1", phase: "active" }), { mode: 0o600 });
expect(() => assertNoPendingLegacyMigration(controller)).not.toThrow();
await rm(marker); await symlink(join(root, "missing"), marker);
expect(() => assertNoPendingLegacyMigration(controller)).toThrow();
} finally { await rm(root, { recursive: true, force: true }); }
});
@@ -0,0 +1,22 @@
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
import { dirname, resolve } from "node:path";
/** Checked before opening either format. An interrupted migration must never
* turn a temporarily absent file into permission to create fresh authority. */
export function assertNoPendingLegacyMigration(stateDirectory: string): void {
const path = resolve(dirname(stateDirectory), "indexed-migration.json");
let fd: number;
try { fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > 16 * 1024 || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_legacy_migration_unsafe");
const bytes = Buffer.alloc(stat.size);
let offset = 0;
while (offset < bytes.length) { const count = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!count) throw new Error("native_legacy_migration_changed"); offset += count; }
const after = fstatSync(fd);
if (stat.size !== after.size || stat.mtimeMs !== after.mtimeMs || stat.ctimeMs !== after.ctimeMs) throw new Error("native_legacy_migration_changed");
const marker = JSON.parse(bytes.toString()) as {schema?:unknown;phase?:unknown};
if (marker.schema !== "paperclip.runner.legacy-activation.v1" || marker.phase !== "active") throw new Error("native_legacy_migration_pending");
} finally { closeSync(fd); }
}
@@ -0,0 +1,192 @@
import { parentPort, workerData } from "node:worker_threads";
import { DatabaseSync, type StatementSync } from "node:sqlite";
import { createHash } from "node:crypto";
import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, realpathSync } from "node:fs";
import { dirname, basename, resolve } from "node:path";
import type { AuthorityCommit, AuthorityRecord, AuthorityWorkRecord } from "./durable-authority-store.js";
// Support the source checkout and the compiled package without inheriting a
// test runner's loader into a storage worker. These modules are type-strippable.
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
const { authorityInteger, authorityJson, validateAuthorityCommit, validateAuthorityPage, validateAuthorityWorkKey, validateAuthorityWorkPage, DurableAuthorityStoreError, MAX_AUTHORITY_RECORD_BYTES } =
await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js");
const { path, binding, create, readOnly } = workerData as { path: string; binding: string; create: boolean; readOnly?: boolean };
let database: DatabaseSync;
function digest(bytes: string): string { return createHash("sha256").update(bytes).digest("hex"); }
function fail(code: InstanceType<typeof DurableAuthorityStoreError>["code"], message: string): never { throw new DurableAuthorityStoreError(code, message); }
function privateFile(filePath: string, optional: boolean): void {
let fd: number;
try { fd = openSync(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
catch (error) { if (optional && (error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; }
try {
const stat = fstatSync(fd);
if (!stat.isFile() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || (process.getuid && stat.uid !== process.getuid())))) fail("invalid_authority", "unsafe storage file");
} finally { closeSync(fd); }
}
function openDatabase(): DatabaseSync {
if (create && readOnly) fail("invalid_authority", "a proof reader cannot create authority");
if (typeof binding !== "string" || !binding.length || binding.length > 1024 || /[\x00-\x1f]/.test(binding)) fail("invalid_authority", "invalid storage binding");
if (create && !existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
const directory = lstatSync(dirname(path));
if (!directory.isDirectory() || directory.isSymbolicLink() || (process.platform !== "win32" && ((directory.mode & 0o077) !== 0 || (process.getuid && directory.uid !== process.getuid())))) fail("invalid_authority", "unsafe storage directory");
if (create) {
try { const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); try { fsyncSync(fd); } finally { closeSync(fd); } }
catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
}
for (const suffix of ["", "-wal", "-shm"]) privateFile(`${path}${suffix}`, suffix.length > 0);
const db = new DatabaseSync(resolve(realpathSync(dirname(path)), basename(path)), { enableForeignKeyConstraints: true, readOnly });
try {
const version = String(db.prepare("SELECT sqlite_version() AS version").get()?.version).split(".").map(Number);
if ((version[0]! * 1_000_000 + version[1]! * 1_000 + version[2]!) < 3_051_003) fail("storage_unavailable", "SQLite build lacks the WAL-reset fix");
db.exec("PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;");
if (readOnly) db.exec("PRAGMA query_only=ON");
else db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA wal_autocheckpoint=256; PRAGMA journal_size_limit=4194304;");
if (create) {
db.exec("BEGIN IMMEDIATE");
try {
db.exec(`CREATE TABLE IF NOT EXISTS authority_binding (singleton INTEGER PRIMARY KEY CHECK(singleton=1), schema_version INTEGER NOT NULL, binding TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS authority_state (singleton INTEGER PRIMARY KEY CHECK(singleton=1), generation INTEGER NOT NULL CHECK(generation>0), body TEXT NOT NULL, digest TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS authority_records (epoch TEXT NOT NULL, kind TEXT NOT NULL, id TEXT NOT NULL, sequence INTEGER NOT NULL, body TEXT NOT NULL, digest TEXT NOT NULL, PRIMARY KEY(epoch,kind,id), UNIQUE(epoch,kind,sequence)) WITHOUT ROWID;
CREATE UNIQUE INDEX IF NOT EXISTS authority_session_effect ON authority_records(id) WHERE kind='effect';`);
db.prepare("INSERT OR IGNORE INTO authority_binding VALUES (1,1,?)").run(binding);
db.exec("COMMIT");
} catch (error) { db.exec("ROLLBACK"); throw error; }
if (process.platform !== "win32") {
const fd = openSync(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY);
try { fsyncSync(fd); } finally { closeSync(fd); }
}
}
const stored = db.prepare("SELECT schema_version,binding FROM authority_binding WHERE singleton=1").get();
if (stored?.schema_version !== 1 || stored.binding !== binding) fail("invalid_authority", "storage binding or version mismatch");
if (!readOnly) db.exec("CREATE TABLE IF NOT EXISTS authority_work(collection TEXT NOT NULL,id TEXT NOT NULL,body TEXT NOT NULL,digest TEXT NOT NULL,PRIMARY KEY(collection,id)) WITHOUT ROWID");
return db;
} catch (error) { db.close(); throw error; }
}
function exactStatement(sql: string): StatementSync {
const statement = database.prepare(sql);
statement.setReadBigInts(true);
return statement;
}
function decode(row: Record<string, unknown>): AuthorityRecord {
if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "receipt digest mismatch");
return { epoch: String(row.epoch), kind: row.kind as AuthorityRecord["kind"], id: String(row.id), sequence: String(row.sequence), body: JSON.parse(row.body) };
}
function dispatch(operation: string, input: Record<string, unknown>): unknown {
switch (operation) {
case "ready": return null;
case "getWork": {
const collection = String(input.collection), id = String(input.id);
validateAuthorityWorkKey(collection, id);
if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_work'").get()) return null;
const row = database.prepare("SELECT CASE WHEN length(CAST(body AS BLOB))<=16384 THEN body END AS body,digest FROM authority_work WHERE collection=? AND id=?").get(collection, id);
if (!row) return null;
if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "outstanding-work digest mismatch");
return { collection, id, body: JSON.parse(row.body), sha256: row.digest };
}
case "readWorkPage": {
const collection = String(input.collection), after = String(input.after), limit = Number(input.limit), generation = String(input.expectedGeneration);
validateAuthorityWorkPage(collection, after, limit, generation);
database.exec("BEGIN");
try {
const current = exactStatement("SELECT generation FROM authority_state WHERE singleton=1").get()?.generation;
if (String(current) !== generation) fail("stale_authority", "outstanding-work page generation changed");
if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_work'").get()) return { records: [], nextAfter: null };
const ids = database.prepare("SELECT id FROM authority_work WHERE collection=? AND id>? ORDER BY id LIMIT ?").all(collection, after, limit);
const records = ids.map(row => dispatch("getWork", { collection, id: row.id }) as AuthorityWorkRecord);
return { records, nextAfter: records.at(-1)?.id ?? null };
} finally { database.exec("ROLLBACK"); }
}
case "load": {
const row = exactStatement("SELECT generation,CASE WHEN length(CAST(body AS BLOB))<=16777216 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_state WHERE singleton=1").get();
if (!row) return null;
if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "current authority digest mismatch");
return { generation: String(row.generation), state: JSON.parse(row.body) };
}
case "getRecord": {
const row = exactStatement("SELECT epoch,kind,id,sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE epoch=? AND kind=? AND id=?").get(String(input.epoch), String(input.kind), String(input.id));
return row ? decode(row) : null;
}
case "getSessionEffect": {
const row = exactStatement("SELECT epoch,kind,id,sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE kind='effect' AND id=?").get(String(input.id));
return row ? decode(row) : null;
}
case "readEvents": {
const after = String(input.after), limit = Number(input.limit), byteBudget = Number(input.byteBudget);
validateAuthorityPage(after, limit, byteBudget);
database.exec("BEGIN");
const records: AuthorityRecord[] = [];
try {
const rows = exactStatement("SELECT id,sequence,length(CAST(body AS BLOB)) AS bytes FROM authority_records WHERE epoch=? AND kind='event' AND sequence>? ORDER BY sequence LIMIT ?").all(String(input.epoch), authorityInteger(after), limit);
let bytes = 0;
for (const row of rows) {
const size = Number(row.bytes);
if (!Number.isSafeInteger(size) || size < 0 || size > MAX_AUTHORITY_RECORD_BYTES) fail("invalid_authority", "receipt exceeds byte capacity");
if (bytes + size > byteBudget) break;
const record = dispatch("getRecord", { epoch: input.epoch, kind: "event", id: row.id }) as AuthorityRecord | null;
if (!record) fail("invalid_authority", "receipt disappeared");
bytes += size;
records.push(record);
}
} finally { database.exec("ROLLBACK"); }
return { records, nextAfter: records.at(-1)?.sequence ?? null };
}
case "commit": {
if (readOnly) fail("invalid_authority", "proof readers cannot commit authority");
const commit = input as unknown as AuthorityCommit;
validateAuthorityCommit(commit);
if (commit.records.some(record => record.sequenceEpoch !== undefined)) fail("invalid_authority", "retained v1 store cannot rotate command sequences");
if (commit.records.some(record => record.kind === "effect" && record.sequence === "0")) fail("invalid_authority", "retained v1 store requires ordered effect receipts");
const expected = authorityInteger(commit.expectedGeneration), generation = expected + 1n;
authorityInteger(String(generation));
database.exec("BEGIN IMMEDIATE");
try {
const current = exactStatement("SELECT generation FROM authority_state WHERE singleton=1").get()?.generation ?? 0n;
if (current !== expected) fail("stale_authority", "authority generation changed");
for (const change of commit.work ?? []) {
const previous = dispatch("getWork", { collection: change.collection, id: change.id }) as AuthorityWorkRecord | null;
if ((previous?.sha256 ?? null) !== change.expectedSha256) fail("stale_authority", "outstanding-work digest changed");
if (change.body === null) database.prepare("DELETE FROM authority_work WHERE collection=? AND id=?").run(change.collection, change.id);
else {
const body = authorityJson(change.body);
database.prepare("INSERT INTO authority_work VALUES(?,?,?,?) ON CONFLICT(collection,id) DO UPDATE SET body=excluded.body,digest=excluded.digest").run(change.collection, change.id, body, digest(body));
}
}
for (const record of commit.records) {
const body = authorityJson(record.body);
const existing = exactStatement("SELECT sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE epoch=? AND kind=? AND id=?").get(record.epoch, record.kind, record.id);
if (existing) {
if (String(existing.sequence) !== record.sequence || existing.body !== body || existing.digest !== digest(body)) fail("receipt_conflict", "exact receipt differs");
} else {
const sequenceOwner = database.prepare("SELECT id FROM authority_records WHERE epoch=? AND kind=? AND sequence=?").get(record.epoch, record.kind, authorityInteger(record.sequence));
if (sequenceOwner) fail("receipt_conflict", "receipt sequence already belongs to another identity");
database.prepare("INSERT INTO authority_records(epoch,kind,id,sequence,body,digest) VALUES (?,?,?,?,?,?)").run(record.epoch, record.kind, record.id, authorityInteger(record.sequence), body, digest(body));
}
}
const body = authorityJson(commit.state);
database.prepare("INSERT INTO authority_state(singleton,generation,body,digest) VALUES (1,?,?,?) ON CONFLICT(singleton) DO UPDATE SET generation=excluded.generation,body=excluded.body,digest=excluded.digest").run(generation, body, digest(body));
database.exec("COMMIT");
return String(generation);
} catch (error) { if (database.isTransaction) database.exec("ROLLBACK"); throw error; }
}
case "close": database.close(); return null;
default: fail("invalid_authority", "unknown storage operation");
}
}
let startupError: unknown;
try { database = openDatabase(); } catch (error) { startupError = error; }
parentPort!.on("message", ({ id, operation, input }: { id: number; operation: string; input: Record<string, unknown> }) => {
try {
if (startupError) throw startupError;
parentPort!.postMessage({ id, value: dispatch(operation, input) });
} catch (error) {
parentPort!.postMessage({ id, error: { code: error instanceof DurableAuthorityStoreError ? error.code : "storage_unavailable", message: error instanceof Error ? error.message : "storage worker failed" } });
}
});
@@ -0,0 +1,23 @@
import { execFile } from "node:child_process";
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
const execute = promisify(execFile);
export function defaultStorageRunnerBinary(): string {
const root = resolve(fileURLToPath(new URL("../..", import.meta.url)));
const name = `paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`;
const staged = resolve(root, "dist/bin", name);
return existsSync(staged) ? staged : resolve(root, "runner/target/debug", name);
}
/** Storage-only native commands never launch a provider or inherit credentials. */
export async function nativeStorageCommand(args: string[], runnerBinary = defaultStorageRunnerBinary()): Promise<void> {
try {
await execute(runnerBinary, ["storage", ...args], { timeout: 30_000, maxBuffer: 8 * 1024, windowsHide: true, env: {} });
} catch (error) {
const failure = error as Error & { stderr?: string };
throw new Error(failure.stderr?.trim() || failure.message, { cause: error });
}
}
@@ -0,0 +1,130 @@
import { randomUUID } from "node:crypto";
import { normalizedEventId } from "./event-epochs.js";
import { expect, it, vi } from "vitest";
import type { PrpEvent } from "../protocol/replay-contract.js";
import { applyNormalizedBatch, acknowledgeNormalizedEvent, type NormalizedDeliveryPort, type NormalizedDeliveryState } from "./normalized-delivery.js";
import { NormalizedDeliveryWriter } from "./normalized-delivery-writer.js";
function event(sequence: number): PrpEvent {
return { schema: "paperclip.prp.event.v1", sourceEventId: `runner:run:${sequence}`, sourceSeq: sequence, sourceInstanceId: "runner", sourceKind: "runner", runId: "run", normalizedSessionId: "session", eventType: "harness.diagnostic", schemaVersion: 1, priority: 1, emittedAt: "2026-09-28T00:00:00.000Z", payload: { code: "test" } };
}
function fixture(initial: NormalizedDeliveryState | null = null) {
let current: NormalizedDeliveryState | null = initial;
const port: NormalizedDeliveryPort = {
epoch: "run",
load: () => structuredClone(current),
commit: vi.fn(async (batch) => current = applyNormalizedBatch(current, batch)),
acknowledge: async (value) => { current = acknowledgeNormalizedEvent(current!, value); },
};
const published: PrpEvent[] = [];
const failed = vi.fn();
let reducer = { value: "before" };
const writer = new NormalizedDeliveryWriter(port, () => structuredClone(reducer), (value) => published.push(value), failed);
return { port, writer, published, failed, setState: (value: string) => { reducer = { value }; } };
}
it("captures a whole synchronous reducer step and publishes only after durable success", async () => {
const { port, writer, published, setState } = fixture();
let release!: () => void;
const gate = new Promise<void>((resolve) => { release = resolve; });
const commit = port.commit;
port.commit = async (batch) => { await gate; return commit(batch); };
writer.emit({ ...event(1), payload: { code: "wire-optional", optional: undefined } });
writer.emit(event(2));
setState("after");
writer.changed({ epoch: "run", sourceSeq: 1, ordinal: 1 });
const done = writer.flush();
expect(published).toEqual([]);
expect(port.load()).toBeNull();
setState("later-uncommitted-step");
release(); await done;
expect(port.load()).toMatchObject({ driver: { value: "after" }, raw: { epoch: "run", sourceSeq: 1, ordinal: 1 }, produced: 2 });
expect(published[0]!.payload).toEqual({ code: "wire-optional" });
expect(Object.hasOwn(published[0]!.payload, "optional")).toBe(false);
await writer.acknowledge(published[0]!);
const replayed: PrpEvent[] = [];
new NormalizedDeliveryWriter(port, () => ({}), (value) => replayed.push(value), () => {});
expect(replayed).toEqual([event(2)]);
});
it("queues captured reducers across the old revision maximum using committed identities", async () => {
const initial: NormalizedDeliveryState = { schema: "paperclip.runner.normalized-delivery.v1", revision: Number.MAX_SAFE_INTEGER,
raw: { epoch: "run", sourceSeq: 0, ordinal: 0 }, driver: {}, produced: 0, acknowledged: 0, pending: [] };
const { port, writer, published, setState, failed } = fixture(initial);
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const commit = port.commit;
const before: Array<number | string> = [], after: Array<number | string> = [];
port.commit = async batch => {
await gate;
before.push(batch.expectedRevision);
const current = await commit(batch); after.push(current.revision); return current;
};
writer.emit(event(1)); setState("first");
await Promise.resolve(); // Capture while the first commit is still held.
writer.emit(event(2)); setState("second");
const done = writer.flush();
expect(published).toEqual([]);
release(); await done;
expect(before).toEqual([Number.MAX_SAFE_INTEGER, after[0]]);
expect(after[0]).toMatch(/^r:/); expect(after[1]).toMatch(/^r:/);
expect(after[0]).not.toBe(after[1]);
expect(port.load()).toMatchObject({ revision: after[1], driver: { value: "second" }, pending: [event(1), event(2)] });
expect(failed).not.toHaveBeenCalled();
await expect(port.commit({ expectedRevision: Number.MAX_SAFE_INTEGER, raw: initial.raw, driver: {}, events: [] })).rejects.toThrow("revision changed");
await expect(port.commit({ expectedRevision: after[0]!, raw: initial.raw, driver: {}, events: [] })).rejects.toThrow("revision changed");
await writer.acknowledge(event(1)); await writer.acknowledge(event(2));
const recovered = new NormalizedDeliveryWriter(port, () => ({ recovered: true }), value => published.push(value), failed);
recovered.emit(event(3)); await recovered.flush();
expect(published).toEqual([event(1), event(2), event(3)]);
expect(port.load()!.revision).not.toBe(after[1]);
});
it("fences a failed write without publishing it or emitting another diagnostic into the failed writer", async () => {
const { port, writer, published, failed } = fixture();
port.commit = async () => { throw new Error("storage_unavailable: worker exited"); };
writer.emit(event(1));
await expect(writer.flush()).rejects.toThrow("worker exited");
expect(port.load()).toBeNull();
expect(published).toEqual([]);
expect(failed).toHaveBeenCalledTimes(1);
expect(() => writer.emit(event(2))).toThrow("worker exited");
await expect(writer.waitForCapacity()).rejects.toThrow("worker exited");
});
it("resumes admission after acknowledgements release pending capacity", async () => {
const { writer, published, port } = fixture();
for (let sequence = 1; sequence <= 128; sequence++) writer.emit(event(sequence));
await writer.flush();
let admitted = false;
const waiting = writer.waitForCapacity().then(() => { admitted = true; });
await Promise.resolve();
expect(admitted).toBe(false);
await writer.acknowledge(published[0]!);
await waiting;
expect(admitted).toBe(true);
expect(port.load()!.pending).toHaveLength(127);
});
it("persists a pending queue across several source namespaces and acknowledges exact identities", async () => {
const initial: NormalizedDeliveryState = { schema: "paperclip.runner.normalized-delivery.v1", revision: 0,
raw: { epoch: "run", sourceSeq: 0, ordinal: 0 }, driver: {}, produced: Number.MAX_SAFE_INTEGER, acknowledged: Number.MAX_SAFE_INTEGER, pending: [] };
const { port, writer, published } = fixture(initial);
let fromEpoch: string | undefined, final = Number.MAX_SAFE_INTEGER;
for (let round = 0; round < 3; round++) {
const nextEpoch = randomUUID();
for (let seq = 1; seq <= 4; seq++) writer.emit({ ...event(seq), sourceEpoch: nextEpoch, sourceEventId: normalizedEventId("runner", "run", seq, nextEpoch),
...(seq === 1 ? { sourceEpochTransition: { schema: "paperclip.prp.event-epoch.v1", runId: "run", transitionId: randomUUID(), fromEpoch: fromEpoch ?? null, nextEpoch, finalOrdinal: final } } : {}) });
fromEpoch = nextEpoch; final = 4;
}
await writer.flush();
expect(port.load()).toMatchObject({ produced: 4, producedEpoch: fromEpoch, acknowledged: Number.MAX_SAFE_INTEGER });
const restored = fixture(port.load());
expect(restored.published).toEqual(published);
await expect(restored.writer.acknowledge(published[4]!)).rejects.toThrow("differs");
for (const value of published) await restored.writer.acknowledge(value);
expect(restored.port.load()).toMatchObject({ produced: 4, producedEpoch: fromEpoch, acknowledged: 4, acknowledgedEpoch: fromEpoch, pending: [] });
expect(new Set(published.map(e => e.sourceEventId)).size).toBe(12);
await expect(restored.writer.acknowledge(published[0]!)).rejects.toThrow("differs");
});
@@ -0,0 +1,92 @@
import type { PrpEvent } from "../protocol/replay-contract.js";
import type { DriverHistoryReceipt, NormalizedDeliveryPort, NormalizedDeliveryRevision, RawDeliveryCursor } from "./normalized-delivery.js";
/** Serializes short reducer steps, not provider RPCs or waits for user input.
* A synchronous step captures its events and resulting state together. Nothing
* reaches the run-log consumer before that transaction is durable. */
export class NormalizedDeliveryWriter {
#revision: NormalizedDeliveryRevision;
#raw: RawDeliveryCursor;
#events: PrpEvent[] = [];
#scheduled = false;
#tail: Promise<void> = Promise.resolve();
#failure: Error | null = null;
#queued = 0;
#capacityWaiters = new Set<() => void>();
constructor(
readonly port: NormalizedDeliveryPort,
private readonly snapshot: () => Record<string, unknown>,
private readonly publish: (event: PrpEvent) => void,
private readonly fail: (error: Error) => void,
private readonly takeReceipts: () => DriverHistoryReceipt[] = () => [],
) {
const restored = port.load();
this.#revision = restored?.revision ?? 0;
this.#raw = restored?.raw ?? { epoch: port.epoch, sourceSeq: 0, ordinal: 0 };
for (const event of restored?.pending ?? []) publish(structuredClone(event));
}
emit(event: PrpEvent): void {
if (this.#failure) throw this.#failure;
// The normalized event contract is JSON, matching the run-log boundary.
// Omitted optional fields must have identical bytes before and after replay.
this.#events.push(JSON.parse(JSON.stringify(event)) as PrpEvent);
this.changed();
}
get failure(): Error | null { return this.#failure; }
changed(raw?: RawDeliveryCursor): void {
if (this.#failure) throw this.#failure;
if (raw) this.#raw = structuredClone(raw);
if (this.#scheduled) return;
this.#scheduled = true;
queueMicrotask(() => this.#capture());
}
#capture(): void {
if (!this.#scheduled || this.#failure) return;
this.#scheduled = false;
try {
if (++this.#queued > 64) throw new Error("storage_pressure: normalized reducer commit queue is full");
const events = this.#events.splice(0);
const batch = { raw: structuredClone(this.#raw), driver: this.snapshot(), events, receipts: this.takeReceipts() };
this.#tail = this.#tail.then(async () => {
if (this.#failure) throw this.#failure;
const committed = await this.port.commit({ ...batch, expectedRevision: this.#revision });
this.#revision = committed.revision;
for (const event of events) this.publish(event);
}).catch((error: unknown) => this.#poison(error)).finally(() => { this.#queued--; });
} catch (error) { this.#poison(error); }
}
#poison(error: unknown): void {
if (this.#failure) return;
this.#failure = error instanceof Error ? error : new Error(String(error));
for (const wake of this.#capacityWaiters) wake();
this.#capacityWaiters.clear();
this.fail(this.#failure);
}
async acknowledge(event: PrpEvent): Promise<void> {
await this.port.acknowledge(event);
for (const wake of this.#capacityWaiters) wake();
this.#capacityWaiters.clear();
}
async waitForCapacity(): Promise<void> {
for (;;) {
if (this.#failure) throw this.#failure;
const pending = this.port.load()?.pending ?? [];
if (pending.length < 128 && Buffer.byteLength(JSON.stringify(pending)) < 2 * 1024 * 1024) return;
await new Promise<void>((resolve) => this.#capacityWaiters.add(resolve));
}
}
async flush(): Promise<void> {
this.#capture();
await this.#tail;
if (this.#failure) throw this.#failure;
}
}
@@ -0,0 +1,143 @@
import { isCommandEpoch } from "./command-epochs.js";
import { advanceSourceCursor, isEventEpochTransition, type EventEpochTransition, type SourceCursor } from "./event-epochs.js";
import { randomUUID } from "node:crypto";
import type { PrpEvent } from "../protocol/replay-contract.js";
import { validatePrpEvent } from "../protocol/replay-contract.js";
import { authorityGeneration, authorityJson, DurableAuthorityStoreError } from "./durable-authority-store.js";
/** A raw frame can expand into several driver notifications. This cursor is
* independent of both the runner ACK and the normalized run-log sequence. */
export interface RawDeliveryCursor { epoch: string; sourceEpoch?: string; sourceSeq: number; ordinal: number }
export const RAW_DELIVERY_FRAME_COMPLETE = Number.MAX_SAFE_INTEGER;
/** Retained checkpoints use safe integers. New revisions are equality-only
* identities, independent of raw and normalized event ordering. */
export type NormalizedDeliveryRevision = number | string;
export interface NormalizedDeliveryState {
schema: "paperclip.runner.normalized-delivery.v1";
revision: NormalizedDeliveryRevision;
raw: RawDeliveryCursor;
driver: Record<string, unknown>;
produced: number;
producedEpoch?: string;
acknowledged: number;
acknowledgedEpoch?: string;
lastSourceEpochTransition?: EventEpochTransition;
/** Only outstanding delivery, never completed history. */
pending: PrpEvent[];
}
export interface NormalizedDeliveryBatch {
expectedRevision: NormalizedDeliveryRevision;
raw: RawDeliveryCursor;
driver: Record<string, unknown>;
events: PrpEvent[];
receipts?: DriverHistoryReceipt[];
}
export type DriverHistoryCollection = "terminal" | "file" | "steering" | "lineage";
export interface DriverHistoryReceipt {
collection: DriverHistoryCollection;
key: string;
value: unknown;
}
export interface NormalizedDeliveryPort {
readonly epoch: string;
readonly eventEpochs?: { limit: number };
load(): NormalizedDeliveryState | null;
commit(batch: NormalizedDeliveryBatch): Promise<NormalizedDeliveryState>;
acknowledge(event: PrpEvent): Promise<void>;
history?: { get(collection: DriverHistoryCollection, key: string): Promise<unknown | null> };
}
export const MAX_NORMALIZED_PENDING_EVENTS = 256;
export const MAX_NORMALIZED_PENDING_BYTES = 4 * 1024 * 1024;
function invalid(message: string): never {
throw new DurableAuthorityStoreError("invalid_authority", `normalized delivery: ${message}`);
}
export function compareRawCursor(a: RawDeliveryCursor, b: RawDeliveryCursor): number {
if (a.epoch !== b.epoch || a.sourceEpoch !== b.sourceEpoch) invalid("raw cursor namespaces cannot be numerically compared");
return a.sourceSeq - b.sourceSeq || a.ordinal - b.ordinal;
}
export function validateNormalizedDelivery(value: NormalizedDeliveryState): void {
if (typeof value.revision === "number") {
if (!Number.isSafeInteger(value.revision) || value.revision < 0) invalid("invalid checkpoint revision");
} else if (typeof value.revision !== "string" || !value.revision.startsWith("r:")) invalid("invalid checkpoint revision");
else authorityGeneration(value.revision);
if (value.schema !== "paperclip.runner.normalized-delivery.v1"
|| typeof value.raw?.epoch !== "string" || !value.raw.epoch || value.raw.epoch.length > 240
|| (value.raw.sourceEpoch !== undefined && !isCommandEpoch(value.raw.sourceEpoch))
|| ![value.raw?.sourceSeq, value.raw?.ordinal, value.produced, value.acknowledged].every((n) => Number.isSafeInteger(n) && n >= 0)
|| (value.producedEpoch !== undefined && !isCommandEpoch(value.producedEpoch))
|| (value.acknowledgedEpoch !== undefined && !isCommandEpoch(value.acknowledgedEpoch))
|| !Array.isArray(value.pending)
|| !value.driver || typeof value.driver !== "object" || Array.isArray(value.driver)) invalid("invalid checkpoint");
if (value.pending.length > MAX_NORMALIZED_PENDING_EVENTS || Buffer.byteLength(authorityJson(value.pending)) > MAX_NORMALIZED_PENDING_BYTES) {
throw new DurableAuthorityStoreError("storage_pressure", "normalized delivery awaiting run-log acknowledgement");
}
if (value.producedEpoch !== undefined || value.lastSourceEpochTransition !== undefined) {
const t = value.lastSourceEpochTransition;
if (!isEventEpochTransition(t) || t.nextEpoch !== value.producedEpoch || t.runId !== value.raw.epoch) invalid("invalid normalized head receipt");
}
let cursor: SourceCursor = { sourceEpoch: value.acknowledgedEpoch, sourceSeq: value.acknowledged };
for (const event of value.pending) {
if (!validatePrpEvent(event).ok || event.runId !== value.raw.epoch) invalid("invalid pending event");
try { cursor = advanceSourceCursor(cursor, event); } catch { invalid("pending event sequence is not contiguous"); }
}
if (cursor.sourceSeq !== value.produced || cursor.sourceEpoch !== value.producedEpoch) invalid("missing pending event");
}
/** Build a candidate without changing published state. The owner commits it
* with the raw cursor and only then exposes its events to the consumer. */
export function applyNormalizedBatch(previous: NormalizedDeliveryState | null, batch: NormalizedDeliveryBatch, rawTransition?: EventEpochTransition): NormalizedDeliveryState {
if (previous) validateNormalizedDelivery(previous);
if (batch.expectedRevision !== (previous?.revision ?? 0)) {
throw new DurableAuthorityStoreError("stale_authority", "normalized consumer revision changed");
}
if (previous && batch.raw.epoch !== previous.raw.epoch) invalid("raw cursor belongs to another run epoch");
const crossing = previous?.raw.sourceEpoch !== batch.raw.sourceEpoch;
if (crossing) {
if (!previous || !isEventEpochTransition(rawTransition) || rawTransition.runId !== batch.raw.epoch
|| rawTransition.fromEpoch !== (previous.raw.sourceEpoch ?? null) || rawTransition.nextEpoch !== batch.raw.sourceEpoch
|| previous.raw.sourceSeq !== rawTransition.finalOrdinal || previous.raw.ordinal !== RAW_DELIVERY_FRAME_COMPLETE
|| batch.raw.sourceSeq !== 0 || batch.raw.ordinal !== 0 || batch.events.length !== 0) invalid("raw epoch boundary has no exact consumed predecessor");
}
if (!crossing && compareRawCursor(batch.raw, previous?.raw ?? { epoch: batch.raw.epoch, sourceSeq: 0, ordinal: 0 }) < 0) invalid(`raw cursor moved backwards (${previous!.raw.sourceSeq}:${previous!.raw.ordinal} -> ${batch.raw.sourceSeq}:${batch.raw.ordinal})`);
const initialSequence = previous?.produced ?? (batch.events[0]?.sourceSeq ?? 1) - 1;
let cursor: SourceCursor = { sourceSeq: initialSequence, sourceEpoch: previous?.producedEpoch };
let lastTransition = previous?.lastSourceEpochTransition;
for (const event of batch.events) {
try { cursor = advanceSourceCursor(cursor, event); } catch { invalid("produced event sequence is not contiguous"); }
if (event.sourceEpochTransition) lastTransition = event.sourceEpochTransition;
}
const state: NormalizedDeliveryState = {
schema: "paperclip.runner.normalized-delivery.v1",
revision: `r:${randomUUID()}`,
raw: structuredClone(batch.raw),
driver: structuredClone(batch.driver),
produced: cursor.sourceSeq,
...(cursor.sourceEpoch ? { producedEpoch: cursor.sourceEpoch } : {}),
...(previous?.acknowledgedEpoch ? { acknowledgedEpoch: previous.acknowledgedEpoch } : {}),
...(lastTransition ? { lastSourceEpochTransition: structuredClone(lastTransition) } : {}),
acknowledged: previous?.acknowledged ?? initialSequence,
pending: [...(previous?.pending ?? []), ...structuredClone(batch.events)],
};
validateNormalizedDelivery(state);
return state;
}
/** ACKs are accepted only for the exact next event, after the run-log writer
* reports durable success. A crash before this commit replays identical bytes. */
export function acknowledgeNormalizedEvent(previous: NormalizedDeliveryState, event: PrpEvent): NormalizedDeliveryState {
const next = previous.pending[0];
if (!next || authorityJson(next) !== authorityJson(event)) invalid("run-log acknowledgement differs from pending event");
const state = { ...previous, acknowledged: event.sourceSeq, pending: previous.pending.slice(1) };
if (event.sourceEpoch) state.acknowledgedEpoch = event.sourceEpoch;
else delete state.acknowledgedEpoch;
validateNormalizedDelivery(state);
return state;
}
@@ -0,0 +1,51 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { DurablePrpControlPlane } from "./durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "./sqlite-authority-store.js";
import { normalizedEpochCloseId, normalizedEventId } from "./event-epochs.js";
import type { PrpEvent } from "../protocol/replay-contract.js";
it("reopens bounded normalized heads across numeric exhaustion, keeps exact closes and refuses reused namespaces", async () => {
const root = await mkdtemp(join(tmpdir(), "normalized-epochs-"));
const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", runId: "run", turnId: "turn", itemId: "item" };
const storage = { path: join(root, "authority.sqlite"), binding: "normalized-epochs", create: true };
let store = await SqliteAuthorityStore.open(storage);
const options = { stateDirectory: root, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` };
let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store });
const event = (sourceSeq: number, sourceEpoch?: string): PrpEvent => ({ schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceEventId: normalizedEventId("runner", "run", sourceSeq, sourceEpoch),
sourceSeq, ...(sourceEpoch ? { sourceEpoch } : {}), sourceInstanceId: "runner", sourceKind: "runner", runId: "run", normalizedSessionId: "session", eventType: "harness.diagnostic", priority: 1,
emittedAt: "2026-09-29T00:00:00.000Z", payload: { code: "epoch-test" } });
const epochs = [randomUUID(), randomUUID(), randomUUID()];
const raw = { epoch: "run", sourceSeq: 0, ordinal: 0 };
try {
let port = core.normalizedDelivery()!;
expect(port.eventEpochs?.limit).toBe(1_048_576);
const lastLegacy = event(Number.MAX_SAFE_INTEGER);
await port.commit({ expectedRevision: 0, raw, driver: {}, events: [lastLegacy] });
const pending = [lastLegacy];
let previous: string | null = null, finalOrdinal = Number.MAX_SAFE_INTEGER;
for (const nextEpoch of epochs) {
const transition = { schema: "paperclip.prp.event-epoch.v1" as const, runId: "run", transitionId: randomUUID(), fromEpoch: previous, nextEpoch, finalOrdinal };
const events = [1, 2, 3, 4].map(n => ({ ...event(n, nextEpoch), ...(n === 1 ? { sourceEpochTransition: transition } : {}) }));
await port.commit({ expectedRevision: port.load()!.revision, raw, driver: { current: nextEpoch }, events }); pending.push(...events);
expect(await store.getRecord("run", "effect", normalizedEpochCloseId("run", "runner", previous))).toMatchObject({ sequence: "0", body: transition });
await core.stop(); await store.close(); store = await SqliteAuthorityStore.open({ ...storage, create: false });
core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); port = core.normalizedDelivery()!;
expect(port.load()!.pending).toEqual(pending);
previous = nextEpoch; finalOrdinal = 4;
}
const before = port.load();
await expect(port.commit({ expectedRevision: before!.revision, raw, driver: {}, events: [{ ...event(1, epochs[0]), sourceEpochTransition: {
schema: "paperclip.prp.event-epoch.v1", runId: "run", transitionId: randomUUID(), fromEpoch: epochs[2]!, nextEpoch: epochs[0]!, finalOrdinal: 4 } }] })).rejects.toThrow("reused");
expect(port.load()).toEqual(before);
for (const value of pending) await port.acknowledge(value);
expect(port.load()).toMatchObject({ producedEpoch: epochs[2], acknowledgedEpoch: epochs[2], produced: 4, acknowledged: 4, pending: [] });
await core.stop();
const get = store.getRecord.bind(store);
vi.spyOn(store, "getRecord").mockImplementation(async (epoch, kind, id) => id.startsWith("normalized-epoch-from-") ? null : get(epoch, kind, id));
await expect(DurablePrpControlPlane.open({ ...options, authorityStore: store })).rejects.toThrow("current recovery evidence");
} finally { await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); }
});
@@ -0,0 +1,45 @@
import { describe, expect, it } from "vitest";
import {
isProviderGenerationSuccessor,
isProviderProcessGeneration,
provesProviderGenerationTransition,
} from "./process-generation.js";
const maxLegacy = Number.MAX_SAFE_INTEGER;
const firstOpaque = "p:00000000-0000-4000-8000-000000000001";
const nextOpaque = "p:00000000-0000-4000-8000-000000000002";
const launchId = "00000000-0000-4000-8000-000000000010";
describe("provider process generation", () => {
it("switches from the exact legacy ceiling to distinct opaque lifetimes", () => {
expect(isProviderProcessGeneration(maxLegacy)).toBe(true);
expect(isProviderProcessGeneration(firstOpaque)).toBe(true);
expect(isProviderGenerationSuccessor(maxLegacy - 1, maxLegacy)).toBe(true);
expect(isProviderGenerationSuccessor(maxLegacy, firstOpaque)).toBe(true);
expect(isProviderGenerationSuccessor(firstOpaque, nextOpaque)).toBe(true);
expect(isProviderGenerationSuccessor(firstOpaque, firstOpaque)).toBe(false);
expect(isProviderGenerationSuccessor(firstOpaque, 1)).toBe(false);
expect(isProviderGenerationSuccessor(maxLegacy, 1)).toBe(false);
});
it("requires a canonical opaque UUID token and exact launch transition", () => {
for (const malformed of [
"p:00000000-0000-0000-8000-000000000001",
"p:00000000-0000-4000-7000-000000000001",
"p:00000000-0000-4000-8000-00000000000Z",
"p:00000000-0000-4000-8000-000000000001 ",
]) {
expect(isProviderProcessGeneration(malformed)).toBe(false);
expect(isProviderGenerationSuccessor(maxLegacy, malformed)).toBe(false);
}
const transition = { from: maxLegacy, to: firstOpaque, launchId };
expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, transition)).toBe(true);
expect(provesProviderGenerationTransition(maxLegacy - 1, firstOpaque, transition)).toBe(false);
expect(provesProviderGenerationTransition(maxLegacy, nextOpaque, transition)).toBe(false);
expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, { ...transition, launchId: "bad" })).toBe(false);
expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, { ...transition, extra: true })).toBe(false);
expect(provesProviderGenerationTransition(firstOpaque, 1, { from: firstOpaque, to: 1, launchId })).toBe(false);
});
});
@@ -0,0 +1,21 @@
/** Process lifetimes use equality; retained small integers preserve old bytes. */
export type ProviderProcessGeneration = number | string;
export function isProviderProcessGeneration(value: unknown, allowZero = false): value is ProviderProcessGeneration {
return (typeof value === "number" && Number.isSafeInteger(value) && value >= (allowZero ? 0 : 1))
|| (typeof value === "string" && /^p:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value));
}
export function isProviderGenerationSuccessor(previous: unknown, next: unknown): boolean {
if (!isProviderProcessGeneration(previous, true) || !isProviderProcessGeneration(next)) return false;
if (typeof previous === "number" && previous < Number.MAX_SAFE_INTEGER) return next === previous + 1;
return typeof next === "string" && next !== previous;
}
/** Numeric successors are retained compatibility. An opaque successor must
* carry the exact predecessor/startup binding committed by the native owner. */
export function provesProviderGenerationTransition(previous: unknown, next: unknown, value: unknown): boolean {
if (!isProviderGenerationSuccessor(previous, next)) return false;
if (typeof next === "number") return true;
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const transition = value as Record<string, unknown>;
return Object.keys(transition).length === 3 && transition.from === previous && transition.to === next
&& typeof transition.launchId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(transition.launchId);
}
@@ -0,0 +1,262 @@
import { describe, expect, it, vi } from "vitest";
import {
DurableAuthorityStoreError,
type DurableAuthorityStore,
} from "./durable-authority-store.js";
import { eventEpochCloseId } from "./event-epochs.js";
import type {
DurableRecoveryCommittedEvent,
DurableRecoveryIdentity,
} from "./prp-transport-types.js";
import {
processOwnerChanges,
processOwnerEvidence,
type ProcessOwnerEvidence,
} from "./process-owner-evidence.js";
const identity: DurableRecoveryIdentity = {
runnerInstanceId: "runner-owner-test",
environmentLeaseId: "lease-owner-test",
runId: "run-owner-test",
normalizedSessionId: "session-owner-test",
turnId: "turn-owner-test",
itemId: "item-owner-test",
};
const previousEpoch = "00000000-0000-4000-8000-000000000101";
const nextEpoch = "00000000-0000-4000-8000-000000000102";
const launchId = "00000000-0000-4000-8000-000000000103";
const opaqueGeneration = "p:00000000-0000-4000-8000-000000000104";
function event(options: {
phase?: "intent" | "spawned" | "initialization_failed";
sourceSeq?: number;
sourceEpoch?: string;
launch?: string;
previousGeneration?: number;
attemptedGeneration?: number | string;
owner?: DurableRecoveryIdentity;
} = {}): DurableRecoveryCommittedEvent {
const phase = options.phase ?? "intent";
const owner = options.owner ?? identity;
const startup = {
schema: "paperclip.provider_startup.v1",
launchId: options.launch ?? launchId,
phase,
attemptedProcessGeneration: options.attemptedGeneration ?? 1,
...(options.previousGeneration !== undefined
? { previousProcessGeneration: options.previousGeneration }
: {}),
configurationFingerprint: `sha256:${"a".repeat(64)}`,
origin: {
runnerInstanceId: owner.runnerInstanceId,
runId: owner.runId,
normalizedSessionId: owner.normalizedSessionId,
turnId: owner.turnId,
itemId: owner.itemId,
},
...(phase === "spawned" ? { processId: 1200, processGroupId: 1200 } : {}),
processTreeRetired: false,
};
return {
sourceSeq: options.sourceSeq ?? 1,
sourceEventId: `event-${options.sourceSeq ?? 1}`,
...(options.sourceEpoch ? { sourceEpoch: options.sourceEpoch } : {}),
eventType: "harness.diagnostic",
priority: 1,
envelope: {
payload: {
payload: { code: "provider_startup_ownership", startup },
},
},
deliveryCount: 1,
logicalEffectCount: 1,
};
}
function ownerFact(options: Parameters<typeof event>[0] = {}): ProcessOwnerEvidence {
return processOwnerEvidence(event(options), identity)!;
}
function authorityWithOwner(
prior: ProcessOwnerEvidence,
receiptBody: Record<string, unknown>,
) {
const getWork = vi.fn(async () => ({
collection: "process-owner" as const,
id: launchId,
body: prior as unknown as Record<string, unknown>,
sha256: "a".repeat(64),
}));
const getRecord = vi.fn(async () => ({
epoch: identity.runId,
kind: "effect" as const,
id: eventEpochCloseId({ runId: identity.runId, fromEpoch: previousEpoch }),
sequence: "0",
body: receiptBody,
}));
return {
authority: { getWork, getRecord } as unknown as DurableAuthorityStore,
getWork,
getRecord,
};
}
function validCloseReceipt(): Record<string, unknown> {
return {
schema: "paperclip.prp.event-epoch.v1",
runId: identity.runId,
transitionId: "00000000-0000-4000-8000-000000000105",
fromEpoch: previousEpoch,
nextEpoch,
finalOrdinal: 2,
};
}
describe("provider process owner evidence", () => {
it("accepts opaque generation startup only with its predecessor and compatible launch UUIDs", () => {
const opaque = processOwnerEvidence(
event({
attemptedGeneration: opaqueGeneration,
previousGeneration: Number.MAX_SAFE_INTEGER,
}),
identity,
);
expect(opaque?.startup).toMatchObject({
attemptedProcessGeneration: opaqueGeneration,
previousProcessGeneration: Number.MAX_SAFE_INTEGER,
});
// Retained startup launch IDs are UUID-shaped identities, not process
// generation tokens; accept a valid older UUID version as native parsing does.
expect(
processOwnerEvidence(
event({ launch: "00000000-0000-1000-8000-000000000103" }),
identity,
),
).not.toBeNull();
expect(() =>
processOwnerEvidence(
event({ attemptedGeneration: opaqueGeneration }),
identity,
),
).toThrow(DurableAuthorityStoreError);
expect(() =>
processOwnerEvidence(
event({ attemptedGeneration: "p:malformed", previousGeneration: 1 }),
identity,
),
).toThrow(DurableAuthorityStoreError);
});
it("rejects evidence whose startup origin belongs to a different task identity", () => {
for (const key of [
"runnerInstanceId",
"runId",
"normalizedSessionId",
"turnId",
"itemId",
] as const) {
const foreign = { ...identity, [key]: `other-${identity[key]}` };
expect(() => processOwnerEvidence(event({ owner: foreign }), identity)).toThrow(
DurableAuthorityStoreError,
);
}
});
it("rejects advancement that changes the launch or its recorded predecessor", async () => {
const prior = ownerFact({
phase: "intent",
attemptedGeneration: opaqueGeneration,
previousGeneration: Number.MAX_SAFE_INTEGER,
});
const differentLaunch = ownerFact({
phase: "spawned",
attemptedGeneration: opaqueGeneration,
previousGeneration: Number.MAX_SAFE_INTEGER,
launch: "00000000-0000-4000-8000-000000000106",
});
const validSuccessor = ownerFact({
phase: "spawned",
attemptedGeneration: opaqueGeneration,
previousGeneration: Number.MAX_SAFE_INTEGER,
});
const differentPredecessor: ProcessOwnerEvidence = {
...validSuccessor,
startup: {
...validSuccessor.startup,
previousProcessGeneration: Number.MAX_SAFE_INTEGER - 1,
},
};
const { authority } = authorityWithOwner(prior, validCloseReceipt());
await expect(
processOwnerChanges(authority, [differentLaunch], {
runId: identity.runId,
}),
).rejects.toMatchObject({ code: "invalid_authority" });
await expect(
processOwnerChanges(authority, [differentPredecessor], {
runId: identity.runId,
}),
).rejects.toMatchObject({ code: "invalid_authority" });
});
it("advances maintenance ownership only through the current run's exact raw epoch close", async () => {
const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch });
const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch });
const { authority, getRecord } = authorityWithOwner(prior, validCloseReceipt());
const changes = await processOwnerChanges(authority, [next], {
runId: identity.runId,
sourceEpoch: nextEpoch,
});
expect(changes).toHaveLength(1);
expect(changes[0]).toMatchObject({
collection: "process-owner",
id: launchId,
expectedSha256: "a".repeat(64),
body: next,
});
expect(getRecord).toHaveBeenCalledWith(
identity.runId,
"effect",
eventEpochCloseId({ runId: identity.runId, fromEpoch: previousEpoch }),
);
});
it("rejects a rollover backed by a close receipt from another raw run", async () => {
const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch });
const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch });
const receipt = { ...validCloseReceipt(), runId: "other-run" };
const { authority } = authorityWithOwner(prior, receipt);
await expect(
processOwnerChanges(authority, [next], {
runId: identity.runId,
sourceEpoch: nextEpoch,
}),
).rejects.toMatchObject({ code: "invalid_authority" });
});
it("rejects maintenance facts outside the current raw run and epoch", async () => {
const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch });
const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch });
const { authority, getRecord } = authorityWithOwner(prior, validCloseReceipt());
await expect(
processOwnerChanges(authority, [next], {
runId: "another-run",
sourceEpoch: nextEpoch,
}),
).rejects.toMatchObject({ code: "invalid_authority" });
await expect(
processOwnerChanges(authority, [next], {
runId: identity.runId,
sourceEpoch: previousEpoch,
}),
).rejects.toMatchObject({ code: "invalid_authority" });
expect(getRecord).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,89 @@
import { isProviderProcessGeneration, isProviderGenerationSuccessor } from "./process-generation.js";
import { eventEpochCloseId, isEventEpochTransition } from "./event-epochs.js";
import { authorityJson, DurableAuthorityStoreError, type AuthorityWorkChange, type DurableAuthorityStore } from "./durable-authority-store.js";
import type { DurableRecoveryCommittedEvent, DurableRecoveryIdentity } from "./prp-transport-types.js";
export interface ProcessOwnerEvidence {
schema: "paperclip.process-owner.v1";
identity: DurableRecoveryIdentity;
sourceEventId: string;
sourceSeq: number;
sourceEpoch?: string;
startup: Record<string, unknown>;
}
const object = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value);
function invalid(): never { throw new DurableAuthorityStoreError("invalid_authority", "provider process ownership evidence is invalid or conflicts with its launch"); }
/** A diagnostic is authoritative only after authenticated PRP validation. The
* relation retains unresolved launch facts, independently of transcript windows.
* Direct-child exit is deliberately not accepted as process-tree retirement. */
export function processOwnerEvidence(event: DurableRecoveryCommittedEvent, identity: DurableRecoveryIdentity): ProcessOwnerEvidence | null {
const wire = event.envelope.payload;
const body = object(wire) && object(wire.payload) ? wire.payload : {};
if (event.eventType !== "harness.diagnostic" || body.code !== "provider_startup_ownership") return null;
const startup = body.startup;
if (!object(startup) || startup.schema !== "paperclip.provider_startup.v1" ||
typeof startup.launchId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(startup.launchId) ||
!["intent", "spawned", "initialization_failed"].includes(String(startup.phase)) ||
!isProviderProcessGeneration(startup.attemptedProcessGeneration) ||
(typeof startup.attemptedProcessGeneration === "string" && !isProviderGenerationSuccessor(startup.previousProcessGeneration, startup.attemptedProcessGeneration)) ||
typeof startup.configurationFingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(startup.configurationFingerprint) ||
startup.processTreeRetired !== false || !object(startup.origin) || ["runnerInstanceId", "runId", "normalizedSessionId", "turnId", "itemId"].some(key => (startup.origin as Record<string, unknown>)[key] !== identity[key as keyof DurableRecoveryIdentity])) invalid();
const hasProcess = Number.isSafeInteger(startup.processId) && Number(startup.processId) > 0 && startup.processId === startup.processGroupId;
if (startup.phase === "intent" ? startup.processId != null || startup.processGroupId != null :
startup.failedStage === "spawn" ? startup.processId != null || startup.processGroupId != null : !hasProcess) invalid();
return { schema: "paperclip.process-owner.v1", identity: structuredClone(identity), sourceEventId: event.sourceEventId, sourceSeq: event.sourceSeq, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), startup: structuredClone(startup) };
}
/** At most one page of new facts; repeated phases for a launch collapse in the
* transaction, while every original event is still an immutable receipt. */
export async function processOwnerChanges(authority: DurableAuthorityStore, facts: readonly ProcessOwnerEvidence[], current: { runId: string; sourceEpoch?: string }): Promise<AuthorityWorkChange[]> {
// Collapse the source page before consulting the index. On interrupted
// backfill the index may already contain a later phase from this same page;
// comparing its first intent against that phase would reject a safe replay.
const latest = new Map<string, ProcessOwnerEvidence>();
for (const fact of facts) {
const id = String(fact.startup.launchId);
const prior = latest.get(id);
if (prior) validateAdvance(prior, fact);
latest.set(id, fact);
}
const changes = new Map<string, AuthorityWorkChange>();
for (const fact of latest.values()) {
const id = String(fact.startup.launchId);
const pending = changes.get(id);
const stored = pending ? null : await authority.getWork("process-owner", id);
const prior = (pending?.body ?? stored?.body) as unknown as ProcessOwnerEvidence | undefined;
if (prior) {
validateAdvance(prior, fact);
if (prior.sourceEpoch !== fact.sourceEpoch) {
if (fact.identity.runId !== current.runId || fact.sourceEpoch !== current.sourceEpoch) invalid();
const receipt = await authority.getRecord(prior.identity.runId, "effect", eventEpochCloseId({ runId: prior.identity.runId, fromEpoch: prior.sourceEpoch ?? null }));
if (!receipt || !isEventEpochTransition(receipt.body) || receipt.body.runId !== prior.identity.runId || receipt.body.fromEpoch !== (prior.sourceEpoch ?? null)
|| receipt.body.finalOrdinal < prior.sourceSeq) invalid();
}
if (authorityJson(prior) === authorityJson(fact)) continue;
}
// An authenticated failed spawn proves that this launch created no child.
// Remove its pending intent in the same transaction as the immutable event
// receipt. A child exit after spawn does not prove a stopped process tree.
if (fact.startup.phase === "initialization_failed" && fact.startup.failedStage === "spawn") {
if (stored) changes.set(id, { collection: "process-owner", id, expectedSha256: stored.sha256, body: null });
continue;
}
changes.set(id, { collection: "process-owner", id, expectedSha256: pending?.expectedSha256 ?? stored?.sha256 ?? null, body: fact as unknown as Record<string, unknown> });
}
return [...changes.values()];
}
function validateAdvance(prior: ProcessOwnerEvidence, fact: ProcessOwnerEvidence): void {
if (prior.schema !== fact.schema || authorityJson(prior.identity) !== authorityJson(fact.identity) ||
prior.startup.launchId !== fact.startup.launchId ||
prior.startup.previousProcessGeneration !== fact.startup.previousProcessGeneration ||
prior.startup.configurationFingerprint !== fact.startup.configurationFingerprint ||
prior.startup.attemptedProcessGeneration !== fact.startup.attemptedProcessGeneration ||
(prior.startup.processId != null && prior.startup.processId !== fact.startup.processId) ||
(prior.sourceEpoch === fact.sourceEpoch && (prior.sourceSeq > fact.sourceSeq || (prior.sourceSeq === fact.sourceSeq && authorityJson(prior) !== authorityJson(fact))))) invalid();
const phase = ["intent", "spawned", "initialization_failed"];
if (phase.indexOf(String(prior.startup.phase)) > phase.indexOf(String(fact.startup.phase))) invalid();
}
@@ -18,6 +18,7 @@ export interface DurableWarmRunTransition {
commandFingerprint: string;
resultDigest: string;
oldAckedSourceSeq: number;
oldSourceEpoch?: string;
connection: Record<string, unknown>;
runnerVersion: string;
runnerDigest: string;
@@ -30,6 +31,7 @@ export interface DurableRecoveryCoreCommand {
schema: "paperclip.prp.command.v1" | "paperclip.prp.command.v2";
commandId: string;
controllerSeq: number;
controllerEpoch?: string;
type: string;
issuedAt: string;
payload: Record<string, unknown>;
@@ -44,6 +46,7 @@ export interface DurableRecoveryCoreCommand {
}
export interface DurableRecoveryCommittedEvent {
sourceEpoch?: string;
sourceSeq: number;
sourceEventId: string;
eventType: string;
@@ -0,0 +1,234 @@
import { mkdtemp, rm, chmod } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { execFile } from "node:child_process";
import { randomUUID, createHash } from "node:crypto";
import { afterEach, expect, it } from "vitest";
import { promisify } from "node:util";
import { SqliteAuthorityStore } from "./sqlite-authority-store.js";
import { authorityInteger, type AuthorityRecord } from "./durable-authority-store.js";
import { defaultStorageRunnerBinary } from "./native-storage-command.js";
const execute = promisify(execFile);
const stores: SqliteAuthorityStore[] = [];
const roots: string[] = [];
afterEach(async () => {
for (const store of stores.splice(0)) await store.close();
for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true });
});
async function fixture() {
const root = await mkdtemp(join(tmpdir(), "runner-authority-"));
roots.push(root);
await chmod(root, 0o700);
const input = { path: join(root, "authority.sqlite"), binding: "company:session:runner:environment", create: true };
const store = await SqliteAuthorityStore.open(input);
stores.push(store);
return { store, input };
}
const record = (id: string, sequence: string, text = id): AuthorityRecord => ({ epoch: "run-1", kind: "event", id, sequence, body: { text } });
it("retains unordered effect identities without consuming a lifetime sequence", async () => {
const { store, input } = await fixture();
expect(store.unorderedEffectReceipts).toBe(true);
const old = { ...record("old-effect", "9223372036854775807"), kind: "effect" as const };
const first = { ...record("new-effect-1", "0"), kind: "effect" as const };
const second = { ...record("new-effect-2", "0"), kind: "effect" as const };
const generation = await store.commit({ expectedGeneration: "0", state: { settled: true }, records: [old, first, second] });
await store.close();
const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened);
for (const receipt of [old, first, second]) {
expect(await reopened.getSessionEffect(receipt.id)).toEqual(receipt);
expect(await reopened.getRecord("run-1", "effect", receipt.id)).toEqual(receipt);
}
for (const conflicting of [{ ...first, body: { text: "changed" } }, { ...old, sequence: "0" }, { ...first, epoch: "run-2" }]) {
await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [conflicting] })).rejects.toThrow("receipt_conflict");
}
expect(await reopened.load()).toEqual({ generation, state: { settled: true } });
});
it("commits current authority with exact receipts, and rejects stale writers", async () => {
const { store, input } = await fixture();
expect(await store.load()).toBeNull();
await store.commit({ expectedGeneration: "0", state: { cursor: "1" }, records: [record("first", "1")] });
await expect(store.commit({ expectedGeneration: "1", state: { cursor: "2" }, records: [record("second", "2"), record("first", "1", "conflicting")] })).rejects.toThrow("receipt_conflict");
await expect(store.commit({ expectedGeneration: "1", state: { cursor: "2" }, records: [record("alias", "1")] })).rejects.toThrow("receipt_conflict");
expect(await store.getRecord("run-1", "event", "second")).toBeNull();
expect(await store.load()).toEqual({ generation: "1", state: { cursor: "1" } });
await expect(store.commit({ expectedGeneration: "0", state: {}, records: [] })).rejects.toThrow("stale_authority");
await store.close();
const reopened = await SqliteAuthorityStore.open({ ...input, create: false });
stores.push(reopened);
expect(await reopened.getRecord("run-1", "event", "first")).toEqual(record("first", "1"));
expect(await reopened.getRecord("run-2", "event", "first")).toBeNull();
const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true });
stores.push(proof);
expect(await proof.load()).toEqual(await reopened.load());
await expect(proof.commit({ expectedGeneration: "1", state: {}, records: [] })).rejects.toThrow("proof readers cannot commit");
await expect(SqliteAuthorityStore.open({ ...input, binding: "wrong-owner", create: false })).rejects.toThrow("binding");
});
it("keeps exact integers above Number.MAX_SAFE_INTEGER and pages by bytes", async () => {
const { store } = await fixture();
const first = "9007199254740993", next = "9007199254740994";
const payload = "x".repeat(600_000);
await store.commit({ expectedGeneration: "0", state: { cursor: next }, records: [record("first", first, payload), record("next", next, payload)] });
const page = await store.readEvents("run-1", "0", 128, 1024 * 1024);
expect(page.records.map((entry) => entry.sequence)).toEqual([first]);
expect((await store.readEvents("run-1", page.nextAfter!, 128, 1024 * 1024)).records.map((entry) => entry.sequence)).toEqual([next]);
expect(authorityInteger(first)).toBe(9007199254740993n);
expect(() => authorityInteger("9007199254740993.0")).toThrow();
});
it("continues past the native revision boundary with exact proof and outstanding-work reads", async () => {
const { store, input } = await fixture();
await store.commit({ expectedGeneration: "0", state: { cursor: "1" }, records: [record("first", "1")], work: [
{ collection: "process-owner", id: "owner", expectedSha256: null, body: { phase: "spawned" } },
] });
await store.close();
const db = new DatabaseSync(input.path);
try { db.prepare("UPDATE current_state SET generation=? WHERE key='authority'").run(9223372036854775807n); }
finally { db.close(); }
const active = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(active);
const first = await active.commit({ expectedGeneration: "9223372036854775807", state: { cursor: "2" }, records: [record("second", "2")] });
expect(first).toMatch(/^r:/);
const second = await active.commit({ expectedGeneration: first, state: { cursor: "3" }, records: [] });
expect(second).not.toBe(first);
await expect(active.commit({ expectedGeneration: first, state: { stale: true }, records: [] })).rejects.toThrow("stale_authority");
const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(proof);
expect(await proof.load()).toEqual({ generation: second, state: { cursor: "3" } });
expect(await proof.load()).toEqual(await active.load());
expect(await proof.getRecord("run-1", "event", "first")).toEqual(record("first", "1"));
expect((await proof.readWorkPage("process-owner", "", 128, second)).records.map(row => row.id)).toEqual(["owner"]);
await expect(proof.readWorkPage("process-owner", "", 128, first)).rejects.toThrow("stale_authority");
expect((await active.readEvents("run-1", "0", 128, 1024 * 1024)).records).toEqual([record("first", "1"), record("second", "2")]);
});
it("does not recreate a missing activated store", async () => {
const { store, input } = await fixture();
await store.close();
await rm(input.path);
await expect(SqliteAuthorityStore.open({ ...input, create: false })).rejects.toThrow("storage_unavailable");
});
it("partitions standalone history and preserves exact event/effect indexes after reopening", async () => {
const { store, input } = await fixture();
await store.close();
const setup = new DatabaseSync(input.path);
try {
expect(setup.prepare("SELECT schema_version FROM store_binding").get()?.schema_version).toBe(2);
setup.exec("UPDATE receipt_storage_config SET partition_bytes=32768");
} finally { setup.close(); }
const active = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(active);
let generation = "0";
const text = "history ".repeat(1024);
for (let index = 1; index <= 64; index += 1) {
generation = await active.commit({ expectedGeneration: generation, state: { latest: index }, records: [record(`event-${index}`, String(index), text),
{ epoch: "run-1", kind: "effect", id: `effect-${index}`, sequence: String(index), body: { accepted: index } }] });
}
await active.close();
const { stdout } = await execute(defaultStorageRunnerBinary(), [
"storage",
"partitions",
"--path",
input.path,
"--binding",
input.binding,
"--limit",
"32",
], { env: {}, maxBuffer: 64 * 1024, timeout: 30_000, windowsHide: true });
const partitionPage = JSON.parse(stdout) as {
schema?: string;
partitions?: unknown[];
};
expect(partitionPage.schema).toBe("paperclip.receipt-partitions.v1");
expect(partitionPage.partitions?.length ?? 0).toBeGreaterThan(1);
const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened);
expect(await reopened.load()).toEqual({ generation: "64", state: { latest: 64 } });
expect(await reopened.getSessionEffect("effect-1")).toMatchObject({ epoch: "run-1", body: { accepted: 1 } });
expect(await reopened.getRecord("run-1", "event", "event-1")).toEqual(record("event-1", "1", text));
const page = await reopened.readEvents("run-1", "30", 8, 1024 * 1024);
expect(page.records.map(record => record.sequence)).toEqual(["31", "32", "33", "34", "35", "36", "37", "38"]);
await expect(reopened.commit({ expectedGeneration: "64", state: { wrong: true }, records: [{ epoch: "run-2", kind: "effect", id: "effect-1", sequence: "1", body: { accepted: 1 } }] })).rejects.toThrow("receipt_conflict");
expect((await reopened.load())?.generation).toBe("64");
}, 60_000);
it("preserves retained version-1 controller files without silently replacing their authority", async () => {
const root = await mkdtemp(join(tmpdir(), "legacy-sqlite-authority-")); roots.push(root);
const path = join(root, "authority.sqlite"), binding = "legacy-retained", body = '{"retained":true}';
const db = new DatabaseSync(path);
await chmod(path, 0o600);
try {
db.exec("CREATE TABLE authority_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE authority_state(singleton INTEGER PRIMARY KEY,generation INTEGER,body TEXT,digest TEXT); CREATE TABLE authority_records(epoch TEXT,kind TEXT,id TEXT,sequence INTEGER,body TEXT,digest TEXT,PRIMARY KEY(epoch,kind,id),UNIQUE(epoch,kind,sequence)); CREATE UNIQUE INDEX authority_session_effect ON authority_records(id) WHERE kind='effect'");
db.prepare("INSERT INTO authority_binding VALUES (1,1,?)").run(binding);
db.prepare("INSERT INTO authority_state VALUES (1,7,?,?)").run(body, createHash("sha256").update(body).digest("hex"));
} finally { db.close(); }
const store = await SqliteAuthorityStore.open({ path, binding, create: false }); stores.push(store);
expect(store.unorderedEffectReceipts).toBe(false);
await expect(store.commit({ expectedGeneration: "7", state: {}, records: [{ ...record("effect", "0"), kind: "effect" }] })).rejects.toThrow("retained v1 store requires ordered effect receipts");
expect(await store.load()).toEqual({ generation: "7", state: { retained: true } });
await store.commit({ expectedGeneration: "7", state: { continued: true }, records: [record("legacy", "1")] });
expect(await store.getRecord("run-1", "event", "legacy")).toEqual(record("legacy", "1"));
await store.close();
const check = new DatabaseSync(path, { readOnly: true });
try { expect(check.prepare("SELECT schema_version FROM authority_binding").get()?.schema_version).toBe(1); }
finally { check.close(); }
});
it("does not use an unfinished native backup as a current-state proof", async () => {
const { store, input } = await fixture();
await store.commit({ expectedGeneration: "0", state: { authority: true }, records: [] });
await store.close();
const db = new DatabaseSync(input.path);
try { db.exec("CREATE TABLE backup_manifest(singleton INTEGER PRIMARY KEY,id TEXT,complete INTEGER); INSERT INTO backup_manifest VALUES(1,'unfinished',0)"); }
finally { db.close(); }
const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(proof);
await expect(proof.load()).rejects.toThrow("backup is incomplete");
});
it("pages outstanding owners at an exact generation and retires only the exact row", async () => {
const { store } = await fixture();
const start = { collection: "process-owner" as const, id: "launch-1", expectedSha256: null, body: { phase: "spawned", processId: 9001 } };
await store.commit({ expectedGeneration: "0", state: { owners: 1 }, records: [record("intent", "1")], work: [start] });
const owner = (await store.getWork("process-owner", "launch-1"))!;
expect((await store.readWorkPage("process-owner", "", 1, "1")).records).toEqual([owner]);
await expect(store.commit({ expectedGeneration: "1", state: { owners: 0 }, records: [record("wrong", "2")], work: [{ ...start, expectedSha256: "0".repeat(64), body: null }] })).rejects.toThrow("stale_authority");
expect(await store.getRecord("run-1", "event", "wrong")).toBeNull();
expect(await store.getWork("process-owner", "launch-1")).toEqual(owner);
await store.commit({ expectedGeneration: "1", state: { owners: 0 }, records: [record("retirement-proof", "2")], work: [{ ...start, expectedSha256: owner.sha256, body: null }] });
expect(await store.getWork("process-owner", "launch-1")).toBeNull();
await expect(store.readWorkPage("process-owner", "", 1, "1")).rejects.toThrow("stale_authority");
expect((await store.readWorkPage("process-owner", "", 128, "2")).records).toEqual([]);
expect(await store.getRecord("run-1", "event", "intent")).toEqual(record("intent", "1"));
});
it("read-only owner pages validate authority and reject a changed generation or corrupt root", async () => {
const { store, input } = await fixture();
await store.commit({ expectedGeneration: "0", state: { owners: 1 }, records: [], work: [{ collection: "process-owner", id: "owner-1", expectedSha256: null, body: { phase: "spawned" } }] });
const reader = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(reader);
expect((await reader.readWorkPage("process-owner", "", 128, "1")).records).toHaveLength(1);
await expect(reader.readWorkPage("process-owner", "", 128, "2")).rejects.toThrow("stale_authority");
await reader.close(); await store.close();
const db = new DatabaseSync(input.path);
try { db.exec("UPDATE current_state SET digest=zeroblob(32) WHERE key='authority'"); } finally { db.close(); }
const damaged = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(damaged);
await expect(damaged.readWorkPage("process-owner", "", 128, "1")).rejects.toThrow("digest mismatch");
});
it("separates renewable command ordering from immutable command identity", async () => {
const { store, input } = await fixture();
expect(store.commandEpochs).toBe(true);
const receipts: AuthorityRecord[] = [undefined, randomUUID(), randomUUID()].map((sequenceEpoch, index) => ({
...record(`command-${index}`, "1"), kind: "command", ...(sequenceEpoch ? { sequenceEpoch } : {}),
}));
const generation = await store.commit({ expectedGeneration: "0", state: { current: receipts[2]!.sequenceEpoch }, records: receipts });
await store.close();
const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened);
for (const receipt of receipts) expect(await reopened.getRecord(receipt.epoch, "command", receipt.id)).toEqual(receipt);
for (const record of [{ ...receipts[1]!, id: "alias" }, { ...receipts[0]!, sequenceEpoch: randomUUID() }]) {
await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [record] })).rejects.toThrow("receipt_conflict");
}
expect((await reopened.load())?.generation).toBe(generation);
});
@@ -0,0 +1,127 @@
import { basename } from "node:path";
import { Worker } from "node:worker_threads";
import {
DurableAuthorityStoreError,
validateAuthorityCommit,
validateAuthorityPage,
validateAuthorityWorkKey,
validateAuthorityWorkPage,
type AuthorityWorkRecord,
type AuthorityWorkPage,
type AuthorityCommit,
type AuthorityPage,
type AuthorityRecord,
type AuthoritySnapshot,
type DurableAuthorityStore,
} from "./durable-authority-store.js";
/** Standalone/eval implementation. Production Paperclip uses its Postgres
* transaction so event acceptance and current authority share one commit. */
export class SqliteAuthorityStore implements DurableAuthorityStore {
readonly location: import("./authority-locator.js").AuthorityLocation;
readonly #worker: Worker;
readonly #pending = new Map<number, { resolve(value: unknown): void; reject(error: Error): void }>();
#nextRequest = 0;
#failure: Error | null = null;
#closed = false;
#unorderedEffectReceipts = false;
#eventEpochs = false;
get eventEpochs(): boolean { return this.#eventEpochs; }
#commandEpochs = false;
get commandEpochs(): boolean { return this.#commandEpochs; }
get unorderedEffectReceipts(): boolean { return this.#unorderedEffectReceipts; }
private constructor(readonly binding: string, path: string, create: boolean, readOnly = false, runnerBinary?: string) {
this.location = { kind: "sqlite", binding, file: basename(path) };
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
this.#worker = new Worker(new URL(`./sqlite-authority-worker.${extension}`, import.meta.url), {
workerData: { binding, path, create, readOnly, runnerBinary },
// The worker is a plain Node module, never a Vitest/tsx loader copy.
execArgv: [],
});
this.#worker.on("message", (message: { id: number; value?: unknown; error?: { code: DurableAuthorityStoreError["code"]; message: string } }) => {
const pending = this.#pending.get(message.id);
if (!pending) return;
this.#pending.delete(message.id);
if (message.error) pending.reject(new DurableAuthorityStoreError(message.error.code, message.error.message));
else pending.resolve(message.value);
});
this.#worker.on("error", (error) => this.#fail(error));
this.#worker.on("exit", (code) => {
if (!this.#closed || this.#pending.size) this.#fail(new DurableAuthorityStoreError("storage_unavailable", `storage worker exited (${code})`));
});
}
static async open(input: { binding: string; path: string; create: boolean; readOnly?: boolean; runnerBinary?: string }): Promise<SqliteAuthorityStore> {
const store = new SqliteAuthorityStore(input.binding, input.path, input.create, input.readOnly, input.runnerBinary);
try {
const ready = await store.#request<{ unorderedEffectReceipts?: boolean; commandEpochs?: boolean; eventEpochs?: boolean } | null>("ready", {});
store.#unorderedEffectReceipts = ready?.unorderedEffectReceipts === true;
store.#commandEpochs = ready?.commandEpochs === true;
store.#eventEpochs = ready?.eventEpochs === true;
return store;
}
catch (error) { await store.#worker.terminate(); throw error; }
}
#fail(error: Error): void {
this.#failure ??= error;
for (const pending of this.#pending.values()) pending.reject(this.#failure);
this.#pending.clear();
}
#request<T>(operation: string, input: unknown): Promise<T> {
if (this.#failure) return Promise.reject(this.#failure);
if (this.#closed) return Promise.reject(new DurableAuthorityStoreError("storage_unavailable", "store is closed"));
if (this.#pending.size >= 8) return Promise.reject(new DurableAuthorityStoreError("storage_pressure", "storage executor queue is full"));
// RPC correlation is local to the current bounded in-flight set, not a
// durable history sequence. Reuse settled IDs instead of overflowing a
// lifetime JavaScript counter in a continuously running storage worker.
do { this.#nextRequest = this.#nextRequest >= 0xffff_ffff ? 1 : this.#nextRequest + 1; }
while (this.#pending.has(this.#nextRequest));
const id = this.#nextRequest;
return new Promise<T>((resolve, reject) => {
this.#pending.set(id, { resolve: (value) => resolve(value as T), reject });
try { this.#worker.postMessage({ id, operation, input }); }
catch (error) { this.#pending.delete(id); reject(error); }
});
}
load(): Promise<AuthoritySnapshot | null> { return this.#request("load", {}); }
commit(input: AuthorityCommit): Promise<string> {
validateAuthorityCommit(input);
return this.#request("commit", input);
}
getRecord(epoch: string, kind: AuthorityRecord["kind"], id: string): Promise<AuthorityRecord | null> {
return this.#request("getRecord", { epoch, kind, id });
}
getSessionEffect(id: string): Promise<AuthorityRecord | null> {
return this.#request("getSessionEffect", { id });
}
getWork(collection: AuthorityWorkRecord["collection"], id: string): Promise<AuthorityWorkRecord | null> {
validateAuthorityWorkKey(collection, id);
return this.#request("getWork", { collection, id });
}
readWorkPage(collection: AuthorityWorkRecord["collection"], after: string, limit: number, expectedGeneration: string): Promise<AuthorityWorkPage> {
validateAuthorityWorkPage(collection, after, limit, expectedGeneration);
return this.#request("readWorkPage", { collection, after, limit, expectedGeneration });
}
readEvents(epoch: string, after: string, limit: number, byteBudget: number, sequenceEpoch?: string): Promise<AuthorityPage> {
validateAuthorityPage(after, limit, byteBudget, sequenceEpoch);
if (sequenceEpoch !== undefined && !this.eventEpochs) return Promise.reject(new DurableAuthorityStoreError("invalid_authority", "event epochs require indexed storage"));
return this.#request("readEvents", { epoch, after, limit, byteBudget, sequenceEpoch });
}
async close(): Promise<void> {
if (this.#closed) return;
try { await this.#request("close", {}); }
finally { this.#closed = true; await this.#worker.terminate(); }
}
}
@@ -0,0 +1,228 @@
import { parentPort, workerData } from "node:worker_threads";
import { spawn, execFile } from "node:child_process";
import { promisify } from "node:util";
import { DatabaseSync } from "node:sqlite";
import { createHash } from "node:crypto";
import { lstatSync, mkdirSync, realpathSync } from "node:fs";
import { basename, dirname, resolve } from "node:path";
import type { AuthorityCommit, AuthorityRecord, AuthoritySnapshot, AuthorityWorkRecord } from "./durable-authority-store.js";
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
const { authorityInteger, authorityJson, validateAuthorityCommit, validateAuthorityPage, validateAuthorityWorkKey, validateAuthorityWorkPage, DurableAuthorityStoreError } =
await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js");
const { defaultStorageRunnerBinary } = await import(new URL(`./native-storage-command.${extension}`, import.meta.url).href) as typeof import("./native-storage-command.js");
const { path, binding, create, readOnly, runnerBinary } = workerData as { path: string; binding: string; create: boolean; readOnly?: boolean; runnerBinary?: string };
type Code = InstanceType<typeof DurableAuthorityStoreError>["code"];
function fail(code: Code, message: string): never { throw new DurableAuthorityStoreError(code, message); }
function privateFile(path: string): void {
// Workers share a POSIX process: opening and closing a probe descriptor can
// cancel SQLite's locks held by a different worker. SQLite owns all database
// and sidecar descriptors; the private-directory preflight is metadata only.
const stat = lstatSync(path);
if (!stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) fail("invalid_authority", "unsafe storage file");
}
let database: DatabaseSync | undefined;
let legacy = false;
let startupError: unknown;
try {
if (readOnly && create) fail("invalid_authority", "a proof reader cannot create authority");
if (typeof binding !== "string" || !binding.length || binding.length > 1024 || /[\x00-\x1f]/.test(binding)) fail("invalid_authority", "invalid storage binding");
if (create) mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
const directory = lstatSync(dirname(path));
if (!directory.isDirectory() || directory.isSymbolicLink() || (process.platform !== "win32" && ((directory.mode & 0o077) !== 0 || directory.uid !== process.getuid?.()))) fail("invalid_authority", "unsafe storage directory");
const exists = lstatSync(path, { throwIfNoEntry: false });
if (!exists && !create) fail("storage_unavailable", "activated store is missing");
if (exists) {
for (const suffix of ["", "-wal", "-shm"]) if (!suffix || lstatSync(path + suffix, { throwIfNoEntry: false })) privateFile(path + suffix);
database = new DatabaseSync(resolve(realpathSync(dirname(path)), basename(path)), { readOnly: true });
database.exec("PRAGMA trusted_schema=OFF; PRAGMA query_only=ON; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;");
legacy = !!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_binding'").get();
if (!legacy) {
const stored = database.prepare("SELECT schema_version,binding FROM store_binding WHERE singleton=1").get();
if (stored?.schema_version !== 2 || stored.binding !== binding) fail("invalid_authority", "storage binding or version mismatch");
}
if (legacy || !readOnly) { database.close(); database = undefined; }
}
} catch (error) { database?.close(); database = undefined; startupError = error; }
if (legacy && !startupError) {
// Retained v1 files keep their original format; only fresh stores select v2.
await import(new URL(`./legacy-sqlite-authority-worker.${extension}`, import.meta.url).href);
} else {
type Reply = { id: number; value?: unknown; error?: { code: Code; message: string } };
const pending = new Map<number, { resolve(value: unknown): void; reject(error: Error): void }>();
let requestId = 0, closing = false, stderr = "";
let failed: Error | undefined;
let output = Buffer.alloc(0);
const child = !startupError && !readOnly ? spawn(runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "rpc", "--path", path, "--binding", binding, "--create", String(create)], { env: {}, stdio: ["pipe", "pipe", "pipe"], windowsHide: true }) : undefined;
function fence(error: Error) {
failed ??= error;
for (const request of pending.values()) request.reject(failed);
pending.clear();
}
child?.stdout.on("data", (chunk: Buffer) => {
if (output.length + chunk.length > 96 * 1024 * 1024) { fence(new DurableAuthorityStoreError("storage_pressure", "storage response exceeds capacity")); child.kill(); return; }
output = Buffer.concat([output, chunk]);
for (;;) {
const end = output.indexOf(10);
if (end < 0) break;
const line = output.subarray(0, end); output = output.subarray(end + 1);
try {
const message = JSON.parse(line.toString()) as Reply;
const request = pending.get(message.id);
if (!request) fail("invalid_authority", "unexpected storage response");
pending.delete(message.id);
if (message.error) request.reject(new DurableAuthorityStoreError(message.error.code, message.error.message));
else request.resolve(message.value);
} catch (error) { fence(error as Error); child.kill(); }
}
});
child?.stderr.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-8192); });
child?.once("error", error => fence(new DurableAuthorityStoreError("storage_unavailable", error.message)));
child?.once("exit", code => { if (!closing || code !== 0 || pending.size) fence(new DurableAuthorityStoreError("storage_unavailable", stderr.trim() || `storage process exited (${code})`)); });
child?.stdin.on("error", error => fence(new DurableAuthorityStoreError("storage_unavailable", error.message)));
parentPort!.once("close", () => { child?.stdin.end(); });
function rpc<T>(operation: string, input: unknown): Promise<T> {
if (failed) return Promise.reject(failed);
if (!child) return Promise.reject(new DurableAuthorityStoreError("storage_unavailable", "storage process is absent"));
do { requestId = requestId >= 0xffff_ffff ? 1 : requestId + 1; }
while (pending.has(requestId));
const id = requestId;
return new Promise<T>((resolve, reject) => {
pending.set(id, { resolve: value => resolve(value as T), reject });
child.stdin.write(`${JSON.stringify({ id, operation, input })}\n`);
});
}
const namespace = (epoch: string, kind: AuthorityRecord["kind"]) => `authority/${kind}/${epoch}`;
const sequenceNamespace = (epoch: string, kind: AuthorityRecord["kind"]) => `sequence/${kind}/${epoch}`;
const encoded = (value: unknown) => Buffer.from(authorityJson(value)).toString("base64");
const decode = <T>(bytes: string): T => JSON.parse(Buffer.from(bytes, "base64").toString()) as T;
async function get(namespace: string, key: string): Promise<AuthorityRecord | null> {
const bytes = await rpc<string | null>("get", { namespace, key });
return bytes === null ? null : decode<AuthorityRecord>(bytes);
}
type NativeWork = { key: string; bytes: string; digest: string };
const decodeWork = (collection: AuthorityWorkRecord["collection"], row: NativeWork): AuthorityWorkRecord => ({ collection, id: row.key, body: decode(row.bytes), sha256: Buffer.from(row.digest, "base64").toString("hex") });
async function dispatch(operation: string, input: Record<string, unknown>): Promise<unknown> {
if (startupError) throw startupError;
if (readOnly) {
if (operation === "ready") return { unorderedEffectReceipts: true, commandEpochs: true, eventEpochs: true };
if (operation === "close") { database!.close(); return null; }
if (operation === "commit") fail("invalid_authority", "proof readers cannot commit authority");
if (!["load", "getWork", "readWorkPage", "getRecord"].includes(operation)) fail("invalid_authority", "proof readers expose current authority and exact references only");
database!.exec("BEGIN");
try {
if (database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_manifest'").get()
&& database!.prepare("SELECT complete FROM backup_manifest WHERE singleton=1").get()?.complete !== 1) fail("invalid_authority", "indexed backup is incomplete");
const statement = database!.prepare("SELECT generation,CASE WHEN length(bytes)<=16777216 THEN bytes END AS bytes,CASE WHEN length(digest)=32 THEN digest END AS digest FROM current_state WHERE key='authority'");
statement.setReadBigInts(true);
const row = statement.get();
if (!row && operation === "load") return null;
if (!row) fail("stale_authority", "outstanding work has no current authority");
if (!(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array) || !createHash("sha256").update(row.bytes).digest().equals(Buffer.from(row.digest))) fail("invalid_authority", "current authority digest mismatch");
const preparedStatement = database!.prepare("SELECT generation,CASE WHEN length(body)<=16777216 THEN body END AS body,digest FROM receipt_prepare WHERE state_key='authority'");
preparedStatement.setReadBigInts(true);
const prepared = preparedStatement.get();
if (prepared && (prepared.generation !== row.generation || !(prepared.body instanceof Uint8Array) || !(prepared.digest instanceof Uint8Array)
|| !Buffer.from(prepared.body).equals(Buffer.from(row.bytes)) || !Buffer.from(prepared.digest).equals(Buffer.from(row.digest)))) fail("invalid_authority", "current authority has conflicting prepared receipts");
if (operation === "getRecord") {
const { stdout } = await promisify(execFile)(runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "read-receipt", "--path", path,
"--binding", binding, "--generation", String(row.generation), "--namespace", namespace(String(input.epoch), input.kind as AuthorityRecord["kind"]), "--key", String(input.id)],
{ env: {}, windowsHide: true, timeout: 30_000, maxBuffer: 2 * 1024 * 1024 });
const bytes = JSON.parse(stdout) as string | null;
return bytes === null ? null : decode<AuthorityRecord>(bytes);
}
if (operation !== "load") {
const collection = String(input.collection) as AuthorityWorkRecord["collection"];
const get = (id: string): AuthorityWorkRecord | null => {
validateAuthorityWorkKey(collection, id);
if (!database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='current_work'").get()) return null;
const row = database!.prepare("SELECT CASE WHEN length(bytes)<=16384 THEN bytes END AS bytes,CASE WHEN length(digest)=32 THEN digest END AS digest FROM current_work WHERE collection=? AND key=?").get(collection, id);
if (!row) return null;
if (!(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array) || !createHash("sha256").update(row.bytes).digest().equals(Buffer.from(row.digest))) fail("invalid_authority", "outstanding-work digest mismatch");
return { collection, id, body: JSON.parse(Buffer.from(row.bytes).toString()), sha256: Buffer.from(row.digest).toString("hex") };
};
if (operation === "getWork") return get(String(input.id));
const after = String(input.after), limit = Number(input.limit), generation = String(input.expectedGeneration);
validateAuthorityWorkPage(collection, after, limit, generation);
if (String(row.generation) !== generation) fail("stale_authority", "outstanding-work page generation changed");
if (!database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='current_work'").get()) return { records: [], nextAfter: null };
const rows = database!.prepare("SELECT key FROM current_work WHERE collection=? AND key>? ORDER BY key LIMIT ?").all(collection, after, limit);
const records = rows.map(row => get(String(row.key))!);
return { records, nextAfter: records.at(-1)?.id ?? null };
}
return { generation: String(row.generation), state: JSON.parse(Buffer.from(row.bytes).toString()) };
} finally { database!.exec("ROLLBACK"); }
}
switch (operation) {
case "ready": await rpc("ready", {}); return { unorderedEffectReceipts: true, commandEpochs: true, eventEpochs: true };
case "load": {
const value = await rpc<{ generation: string; bytes: string } | null>("load", { key: "authority" });
return value && { generation: value.generation, state: decode<AuthoritySnapshot["state"]>(value.bytes) };
}
case "getRecord": return get(namespace(String(input.epoch), input.kind as AuthorityRecord["kind"]), String(input.id));
case "getSessionEffect": return get("session-effects", String(input.id));
case "getWork": {
const collection = String(input.collection) as AuthorityWorkRecord["collection"], id = String(input.id);
validateAuthorityWorkKey(collection, id);
const row = await rpc<NativeWork | null>("getWork", { collection, key: id });
return row && decodeWork(collection, row);
}
case "readWorkPage": {
const collection = String(input.collection) as AuthorityWorkRecord["collection"], after = String(input.after), limit = Number(input.limit), expectedGeneration = String(input.expectedGeneration);
validateAuthorityWorkPage(collection, after, limit, expectedGeneration);
const assertGeneration = async () => {
const state = await rpc<{ generation: string } | null>("load", { key: "authority" });
if (!state || state.generation !== expectedGeneration) fail("stale_authority", "outstanding-work page generation changed");
};
await assertGeneration();
const rows = await rpc<NativeWork[]>("workPage", { collection, after, limit: String(limit) });
await assertGeneration();
return { records: rows.map(row => decodeWork(collection, row)), nextAfter: rows.at(-1)?.key ?? null };
}
case "commit": {
const commit = input as unknown as AuthorityCommit;
validateAuthorityCommit(commit);
const receipts = commit.records.flatMap(record => {
const bytes = encoded(record);
const entries = [{ namespace: namespace(record.epoch, record.kind), key: record.id, bytes }];
if (record.kind === "effect") entries.push({ namespace: "session-effects", key: record.id, bytes });
else entries.push({ namespace: sequenceNamespace(record.epoch, record.kind) + (record.sequenceEpoch ? `/${record.sequenceEpoch}` : ""), key: record.sequence.padStart(19, "0"), bytes: encoded(record.id) });
return entries;
});
const work = (commit.work ?? []).map(change => ({ collection: change.collection, key: change.id,
expectedDigest: change.expectedSha256 === null ? null : Buffer.from(change.expectedSha256, "hex").toString("base64"), bytes: change.body === null ? null : encoded(change.body) }));
return rpc("commit", { key: "authority", expectedGeneration: commit.expectedGeneration, bytes: encoded(commit.state), receipts, work });
}
case "readEvents": {
const epoch = String(input.epoch), after = String(input.after), limit = Number(input.limit), byteBudget = Number(input.byteBudget);
validateAuthorityPage(after, limit, byteBudget, input.sequenceEpoch as string | undefined);
const rows = await rpc<Array<{ key: string; bytes: string }>>("page", { namespace: sequenceNamespace(epoch, "event") + (input.sequenceEpoch ? `/${input.sequenceEpoch}` : ""), after: authorityInteger(after).toString().padStart(19, "0"), limit: String(limit), byteBudget: "1048576" });
const records: AuthorityRecord[] = [];
let bytes = 0;
for (const row of rows) {
const record = await get(namespace(epoch, "event"), decode<string>(row.bytes));
if (!record || record.sequenceEpoch !== input.sequenceEpoch || record.sequence.padStart(19, "0") !== row.key) fail("invalid_authority", "receipt sequence index changed");
const size = Buffer.byteLength(authorityJson(record.body));
if (bytes + size > byteBudget) break;
bytes += size; records.push(record);
}
return { records, nextAfter: records.at(-1)?.sequence ?? null };
}
case "close": {
closing = true;
const exited = new Promise<void>((resolve, reject) => { child!.once("exit", code => code === 0 ? resolve() : reject(new Error(`storage close failed (${code})`))); });
await rpc("close", {}); child!.stdin.end(); await exited; return null;
}
default: fail("invalid_authority", "unknown storage operation");
}
}
// Serialize temporary copies and commit order; the owner admits at most 8.
let chain = Promise.resolve();
parentPort!.on("message", ({ id, operation, input }: { id: number; operation: string; input: Record<string, unknown> }) => {
chain = chain.then(async () => {
try { parentPort!.postMessage({ id, value: await dispatch(operation, input) }); }
catch (error) { parentPort!.postMessage({ id, error: { code: error instanceof DurableAuthorityStoreError ? error.code : "storage_unavailable", message: error instanceof Error ? error.message : "storage worker failed" } }); }
});
});
}
@@ -16,6 +16,25 @@ function nodeTransport(
}
describe("Codex app-server transport limits", () => {
it("correlates concurrent out-of-order responses by opaque request identity", async () => {
const transport = nodeTransport(`
const readline = require("node:readline");
const pending = [];
readline.createInterface({input:process.stdin}).on("line", line => {
const request = JSON.parse(line); pending.push(request);
if (pending.length === 3) for (const value of pending.reverse()) {
process.stdout.write(JSON.stringify({id:value.id,result:{id:value.id,method:value.method}})+"\\n");
}
});
`);
try {
const values = await Promise.all(["first", "second", "third"].map(method => transport.request(method, {})));
expect(values.map(value => value.method)).toEqual(["first", "second", "third"]);
expect(new Set(values.map(value => value.id)).size).toBe(3);
for (const value of values) expect(value.id).toMatch(/^paperclip-rpc-[a-f0-9-]{36}$/);
} finally { await transport.close(); }
});
it("passes only bounded controller-projected GitHub credentials", () => {
expect(
createSanitizedCodexEnvironment({
@@ -1,10 +1,14 @@
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import type { HarnessRuntimeRequestResolution } from "../../contracts/harness-driver.js";
import { githubCredentialEnvironment } from "../../github-credential-environment.js";
import type { NormalizedDeliveryPort, RawDeliveryCursor } from "../../control-plane/normalized-delivery.js";
export interface CodexRpcNotification {
method: string;
params: Record<string, unknown>;
/** Durable consumer position, always present on indexed runner deliveries. */
paperclipDelivery?: RawDeliveryCursor;
/** Internal-only correlation retained outside provider params and canonical PRP payloads. */
paperclipTrace?: {
sourceEventId: string;
@@ -36,6 +40,8 @@ export interface CodexAppServerTransport {
): Promise<Record<string, unknown>>;
notify(method: string, params?: Record<string, unknown>): void;
notifications(): AsyncIterable<CodexRpcNotification>;
normalizedDelivery?(): NormalizedDeliveryPort | null;
setBeforeRunRotation?(settle: () => Promise<void>): void;
setServerRequestHandler(handler: CodexServerRequestHandler): void;
/**
* Optional provider-neutral resolution path used when a transport has
@@ -367,8 +373,7 @@ const MAX_DIAGNOSTIC_LINE_BYTES = 16 * 1024;
export class ProcessCodexAppServerTransport implements CodexAppServerTransport {
#process: ChildProcessWithoutNullStreams;
#notifications: BoundedAsyncQueue<CodexRpcNotification>;
#pending = new Map<number, PendingRequest>();
#nextId = 1;
#pending = new Map<string | number, PendingRequest>();
#serverRequestHandler: CodexServerRequestHandler = async () => ({
success: false,
contentItems: [{ type: "input_text", text: "Unsupported client request." }],
@@ -502,7 +507,8 @@ export class ProcessCodexAppServerTransport implements CodexAppServerTransport {
),
);
}
const id = this.#nextId++;
const id = `paperclip-rpc-${randomUUID()}`;
if (this.#pending.has(id)) return Promise.reject(new Error("codex app-server fresh request identity unavailable"));
return new Promise((resolve, reject) => {
this.#pending.set(id, { resolve, reject });
try {
@@ -634,10 +640,10 @@ export class ProcessCodexAppServerTransport implements CodexAppServerTransport {
const message = value;
const hasResult = Object.hasOwn(message, "result");
const hasError = Object.hasOwn(message, "error");
if (typeof message.id === "number" && (hasResult || hasError)) {
if ((typeof message.id === "number" || typeof message.id === "string") && (hasResult || hasError)) {
if (
!Number.isSafeInteger(message.id) ||
message.id <= 0 ||
(typeof message.id === "number" && (!Number.isSafeInteger(message.id) || message.id <= 0)) ||
(typeof message.id === "string" && (message.id.length === 0 || message.id.length > 160)) ||
hasResult === hasError
) {
this.#fatal(
@@ -379,7 +379,10 @@ export class CodexAppServerDriver implements HarnessDriver {
);
await cancellation.wait(this.#persistProcessOwnership(transport));
const existingThread = record(existing.thread);
existingThread.turns = await cancellation.wait(readCodexTurnMetadata(transport, snapshot.driverSessionId));
const normalizedRecovery = transport.normalizedDelivery?.()?.load() ?? null;
// The indexed reducer plus its unconsumed raw suffix owns recovery.
// Enumerating provider turns here would reintroduce lifetime-sized I/O.
existingThread.turns = normalizedRecovery ? [] : await cancellation.wait(readCodexTurnMetadata(transport, snapshot.driverSessionId));
if (text(existingThread.id) !== snapshot.driverSessionId) {
await cancellation.wait(cancellation.close());
return {
@@ -473,7 +476,7 @@ export class CodexAppServerDriver implements HarnessDriver {
const goal = await cancellation.wait(
this.#discoverGoal(transport, opened.threadId),
);
const recoveringAutonomousGoal = snapshot.goal?.status === "active"
const recoveringAutonomousGoal = normalizedRecovery === null && snapshot.goal?.status === "active"
&& goal != null
&& goal.createdAt === snapshot.goal.createdAt;
if (recoveringAutonomousGoal) {
@@ -496,7 +499,7 @@ export class CodexAppServerDriver implements HarnessDriver {
recoveredActiveTurnId = active.length === 1 ? text(active[0]?.id) : recoveredActiveTurnId;
}
if (
!recoveringAutonomousGoal &&
normalizedRecovery === null && !recoveringAutonomousGoal &&
!this.#direct() &&
snapshot.semanticResult == null &&
recoveredActiveTurnId === null &&
@@ -608,6 +611,7 @@ export class CodexAppServerDriver implements HarnessDriver {
stalePendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
lineage: snapshot.lineage,
sourceSequence: snapshot.lastSourceSequence ?? 0,
sourceEpoch: snapshot.lastSourceEpoch,
});
// A provider may settle the checkpointed turn while this controller is
// disconnected (including during timeout cleanup). Reopening a thread
@@ -616,8 +620,8 @@ export class CodexAppServerDriver implements HarnessDriver {
// nor submit the original work again. Missing/conflicting history still
// fails closed in reconcile().
if (
recoveredActiveTurnId !== null ||
reconcileUncheckpointedDispositionTurn
!session.deliveryRestored && (recoveredActiveTurnId !== null ||
reconcileUncheckpointedDispositionTurn)
) {
await cancellation.wait(session.reconcile?.() ?? Promise.resolve({}));
}
@@ -904,6 +908,7 @@ export class CodexAppServerDriver implements HarnessDriver {
stalePendingRuntimeRequests?: HarnessRuntimeRequest[];
lineage?: HarnessThreadLineageEntry[];
sourceSequence: number;
sourceEpoch?: string;
}): CodexHarnessSession {
return new CodexHarnessSession({
...input,
@@ -1,5 +1,8 @@
import { realpathSync } from "node:fs";
import { mkdtempSync, realpathSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterAll } from "vitest";
import {
CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
@@ -45,7 +48,10 @@ import {
type CodexTraceInterpretation,
} from "./app-server-transport.js";
export const WORKSPACE = realpathSync.native(process.cwd());
// A managed checkout can live under ~/.codex, which production correctly
// rejects as provider working space. Fake sessions own a real private fixture.
export const WORKSPACE = realpathSync.native(mkdtempSync(join(tmpdir(), "codex-driver-test-")));
afterAll(() => rmSync(WORKSPACE, { recursive: true, force: true }));
export class TestQueue<T> implements AsyncIterable<T> {
values: T[] = [];
@@ -59,14 +59,17 @@ export class CodexHarnessSession
extends CodexSessionState
implements HarnessSession
{
#eventCommitter: ((event: PrpEvent) => Promise<void>) | null = null;
constructor(input: CodexSessionStateInput) {
super(input);
this.transport.setServerRequestHandler((request) =>
handleServerRequest(this, request),
);
initializeCodexSessionEvents(this, input);
if (this.terminal) {
this.eventQueue.close();
this.transport.setBeforeRunRotation?.(() => this.flushEventDelivery());
if (!this.deliveryRestored) initializeCodexSessionEvents(this, input);
if (this.terminal && !this.deliveryWriter) {
this.closeEvents();
} else {
void pumpNotifications(this);
}
@@ -116,6 +119,7 @@ export class CodexHarnessSession
this.resultFingerprint = null;
this.resultCallId = null;
this.resultTurnId = null;
this.durableTerminal = null;
this.dispositionOnlyRecoveryConsumed = false;
this.dispositionOnlyRecoveryTurnId = null;
this.terminal = false;
@@ -124,6 +128,10 @@ export class CodexHarnessSession
this.protocolFailed = false;
this.protocolFailureCode = null;
this.protocolFailureMessage = null;
this.deliveryWriter = null;
this.deliveryRestored = false;
if (this.transport.normalizedDelivery?.()) { this.sourceSequence = 0; this.sourceSequenceEpoch = null; }
this.initializeDelivery();
this.emit("run.attached", { runId: input.runId, sameSession: true });
}
@@ -135,6 +143,28 @@ export class CodexHarnessSession
return this.eventQueue;
}
async acknowledgeEvent(event: PrpEvent): Promise<void> {
await this.deliveryWriter?.acknowledge(event);
}
setEventCommitter(commit: (event: PrpEvent) => Promise<void>): void {
this.#eventCommitter = commit;
}
async flushEventDelivery(): Promise<void> {
const writer = this.deliveryWriter;
if (!writer) return;
await writer.flush();
for (;;) {
const event = writer.port.load()?.pending[0];
if (!event) return;
if (!this.#eventCommitter) throw new Error("normalized event sink is unavailable before run rotation");
await this.#eventCommitter(event);
await writer.acknowledge(event);
this.eventQueue.discard((queued) => queued.sourceEventId === event.sourceEventId);
}
}
async startTurn(input: {
message: NativeUserMessage;
/** Set by orchestration only after successful provider-session recovery. */
@@ -308,6 +338,7 @@ export class CodexHarnessSession
this.requireCapability("steering");
this.requireActiveTurn(input.turnId, "steering");
if (input.correlationId) {
await this.prefetchHistory("steering", [input.correlationId]);
const acknowledgedTurnId = this.acknowledgedSteeringCorrelations.get(
input.correlationId,
);
@@ -344,7 +375,7 @@ export class CodexHarnessSession
turnId: input.turnId,
itemId: input.correlationId
? `${input.turnId}:steer:${input.correlationId}`
: `${input.turnId}:steer:${++this.steerSequence}`,
: `${input.turnId}:steer:${randomUUID()}`,
},
);
} catch (error) {
@@ -372,7 +403,7 @@ export class CodexHarnessSession
text: "Interrupt queued until the provider assigns the turn identity.",
status: "queued",
},
{ itemId: `interrupt:queued:${++this.interruptSequence}` },
{ itemId: `interrupt:queued:${randomUUID()}` },
);
return;
}
@@ -403,7 +434,7 @@ export class CodexHarnessSession
},
{
turnId,
itemId: `${turnId}:interrupt:${++this.interruptSequence}`,
itemId: `${turnId}:interrupt:${randomUUID()}`,
},
);
} catch (error) {
@@ -599,7 +630,7 @@ export class CodexHarnessSession
action: input.action,
goal,
},
{ itemId: `${this.opened.threadId}:goal:${this.sourceSequence + 1}` },
{ itemId: `${this.opened.threadId}:goal:${randomUUID()}` },
);
this.emitGoalEvent(
input.action === "clear"
@@ -649,6 +680,15 @@ export class CodexHarnessSession
async reconcile(): Promise<Record<string, unknown>> {
this.assertProtocolIntegrity();
this.requireCapability("reconciliation");
if (this.deliveryWriter) {
await this.deliveryWriter.flush();
const snapshot = await this.read();
const thread = record(snapshot.thread);
if (text(thread.id) !== this.opened.threadId || (this.opened.providerSessionId !== null && text(thread.sessionId) !== this.opened.providerSessionId)) {
throw new HarnessReconciliationError("thread/read returned a different indexed provider session");
}
return snapshot;
}
const snapshot = await this.read();
const thread = record(snapshot.thread);
if (text(thread.id) !== this.opened.threadId) {
@@ -768,6 +808,7 @@ export class CodexHarnessSession
async snapshot(): Promise<PersistedHarnessSession> {
this.assertProtocolIntegrity();
await this.deliveryWriter?.flush();
return {
driverKind: this.driverKind,
workingDirectory: this.opened.context.workingDirectory,
@@ -795,6 +836,7 @@ export class CodexHarnessSession
turnId,
fingerprint,
})),
...(this.deliveryWriter && this.durableTerminal ? { durableTerminal: structuredClone(this.durableTerminal) } : {}),
dispositionOnlyRecoveryConsumed: this.dispositionOnlyRecoveryConsumed,
dispositionOnlyRecoveryTurnId: this.dispositionOnlyRecoveryTurnId,
pendingRuntimeRequests: this.pendingRuntimeRequests(),
@@ -802,11 +844,14 @@ export class CodexHarnessSession
this.currentGoal === null ? null : structuredClone(this.currentGoal),
lineage: this.lineage(),
lastSourceSequence: this.sourceSequence,
...(this.sourceSequenceEpoch ? { lastSourceEpoch: this.sourceSequenceEpoch } : {}),
};
}
async close(input?: { reason: string }): Promise<void> {
this.cancelPendingRequests("session_closed");
await this.deliveryWriter?.flush();
if (this.#eventCommitter) await this.flushEventDelivery();
this.eventQueue.close();
await this.transport.close(input?.reason);
}
@@ -0,0 +1,87 @@
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { DurablePrpControlPlane } from "../../control-plane/durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "../../control-plane/sqlite-authority-store.js";
import { CodexHistoryMap } from "./codex-history-cache.js";
it("evicts only cache entries: ancient identities remain exact after 2,000 receipts and reopen", async () => {
const root = await mkdtemp(join(tmpdir(), "driver-history-"));
const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", runId: "run-1", turnId: "turn", itemId: "item" };
const config = { path: join(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true };
let store = await SqliteAuthorityStore.open(config);
const options = { stateDirectory: root, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` };
let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store });
try {
let port = core.normalizedDelivery()!;
const cache = new CodexHistoryMap<string>("steering", port.history!, [], false);
for (let page = 0; page < 20; page++) {
for (let i = 0; i < 100; i++) cache.set(`call-${page * 100 + i}`, `turn-${page * 100 + i}`);
await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: { epoch: identity.runId, sourceSeq: 0, ordinal: 0 }, driver: { cache: [...cache] }, events: [], receipts: cache.takeReceipts() });
expect(cache.size).toBeLessThanOrEqual(128);
expect(JSON.stringify((await store.load())!.state).length).toBeLessThan(20_000);
}
expect(() => cache.get("call-0")).toThrow("exact receipt read");
await cache.prefetch(["call-0", "never-accepted"]);
expect(cache.get("call-0")).toBe("turn-0");
expect(cache.has("never-accepted")).toBe(false);
await core.stop(); await store.close();
store = await SqliteAuthorityStore.open({ ...config, create: false });
core = await DurablePrpControlPlane.open({ ...options, authorityStore: store });
port = core.normalizedDelivery()!;
expect(await port.history!.get("steering", "call-0")).toBe("turn-0");
await core.rotateRunIdentity({ ...identity, runId: "run-2", turnId: "turn-2", itemId: "item-2" });
await expect(port.history!.get("steering", "call-0")).rejects.toThrow("retired run");
port = core.normalizedDelivery()!;
expect(await port.history!.get("steering", "call-0")).toBe("turn-0");
const before = await store.load();
await expect(port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: { epoch: "run-2", sourceSeq: 0, ordinal: 0 }, driver: {}, events: [], receipts: [
{ collection: "steering", key: "new-before-conflict", value: "must-not-commit" },
{ collection: "steering", key: "call-0", value: "different-turn" },
] })).rejects.toThrow("receipt_conflict");
expect(await port.history!.get("steering", "new-before-conflict")).toBeNull();
expect(await store.load()).toEqual(before);
} finally { await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); }
}, 120_000);
it("normalizes children of evicted ancestors through exact indexed lineage without growing current state", async () => {
const { FakeCodexTransport, makeDriver, WORKSPACE } = await import("./codex-app-server-driver.test-support.js");
const { vi } = await import("vitest");
const root = await mkdtemp(join(tmpdir(), "driver-lineage-history-"));
const identity = { runnerInstanceId: "runner-codex", environmentLeaseId: "lease", normalizedSessionId: "session-1", runId: "run-1", turnId: "turn-1", itemId: "item-1" };
const store = await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true });
const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, authorityStore: store, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
const port = core.normalizedDelivery()!;
class Transport extends FakeCodexTransport { normalizedDelivery() { return port; } }
const transport = new Transport("thread-root");
const session = await makeDriver([transport]).openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: WORKSPACE });
let lastThread: string | null = null;
let failure: unknown;
session.setEventCommitter!(async event => {
if (event.eventType === "session.failed") failure = event;
if (event.eventType === "item.started" && event.payload.kind === "thread_lineage") lastThread = (event.payload.lineage as { threadId: string }).threadId;
});
const started = (id: string, parent: string, depth: number) => transport.queue.push({ method: "thread/started", params: { thread: { id, sessionId: "codex-account-session", source: { subAgent: { thread_spawn: { parent_thread_id: parent, depth } } }, status: { type: "idle" } } } });
try {
await session.startTurn({ message: { text: "Work" } });
await session.flushEventDelivery!();
for (let index = 0; index < 300; index++) {
started(`thread-child-${index}`, "thread-root", 1);
await vi.waitFor(async () => { expect(await port.history!.get("lineage", `thread-child-${index}`)).not.toBeNull(); }, { interval: 1, timeout: 5_000 });
await session.flushEventDelivery!();
expect(failure).toBeUndefined();
expect(lastThread).toBe(`thread-child-${index}`);
transport.queue.push({ method: "thread/closed", params: { threadId: `thread-child-${index}` } });
}
await session.flushEventDelivery!();
expect(session.lineage!().length).toBeLessThanOrEqual(128);
expect(session.lineage!().some(thread => thread.threadId === "thread-child-0")).toBe(false);
started("thread-grandchild", "thread-child-0", 2);
await vi.waitFor(async () => { expect(await port.history!.get("lineage", "thread-grandchild")).not.toBeNull(); }, { interval: 1, timeout: 5_000 });
await session.flushEventDelivery!();
expect(await port.history!.get("lineage", "thread-grandchild")).toMatchObject({ threadId: "thread-grandchild", parentThreadId: "thread-child-0", depth: 2 });
expect(Buffer.byteLength(JSON.stringify((await store.load())!.state))).toBeLessThan(100_000);
expect(port.load()!.pending).toEqual([]);
} finally { await session.close({ reason: "test complete" }); transport.queue.close(); await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); }
}, 120_000);
@@ -0,0 +1,77 @@
import type { DriverHistoryCollection, DriverHistoryReceipt, NormalizedDeliveryPort } from "../../control-plane/normalized-delivery.js";
const CACHE_ENTRIES = 128;
/** A cache miss is never permission to forget a completed action. Callers must
* prefetch the exact keys before entering their synchronous reducer step. */
export class CodexHistoryMap<T> extends Map<string, T> {
#absent = new Set<string>();
#dirty = new Map<string, T>();
constructor(readonly collection: DriverHistoryCollection, private readonly history: NonNullable<NormalizedDeliveryPort["history"]>, entries: Iterable<[string, T]>, restored: boolean, private readonly encode: (value: T) => unknown = (value) => value, private readonly decode: (value: unknown) => T = (value) => value as T) {
super();
for (const [key, value] of entries) {
if (!restored) this.#dirty.set(key, structuredClone(value));
super.set(key, structuredClone(value));
}
if (this.size > CACHE_ENTRIES) throw new Error("indexed driver cache requires a fenced history import");
}
override get(key: string): T | undefined {
if (super.has(key)) return super.get(key);
if (key === "" || this.#absent.has(key)) return undefined;
throw new Error(`indexed driver ${this.collection} lookup requires an exact receipt read`);
}
override has(key: string): boolean { return this.get(key) !== undefined; }
override set(key: string, value: T): this {
this.#absent.delete(key);
this.#dirty.set(key, structuredClone(value));
super.delete(key);
super.set(key, value);
this.#trim();
return this;
}
override clear(): void {
super.clear();
this.#absent.clear();
this.#dirty.clear();
}
#trim(): void {
while (this.size > CACHE_ENTRIES) super.delete(super.keys().next().value!);
while (this.#absent.size > CACHE_ENTRIES) this.#absent.delete(this.#absent.values().next().value!);
}
async prefetch(keys: Iterable<string>): Promise<void> {
const requested = [...new Set(keys)].filter(Boolean);
if (requested.length > CACHE_ENTRIES) throw new Error("storage_pressure: driver receipt lookup batch exceeds capacity");
for (const key of requested) {
if (super.has(key)) { const value = super.get(key)!; super.delete(key); super.set(key, value); continue; }
if (this.#absent.has(key)) { this.#absent.delete(key); this.#absent.add(key); continue; }
const value = await this.history.get(this.collection, key);
if (value === null) this.#absent.add(key);
else super.set(key, this.decode(structuredClone(value)));
this.#trim();
}
}
takeReceipts(): DriverHistoryReceipt[] {
const receipts = [...this.#dirty].map(([key, value]) => ({ collection: this.collection, key, value: this.encode(value) }));
this.#dirty.clear();
return receipts;
}
}
export class CodexHistorySet implements Iterable<string> {
readonly map: CodexHistoryMap<boolean>;
constructor(history: NonNullable<NormalizedDeliveryPort["history"]>, entries: Iterable<string>, restored: boolean) {
this.map = new CodexHistoryMap<boolean>("file", history, [...entries].map((key) => [key, true]), restored);
}
has(key: string): boolean { return this.map.has(key); }
add(key: string): this { this.map.set(key, true); return this; }
clear(): void { this.map.clear(); }
[Symbol.iterator](): IterableIterator<string> { return this.map.keys(); }
}
@@ -0,0 +1,83 @@
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { SqliteAuthorityStore } from "../../control-plane/sqlite-authority-store.js";
import { applyNormalizedBatch, acknowledgeNormalizedEvent, type NormalizedDeliveryState, type NormalizedDeliveryPort } from "../../control-plane/normalized-delivery.js";
import { FakeCodexTransport, makeDriver, WORKSPACE, type PrpEvent } from "./codex-app-server-driver.test-support.js";
it.each([undefined, 4])("recovers a pending question from SQLite with normalized epoch limit %s without enumerating provider history", async limit => {
const root = await mkdtemp(join(tmpdir(), "codex-normalizer-"));
const config = { path: join(root, "consumer.sqlite"), binding: "test-normalized-consumer", create: true };
let store = await SqliteAuthorityStore.open(config);
let current: NormalizedDeliveryState | null = null;
let generation = "0";
let tail: Promise<unknown> = Promise.resolve();
const serialize = <T>(action: () => Promise<T>): Promise<T> => {
const result = tail.then(action); tail = result.catch(() => undefined); return result;
};
const port: NormalizedDeliveryPort = {
epoch: "run-1",
...(limit ? { eventEpochs: { limit } } : {}),
load: () => structuredClone(current),
commit: (batch) => serialize(async () => {
const candidate = applyNormalizedBatch(current, batch);
generation = await store.commit({ expectedGeneration: generation, state: { consumer: candidate }, records: [] });
current = candidate;
return structuredClone(candidate);
}),
acknowledge: (event) => serialize(async () => {
const candidate = acknowledgeNormalizedEvent(current!, event);
generation = await store.commit({ expectedGeneration: generation, state: { consumer: candidate }, records: [] });
current = candidate;
}),
};
class Transport extends FakeCodexTransport { normalizedDelivery() { return port; } }
const first = new Transport(), second = new Transport();
const driver = makeDriver([first, second]);
try {
const session = await driver.openSession({ runId: "run-1", normalizedSessionId: "session-1", workingDirectory: WORKSPACE });
const written: PrpEvent[] = [];
session.setEventCommitter!(async (event) => { written.push(event); });
await session.startTurn({ message: { text: "Work" } });
await session.flushEventDelivery!();
const checkpointBeforeRawDelivery = await session.snapshot();
first.queue.push({ method: "paperclip/runtimeRequest", params: { request: {
id: "question-1", method: "item/tool/requestUserInput", params: {
threadId: "thread-1", turnId: "turn-1", questions: [{ id: "color", header: "Color", question: "Which color?", options: [{ label: "Blue", description: "Blue" }, { label: "Red", description: "Red" }] }],
},
} }, paperclipDelivery: { epoch: "run-1", sourceSeq: 1, ordinal: 1 } });
await vi.waitFor(() => expect(port.load()?.raw).toEqual({ epoch: "run-1", sourceSeq: 1, ordinal: 1 }));
const pending = port.load()!.pending;
if (limit) expect(port.load()!.producedEpoch).toBeTruthy();
expect(pending.some((event) => event.eventType === "runtime_request.created")).toBe(true);
// Simulate the controller disappearing after mapping but before append.
// The fake provider remains; no close/cancellation is sent to it.
first.queue.close();
await tail; await store.close();
store = await SqliteAuthorityStore.open({ ...config, create: false });
const restored = (await store.load())!;
generation = restored.generation;
current = restored.state.consumer as unknown as NormalizedDeliveryState;
const recovery = await driver.recoverSession(checkpointBeforeRawDelivery, { signal: new AbortController().signal });
expect(recovery.recovered).toBe(true);
const recovered = recovery.session!;
expect(second.calls.some((call) => call.method === "thread/turns/list" || call.method === "thread/items/list")).toBe(false);
expect(recovered.pendingRuntimeRequests!().map((request) => request.requestId)).toEqual(["question-1"]);
const reader = recovered.events()[Symbol.asyncIterator]();
for (const expected of pending) {
const actual = (await reader.next()).value!;
expect(actual).toEqual(expected);
await recovered.acknowledgeEvent!(actual);
}
expect(port.load()!.pending).toEqual([]);
recovered.setEventCommitter!(async (event) => { written.push(event); });
await recovered.resolveRuntimeRequest!({ requestId: "question-1", turnId: "turn-1", resolution: { action: "submit", answers: { color: { answers: ["Blue"] } } } });
await recovered.flushEventDelivery!();
expect(recovered.pendingRuntimeRequests!()).toEqual([]);
expect(written.filter((event) => event.eventType === "runtime_request.resolved")).toHaveLength(1);
await recovered.close({ reason: "test complete" });
} finally {
first.queue.close(); second.queue.close(); await tail; await store.close(); await rm(root, { recursive: true, force: true });
}
});
@@ -54,7 +54,7 @@ async function authenticatedRunner(
};
const digest = (domain: string, parts: Buffer[]) =>
createHash("sha256").update(framed(domain, parts)).digest();
const credential = Buffer.from(core.issueBootstrapTicket());
const credential = Buffer.from((await core.issueBootstrapTicket()));
const authKey = digest("paperclip-runner-auth-key-v1", [credential]);
const mac = (domain: string, parts: Buffer[]) =>
createHmac("sha256", authKey).update(framed(domain, parts)).digest();
@@ -607,8 +607,8 @@ describe("Codex protocol integrity propagation", () => {
result,
},
});
await vi.waitFor(() =>
expect(core.getCommand(command.commandId)?.status).toBe("completed"),
await vi.waitFor(async () =>
expect((await core.getCommand(command.commandId))?.status).toBe("completed"),
);
};
const event = (
@@ -1,8 +1,10 @@
import { randomUUID } from "node:crypto";
import { observeCodexUsage, codexRunUsage } from "./codex-usage-baseline.js";
import { classifyCodexNotification } from "./codex-notification-identity.js";
import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js";
import { canonicalProviderEventsFromCodex, isCanonicalProviderEventType } from "../../provider-events.js";
import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js";
import { handleServerRequest } from "./codex-session-server-requests.js";
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js";
import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js";
@@ -39,9 +41,31 @@ import {
export async function pumpNotifications(state: CodexSessionState): Promise<void> {
try {
for await (const notification of state.transport.notifications()) {
await state.deliveryWriter?.waitForCapacity();
if (notification.method === "paperclip/runtimeRequest") {
const request = notification.params.request as unknown as import("./app-server-transport.js").CodexRpcServerRequest;
void handleServerRequest(state, { ...request, paperclipDelivery: notification.paperclipDelivery }).catch((error) => state.eventQueue.fail(error));
await state.deliveryWriter?.flush();
continue;
}
if (notification.method === "turn/completed") await state.turnStartSettled;
const params = notification.params;
const lineage = lineageFromThread(params.thread);
await state.prefetchHistory("lineage", [state.opened.threadId, text(params.threadId), text(record(params.turn).threadId), lineage.threadId, lineage.parentThreadId ?? ""]);
const turnId = text(params.turnId, text(record(params.turn).id, state.activeTurnId ?? ""));
await state.prefetchHistory("terminal", [turnId]);
const item = itemFromParams(params);
if (notification.method === "item/completed" && text(item.type) === "agentMessage") {
await state.prefetchHistory("file", paperclipWorkspaceFileReferencesFromText(state.opened.context.workingDirectory, text(item.text), turnId).map((reference) => reference.referenceId));
}
await mapNotification(state, notification);
await state.deliveryWriter?.flush();
}
} catch (error) {
if (state.deliveryWriter?.failure) {
state.eventQueue.fail(state.deliveryWriter.failure);
return;
}
if (error instanceof NativeSessionProtocolIntegrityError) {
state.failProtocolIntegrity(error);
return;
@@ -59,26 +83,17 @@ export async function pumpNotifications(state: CodexSessionState): Promise<void>
}
async function mapNotification(state: CodexSessionState, notification: CodexRpcNotification): Promise<void> {
const sourceSequenceBefore = state.sourceSequence;
const emittedEventIds: string[] = [];
let rejected = false;
try {
await mapNotificationBody(state, notification);
state.capturedEventIds.run(emittedEventIds, () => mapNotificationBody(state, notification));
state.deliveryWriter?.changed(notification.paperclipDelivery);
} catch (error) {
rejected = true;
throw error;
} finally {
const correlation = notification.paperclipTrace;
if (correlation !== undefined) {
const emittedEventIds: string[] = [];
for (
let sourceSeq = sourceSequenceBefore + 1;
sourceSeq <= state.sourceSequence;
sourceSeq += 1
) {
emittedEventIds.push(
`${state.runnerInstanceId}:${state.runId}:${sourceSeq}`,
);
}
const disposition: CodexTraceInterpretation["disposition"] = rejected
? "rejected"
: emittedEventIds.length > 0
@@ -104,7 +119,7 @@ async function mapNotification(state: CodexSessionState, notification: CodexRpcN
}
}
async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise<void> {
function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): void {
if (!isSupportedCodexNotificationMethod(notification.method)) return;
if (notification.method === "item/completed" && notification.params.kind === "steering_acknowledgement"
&& !notification.params.threadId && !notification.params.turnId && !notification.params.thread && !notification.params.turn) return;
@@ -270,7 +285,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
},
{
turnId: turnId || undefined,
itemId: `${threadId}:goal:update:${state.sourceSequence + 1}`,
itemId: `${threadId}:goal:update:${randomUUID()}`,
},
);
state.emitGoalEvent("session.goal.updated", goal, {
@@ -289,7 +304,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
action: "notification",
goal: null,
},
{ itemId: `${threadId}:goal:clear:${state.sourceSequence + 1}` },
{ itemId: `${threadId}:goal:clear:${randomUUID()}` },
);
state.emitGoalEvent("session.goal.cleared", null, {
workingNow: state.activeTurnId !== null,
@@ -429,7 +444,6 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
// channel before turn/start's own response settles on the request
// channel. Wait for the pending turn/start to settle first, so
// turn.accepted always precedes the terminal event for the same turn.
await state.turnStartSettled;
if (state.terminalTurns.has(turnId)) {
mapTerminalTurn(state, turn, turnId);
return;
@@ -448,6 +462,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
kind: text(item.type, "unknown"),
channel,
providerPhase: text(item.phase) || undefined,
...(item.outputBody ? { outputBody: item.outputBody } : {}),
text: itemText(item),
item,
}),
@@ -467,6 +482,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
kind: text(item.type, "unknown"),
channel,
providerPhase: text(item.phase) || undefined,
...(item.outputBody ? { outputBody: item.outputBody } : {}),
text: itemText(item),
item,
}),
@@ -519,6 +535,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
kind: deltaKind,
channel,
providerMethod: notification.method,
...(params.outputBody ? { outputBody: params.outputBody } : {}),
text: text(params.delta, text(params.patch, text(params.output))),
update: params,
}),
@@ -546,7 +563,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
{ kind: "usage", usage: state.usageSnapshot },
{
turnId,
itemId: `${turnId}:usage:${state.sourceSequence + 1}`,
itemId: `${turnId}:usage:${randomUUID()}`,
},
);
return;
@@ -29,28 +29,24 @@ export async function handleServerRequest(
state: CodexSessionState,
request: CodexRpcServerRequest,
): Promise<Record<string, unknown>> {
const sourceSequenceBefore = state.sourceSequence;
const emittedEventIds: string[] = [];
let rejected = false;
try {
const response = await handleServerRequestBody(state, request);
if (request.method !== "item/tool/call") state.pendingRuntimeRequestSources.set(String(request.id), structuredClone(request));
const pendingResponse = state.capturedEventIds.run(emittedEventIds, () => handleServerRequestBody(state, request));
// Runtime-input admission executes synchronously, then waits for a user.
// Persist that admission without holding the notification pump hostage.
if (request.method !== "item/tool/call") state.deliveryWriter?.changed(request.paperclipDelivery);
const response = await pendingResponse;
rejected = response.success === false;
return response;
} catch (error) {
rejected = true;
throw error;
} finally {
state.pendingRuntimeRequestSources.delete(String(request.id));
const correlation = request.paperclipTrace;
if (correlation !== undefined) {
const emittedEventIds: string[] = [];
for (
let sourceSeq = sourceSequenceBefore + 1;
sourceSeq <= state.sourceSequence;
sourceSeq += 1
) {
emittedEventIds.push(
`${state.runnerInstanceId}:${state.runId}:${sourceSeq}`,
);
}
try {
state.transport.recordTraceInterpretation?.({
sourceEventId: correlation.sourceEventId,
@@ -1,3 +1,6 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { randomUUID } from "node:crypto";
import { normalizedEventId, type EventEpochTransition } from "../../control-plane/event-epochs.js";
import { type CodexUsageBaseline, codexRunUsage } from "./codex-usage-baseline.js";
import type {
HarnessRuntimeRequest,
@@ -31,6 +34,13 @@ import type {
PendingRuntimeRequest,
} from "./codex-driver-types.js";
import { canonicalJson, record } from "./codex-driver-values.js";
import { NormalizedDeliveryWriter } from "../../control-plane/normalized-delivery-writer.js";
import type { CodexRpcServerRequest } from "./app-server-transport.js";
import { createCodexQuestionResponseContext, normalizeCodexQuestionSet } from "./codex-question-adapter.js";
import { CodexHistoryMap, CodexHistorySet } from "./codex-history-cache.js";
const durableScalars = ["sourceSequence", "sourceSequenceEpoch", "activeTurnId", "usageSnapshot", "codexUsageBaseline", "result", "resultFingerprint", "resultCallId", "resultTurnId", "protocolFailed", "protocolFailureCode", "protocolFailureMessage", "terminal", "dispositionOnlyRecoveryAvailable", "dispositionOnlyRecoveryConsumed", "dispositionOnlyRecoveryTurnId", "turnStarted", "turnStartPending", "notificationIdentityDiagnostics", "currentGoal", "interruptQueued", "steerSequence", "interruptSequence", "durableTerminal"] as const;
const durableMaps = ["terminalTurns", "workspaceChangesByTurn", "itemChannels", "lineageByThread", "acknowledgedSteeringCorrelations"] as const;
class AsyncQueue<T> implements AsyncIterable<T> {
#values: T[] = [];
@@ -68,6 +78,10 @@ class AsyncQueue<T> implements AsyncIterable<T> {
this.#values = [];
}
discard(predicate: (value: T) => boolean): void {
this.#values = this.#values.filter((value) => !predicate(value));
}
[Symbol.asyncIterator](): AsyncIterator<T> {
return {
next: async () => {
@@ -105,7 +119,13 @@ export class CodexSessionState {
readonly completionFeedback: CodexAppServerDriverOptions["completionFeedback"];
readonly dynamicToolHandler: CodexAppServerDriverOptions["dynamicToolHandler"];
readonly eventQueue = new AsyncQueue<PrpEvent>();
deliveryWriter: NormalizedDeliveryWriter | null = null;
deliveryRestored = false;
durableTerminal: { event: PrpEvent; semanticFingerprint: string | null } | null = null;
readonly pendingRuntimeRequestSources = new Map<string, CodexRpcServerRequest>();
sourceSequence: number;
sourceSequenceEpoch: string | null = null;
readonly capturedEventIds = new AsyncLocalStorage<string[]>();
activeTurnId: string | null;
usageSnapshot: Record<string, unknown> | null = null;
codexUsageBaseline: CodexUsageBaseline | null = null;
@@ -131,20 +151,20 @@ export class CodexSessionState {
dispositionOnlyRecoveryConsumed = false;
dispositionOnlyRecoveryTurnId: string | null = null;
turnStarted = false;
readonly terminalTurns = new Map<string, string>();
terminalTurns: Map<string, string> = new Map();
readonly workspaceChangesByTurn = new Map<string, Record<string, unknown>>();
readonly emittedFileReferences = new Set<string>();
emittedFileReferences: Set<string> | CodexHistorySet = new Set();
readonly itemChannels = new Map<
string,
"progress" | "final" | "summary" | "detail" | "unknown"
>();
readonly pendingRuntimeRequestMap = new Map<string, PendingRuntimeRequest>();
notificationIdentityDiagnostics = 0;
readonly lineageByThread = new Map<string, HarnessThreadLineageEntry>();
lineageByThread: Map<string, HarnessThreadLineageEntry> = new Map();
currentGoal: HarnessThreadGoal | null = null;
interruptQueued = false;
steerSequence = 0;
readonly acknowledgedSteeringCorrelations = new Map<string, string>();
acknowledgedSteeringCorrelations: Map<string, string> = new Map();
interruptSequence = 0;
constructor(input: {
@@ -165,6 +185,7 @@ export class CodexSessionState {
lineage?: HarnessThreadLineageEntry[];
goal?: HarnessThreadGoal | null;
sourceSequence: number;
sourceEpoch?: string;
now: () => Date;
runnerInstanceId: string;
driverKind: string;
@@ -189,6 +210,7 @@ export class CodexSessionState {
this.conversationMode = input.conversationMode;
this.activeTurnId = input.activeTurnId ?? null;
this.sourceSequence = input.sourceSequence;
this.sourceSequenceEpoch = input.sourceEpoch ?? null;
this.now = input.now;
this.runnerInstanceId = input.runnerInstanceId;
this.driverKind = input.driverKind;
@@ -279,6 +301,96 @@ export class CodexSessionState {
this.dispositionOnlyRecoveryConsumed =
dispositionOnlyRecoveryPreviouslyConsumed;
this.dispositionOnlyRecoveryTurnId = dispositionOnlyRecoveryTurnId;
this.initializeDelivery();
}
initializeDelivery(): void {
const port = this.transport.normalizedDelivery?.();
if (!port) return;
const saved = port.load();
if (saved) {
const state = saved.driver;
if (state.schema !== "paperclip.codex.reducer.v1" || state.runId !== this.runId || state.sessionId !== this.normalizedSessionId || state.threadId !== this.opened.threadId) throw new HarnessReconciliationError("normalized reducer binding mismatch");
const scalars = record(state.scalars);
for (const key of durableScalars) {
if (key === "sourceSequenceEpoch" && !(key in scalars)) continue; // Retained pre-epoch reducer.
if (!(key in scalars)) throw new HarnessReconciliationError(`normalized reducer lacks ${key}`);
(this as unknown as Record<string, unknown>)[key] = structuredClone(scalars[key]);
}
if (this.sourceSequence !== saved.produced || (this.sourceSequenceEpoch ?? undefined) !== saved.producedEpoch) throw new HarnessReconciliationError("normalized reducer cursor mismatch");
for (const key of durableMaps) {
const entries = state[key];
if (!Array.isArray(entries)) throw new HarnessReconciliationError(`normalized reducer lacks ${key}`);
const target = this[key] as Map<string, unknown>;
target.clear();
for (const [id, value] of entries) target.set(id, structuredClone(value));
}
this.emittedFileReferences.clear();
if (!Array.isArray(state.emittedFileReferences) || !Array.isArray(state.pendingRequests)) throw new HarnessReconciliationError("invalid normalized reducer collections");
for (const id of state.emittedFileReferences) this.emittedFileReferences.add(id);
for (const pending of state.pendingRequests) {
const request = pending.request as HarnessRuntimeRequest;
const source = pending.source as CodexRpcServerRequest;
if (!source || String(source.id) !== request.requestId || source.method !== request.method) throw new HarnessReconciliationError("normalized runtime request binding mismatch");
const responseContext = createCodexQuestionResponseContext();
normalizeCodexQuestionSet(source.method, source.params, responseContext);
this.pendingRuntimeRequestSources.set(request.requestId, source);
// Runnerd owns resolution delivery; the vanished controller's JS
// promise grants no authority and needs no reconstructed callback.
this.pendingRuntimeRequestMap.set(request.requestId, { request, responseContext, settle: () => {} });
}
this.deliveryRestored = true;
}
if (port.history) {
this.lineageByThread = new CodexHistoryMap("lineage", port.history, this.lineageByThread, saved !== null || this.lineageByThread instanceof CodexHistoryMap,
({ threadId, providerSessionId, parentThreadId, depth }) => ({ threadId, providerSessionId, parentThreadId, depth }),
(value) => ({ ...value as Pick<HarnessThreadLineageEntry, "threadId" | "providerSessionId" | "parentThreadId" | "depth">, nickname: null, role: null, status: "unknown" }));
this.terminalTurns = new CodexHistoryMap("terminal", port.history, this.terminalTurns, saved !== null);
this.acknowledgedSteeringCorrelations = new CodexHistoryMap("steering", port.history, this.acknowledgedSteeringCorrelations, saved !== null || this.acknowledgedSteeringCorrelations instanceof CodexHistoryMap);
this.emittedFileReferences = new CodexHistorySet(port.history, this.emittedFileReferences, saved !== null || this.emittedFileReferences instanceof CodexHistorySet);
}
this.deliveryWriter = new NormalizedDeliveryWriter(port, () => this.deliveryCheckpoint(), (event) => this.eventQueue.push(event), (error) => {
this.eventQueue.fail(error);
void this.transport.detachControllerForRestart?.().catch(() => undefined);
}, () => [
...(this.lineageByThread instanceof CodexHistoryMap ? this.lineageByThread.takeReceipts() : []),
...(this.terminalTurns instanceof CodexHistoryMap ? this.terminalTurns.takeReceipts() : []),
...(this.acknowledgedSteeringCorrelations instanceof CodexHistoryMap ? this.acknowledgedSteeringCorrelations.takeReceipts() : []),
...(this.emittedFileReferences instanceof CodexHistorySet ? this.emittedFileReferences.map.takeReceipts() : []),
]);
}
async prefetchHistory(collection: "terminal" | "file" | "steering" | "lineage", keys: string[]): Promise<void> {
const cache = collection === "lineage" ? this.lineageByThread : collection === "terminal" ? this.terminalTurns : collection === "steering" ? this.acknowledgedSteeringCorrelations
: this.emittedFileReferences instanceof CodexHistorySet ? this.emittedFileReferences.map : null;
if (!(cache instanceof CodexHistoryMap)) return;
await this.deliveryWriter?.flush();
await cache.prefetch(keys);
}
deliveryCheckpoint(): Record<string, unknown> {
for (const id of this.pendingRuntimeRequestSources.keys()) if (!this.pendingRuntimeRequestMap.has(id)) this.pendingRuntimeRequestSources.delete(id);
return JSON.parse(JSON.stringify({
schema: "paperclip.codex.reducer.v1", runId: this.runId, sessionId: this.normalizedSessionId, threadId: this.opened.threadId,
scalars: Object.fromEntries(durableScalars.map((key) => [key, this[key]])),
...Object.fromEntries(durableMaps.map((key) => [key, [...this[key]]])),
emittedFileReferences: [...this.emittedFileReferences],
pendingRequests: [...this.pendingRuntimeRequestMap].map(([id, pending]) => ({ request: pending.request, source: this.pendingRuntimeRequestSources.get(id) })),
})) as Record<string, unknown>;
}
publishEvent(event: PrpEvent): void {
this.capturedEventIds.getStore()?.push(event.sourceEventId);
if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) this.durableTerminal = {
event: JSON.parse(JSON.stringify(event)) as PrpEvent, semanticFingerprint: this.resultFingerprint,
};
if (this.deliveryWriter) this.deliveryWriter.emit(event);
else this.eventQueue.push(event);
}
closeEvents(): void {
if (!this.deliveryWriter) this.eventQueue.close();
else void Promise.resolve().then(() => this.deliveryWriter!.flush()).then(() => this.eventQueue.close(), (error) => this.eventQueue.fail(error));
}
requireActiveTurn(turnId: string, operation: string): void {
@@ -417,7 +529,7 @@ export class CodexSessionState {
this.activeTurnId = null;
}
this.terminal = true;
this.eventQueue.close();
this.closeEvents();
// Notification failure can initiate cleanup before the owning runtime joins
// it. Observe this background rejection immediately so a deleted remote
// sandbox cannot crash the controller. The transport retains its original
@@ -426,16 +538,30 @@ export class CodexSessionState {
void this.transport.close(`protocol_failure:${code}`).catch(() => undefined);
}
private allocateSource(): Pick<PrpEvent, "sourceEventId" | "sourceSeq" | "sourceEpoch" | "sourceEpochTransition"> {
const limit = this.deliveryWriter?.port.eventEpochs?.limit;
let transition: EventEpochTransition | undefined;
if (limit !== undefined && this.sourceSequence >= limit) {
transition = { schema: "paperclip.prp.event-epoch.v1", runId: this.runId, transitionId: randomUUID(),
fromEpoch: this.sourceSequenceEpoch, nextEpoch: randomUUID(), finalOrdinal: this.sourceSequence };
this.sourceSequenceEpoch = transition.nextEpoch;
this.sourceSequence = 0;
}
if (!Number.isSafeInteger(this.sourceSequence) || this.sourceSequence >= Number.MAX_SAFE_INTEGER) throw new HarnessReconciliationError("normalized source sequence requires epoch support");
const sourceSeq = ++this.sourceSequence;
return { sourceEventId: normalizedEventId(this.runnerInstanceId, this.runId, sourceSeq, this.sourceSequenceEpoch ?? undefined), sourceSeq,
...(this.sourceSequenceEpoch ? { sourceEpoch: this.sourceSequenceEpoch } : {}), ...(transition ? { sourceEpochTransition: transition } : {}) };
}
emit(
eventType: PrpEvent["eventType"],
payload: Record<string, unknown>,
refs: { turnId?: string; itemId?: string } = {},
): void {
const sourceSeq = ++this.sourceSequence;
this.eventQueue.push({
const source = this.allocateSource();
this.publishEvent({
schema: "paperclip.prp.event.v1",
sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`,
sourceSeq,
...source,
sourceInstanceId: this.runnerInstanceId,
sourceKind: "runner",
runId: this.runId,
@@ -506,11 +632,10 @@ export class CodexSessionState {
| "session.goal.cleared",
payload: Record<string, unknown>,
): void {
const sourceSeq = ++this.sourceSequence;
this.eventQueue.push({
const source = this.allocateSource();
this.publishEvent({
schema: "paperclip.prp.event.v2",
sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`,
sourceSeq,
...source,
sourceInstanceId: this.runnerInstanceId,
sourceKind: "runner",
runId: this.runId,
@@ -158,6 +158,8 @@ export function mapTerminalTurn(
: "turn.completed";
state.cancelPendingRequests("turn_terminal");
state.activeTurnId = null;
state.workspaceChangesByTurn.delete(turnId);
state.itemChannels.clear();
state.turnStarted = false;
state.emit(
eventType,
@@ -186,14 +186,12 @@ describe("runner semantic MCP bridge", () => {
expect(handler).toHaveBeenCalledTimes(2);
});
it("times out calls and honors MCP cancellation", async () => {
const starts: string[] = [];
it("times out calls", async () => {
const bridge = await startRunnerToolBridge({
tools: [tool("documents.read")],
timeoutMs: 20,
handler: ({ callId, signal }) =>
handler: ({ signal }) =>
new Promise((_resolve, reject) => {
starts.push(callId);
signal.addEventListener(
"abort",
() => reject(new Error("handler aborted")),
@@ -217,29 +215,56 @@ describe("runner semantic MCP bridge", () => {
content: [{ text: "Paperclip tool call timed out" }],
},
});
});
it("honors MCP cancellation", async () => {
const starts: string[] = [];
const bridge = await startRunnerToolBridge({
tools: [tool("documents.read")],
timeoutMs: 5_000,
handler: ({ callId, signal }) =>
new Promise((_resolve, reject) => {
starts.push(callId);
signal.addEventListener(
"abort",
() => reject(new Error("handler aborted")),
{ once: true },
);
}),
});
bridges.push(bridge);
const pending = rpc(bridge, {
id: "cancel-me",
method: "tools/call",
params: { name: "documents.read", arguments: {} },
});
await vi.waitFor(() => expect(starts).toContain("cancel-me"), {
interval: 1,
timeout: 15,
});
expect(
(
await rpc(bridge, {
method: "notifications/cancelled",
params: { requestId: "cancel-me" },
})
).status,
).toBe(202);
expect(await (await pending).json()).toMatchObject({
result: {
isError: true,
content: [{ text: "Paperclip tool call cancelled" }],
},
});
// Attach a rejection handler immediately so a failed start gate cannot
// leave an unhandled loopback fetch while teardown closes the server.
void pending.catch(() => undefined);
try {
await vi.waitFor(() => expect(starts).toContain("cancel-me"), {
interval: 10,
timeout: 2_000,
});
expect(
(
await rpc(bridge, {
method: "notifications/cancelled",
params: { requestId: "cancel-me" },
})
).status,
).toBe(202);
expect(await (await pending).json()).toMatchObject({
result: {
isError: true,
content: [{ text: "Paperclip tool call cancelled" }],
},
});
} finally {
await bridge.close();
await pending.catch(() => undefined);
}
});
it("preserves successful mutation identity when the result is oversized", async () => {
+18
View File
@@ -29,6 +29,8 @@ export {
DurablePrpControlPlane,
DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES,
inspectWarmRunTransition,
stageLegacyControlPlaneAuthority,
type LegacyAuthorityImportOptions,
type DurablePrpControlPlaneOptions,
type PrpWireConnection,
type PrpWireAttachment,
@@ -78,3 +80,19 @@ export * from "./generated/capability-contract.js";
export * from "./semantic-tools/index.js";
export * as acceptedCapabilitySemanticTools from "./semantic-tools/index.js";
export * from "./compatibility.js";
export * from "./control-plane/durable-authority-store.js";
export * from "./control-plane/current-authority-evidence.js";
export { SqliteAuthorityStore } from "./control-plane/sqlite-authority-store.js";
export { readDurableControlPlaneState, parseAuthorityLocator, type AuthorityLocation, type AuthorityLocator, type ExternalAuthorityReader } from "./control-plane/authority-locator.js";
export { readIndexedLocalState, suspendIndexedRunnerState } from "./control-plane/indexed-local-state-reader.js";
export type { IndexedLocalSnapshot } from "./control-plane/indexed-local-state-reader.js";
export { IndexedInspectionDecoder } from "./control-plane/indexed-inspection-codec.js";
export { migrateLegacySessionAuthority, type LegacySessionMigrationOptions, type PreparedLocalSession } from "./control-plane/legacy-authority-activation.js";
export { assertNoPendingLegacyMigration } from "./control-plane/legacy-migration-gate.js";
export { advanceSourceCursor, normalizedEventId, isEventEpochTransition } from "./control-plane/event-epochs.js";
export type { SourceCursor, EventEpochTransition } from "./control-plane/event-epochs.js";
export { isProviderProcessGeneration, provesProviderGenerationTransition } from "./control-plane/process-generation.js";
@@ -0,0 +1,37 @@
import { resolve } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
afterEach(() => vi.unstubAllEnvs());
describe("isolated Cargo test binary resolution", () => {
const workspace = resolve("/tmp/paperclip-runner-workspace");
const executable = (name: string) => `${name}${process.platform === "win32" ? ".exe" : ""}`;
it("preserves the normal target path when CARGO_TARGET_DIR is unset", () => {
vi.stubEnv("CARGO_TARGET_DIR", "");
expect(resolveRunnerCargoTestBinary(workspace, "debug", "fake-codex-app-server"))
.toBe(resolve(workspace, "target", "debug", executable("fake-codex-app-server")));
});
it("resolves absolute and workspace-relative isolated target directories", () => {
vi.stubEnv("CARGO_TARGET_DIR", "/tmp/isolated-cargo-target");
expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd"))
.toBe(resolve("/tmp/isolated-cargo-target", "release", executable("paperclip-runnerd")));
vi.stubEnv("CARGO_TARGET_DIR", "target/history-development");
expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd"))
.toBe(resolve(workspace, "target/history-development", "release", executable("paperclip-runnerd")));
});
it("uses staged/default resolution only when isolation is not configured", () => {
const fallback = vi.fn(() => "/staged/paperclip-runnerd");
vi.stubEnv("CARGO_TARGET_DIR", "");
expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback))
.toBe("/staged/paperclip-runnerd");
expect(fallback).toHaveBeenCalledOnce();
vi.stubEnv("CARGO_TARGET_DIR", "target/isolated");
expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback))
.toBe(resolve(workspace, "target/isolated", "debug", executable("paperclip-runnerd")));
expect(fallback).toHaveBeenCalledOnce();
});
});
@@ -0,0 +1,109 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { createHash } from "node:crypto";
import { mkdtemp, rename, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { expect, it } from "vitest";
import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
const seconds = Number(process.env.PAPERCLIP_ACTIVE_HISTORY_SOAK_SECONDS ?? 0);
// The short lane checks the fixture; only 259200 seconds is a 72-hour result.
it.skipIf(![60, 259200].includes(seconds))("keeps generating and durably consuming output during one active turn", async () => {
const root = await mkdtemp(join(tmpdir(), "indexed-active-history-"));
const identity = { runnerInstanceId: "active-history-runner", environmentLeaseId: "active-history-lease", normalizedSessionId: "active-history-session",
runId: "active-history-run", turnId: "active-history-turn", itemId: "active-history-item" };
let authority: SqliteAuthorityStore | undefined;
const sink = await SqliteAuthorityStore.open({ path: join(root, "normalized.sqlite"), binding: "active-history-sink", create: true });
const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--soak-output-on-steer", "--require-lightweight-history"],
stateDirectory: root, prpIdentity: identity, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "active-history", create: true }),
});
const driver = new CodexAppServerDriver({ taskEnvelope: { schema: "paperclip.skillless_task.v1", objective: "Qualify a long active turn with growing output.",
completionContract: { revision: "active-history-v1", criteria: [{ id: "history", requirement: "Durably deliver ordinary output." }] }, constraints: [], expectedResultSchema: "paperclip.run_result.v1" },
runnerInstanceId: identity.runnerInstanceId, approvalPolicy: "never", includeCollaborationModeInstructions: false,
environment: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: "/isolated/home", CODEX_HOME: "/isolated/codex-home", LANG: "C.UTF-8" },
transportFactory: () => bundle.transport, requireProviderSessionIdentity: true });
let session: Awaited<ReturnType<typeof driver.openSession>> | undefined;
let generation = "0", events = 0, items = 0, bytes = 0, peakCurrentBytes = 0, probes = 0, maxProbeMs = 0;
let body: { id: string; offset: number; digest: ReturnType<typeof createHash> } | null = null;
let consumer: Promise<void> | undefined, failure: unknown, closing = false;
let started = performance.now(), startedAt = new Date().toISOString();
const report = async (status: string, error?: unknown) => {
const target = process.env.PAPERCLIP_HISTORY_REPORT;
if (!target) return;
await writeFile(`${target}.tmp`, JSON.stringify({ schema: "paperclip.indexed-active-history.v1", status,
provider: "credential-free fake Codex; actual driver and durable event sink", seconds, startedAt,
updatedAt: new Date().toISOString(), elapsedMs: performance.now() - started, events, items, bytes, probes,
peakCurrentBytes, maxProbeMs, runnerPid: bundle.evidence().runnerPid, providerPid: bundle.evidence().providerPid,
...(error ? { error: error instanceof Error ? error.message : String(error) } : {}) }, null, 2), { mode: 0o600 });
await rename(`${target}.tmp`, target);
};
try {
session = await driver.openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: root });
const commit = async (event: import("../protocol/replay-contract.js").PrpEvent) => {
generation = await sink.commit({ expectedGeneration: generation, state: { lastSourceSeq: event.sourceSeq },
records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record<string, unknown> }] });
};
session.setEventCommitter!(commit);
consumer = (async () => {
for await (const event of session!.events()) {
await commit(event); await session!.acknowledgeEvent!(event); events++;
if (event.eventType === "output.body.chunk") {
const ref = event.payload.body as Record<string, unknown>, text = String(event.payload.text), offset = Number(event.payload.offset);
if (offset === 0) { expect(body).toBeNull(); body = { id: String(ref.bodyId), offset: 0, digest: createHash("sha256") }; }
expect(body?.id).toBe(ref.bodyId); expect(body?.offset).toBe(offset);
expect(createHash("sha256").update(text).digest("hex")).toBe(event.payload.sha256);
body!.digest.update(text); body!.offset += Buffer.byteLength(text); bytes += Buffer.byteLength(text);
}
if (event.eventType === "item.completed" && event.itemId?.startsWith("soak-output-")) {
const ref = event.payload.outputBody as Record<string, unknown>;
expect(body?.id).toBe(ref.bodyId); expect(String(body!.offset)).toBe(ref.byteLength);
expect(body!.digest.digest("hex")).toBe(ref.sha256);
body = null; items++;
}
if (!closing && ["turn.completed", "turn.failed", "session.failed"].includes(event.eventType)) throw new Error(`active soak ended early: ${event.eventType}`);
}
if (!closing) throw new Error("active soak event stream closed early");
})().catch(error => { failure = error; });
const turn = await session.startTurn({ message: { text: "Keep this turn active while generating qualification output." } });
const owner = bundle.evidence();
started = performance.now(); startedAt = new Date().toISOString();
await report("running");
while (performance.now() - started < seconds * 1000) {
if (failure) throw failure;
const before = performance.now(), previous = items;
await session.steer!({ turnId: turn.turnId, message: { text: `Produce qualification output ${probes + 1}.` }, correlationId: `soak-steer-${probes + 1}` });
await bundle.transport.request("thread/read", { threadId: session.ids().driverSessionId, includeTurns: false });
const deadline = performance.now() + 30_000;
while (items === previous && !failure && performance.now() < deadline) await new Promise(resolve => setTimeout(resolve, 25));
if (failure) throw failure;
expect(items).toBeGreaterThan(previous);
maxProbeMs = Math.max(maxProbeMs, performance.now() - before); probes++;
expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, providerPid: owner.providerPid, runnerExited: false });
peakCurrentBytes = Math.max(peakCurrentBytes, Buffer.byteLength(JSON.stringify((await authority!.load())!.state)));
expect(peakCurrentBytes).toBeLessThan(8 * 1024 * 1024);
await report("running");
await new Promise(resolve => setTimeout(resolve, Math.min(seconds === 60 ? 1000 : 30_000, Math.max(0, seconds * 1000 - (performance.now() - started)))));
}
closing = true; await report("settling");
await session.close({ reason: "active history qualification complete" }); await consumer; session = undefined;
if (failure) throw failure;
await bundle.transport.close(); await authority!.close(); authority = undefined; await sink.close();
expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
await rm(root, { recursive: true, force: true });
await report("passed");
} catch (error) { await report("failed", error); throw error; }
finally { closing = true; await session?.close({ reason: "active history qualification cleanup" }); await consumer;
await bundle.transport.close(); await authority?.close(); await sink.close(); await rm(root, { recursive: true, force: true }); }
}, (seconds + 3600) * 1000);
@@ -0,0 +1,147 @@
import { nativeStorageCommand } from "../control-plane/native-storage-command.js";
import { authorityJson } from "../control-plane/durable-authority-store.js";
import { createHash } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, mkdtempSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { expect, it } from "vitest";
import { finishIndexedRunnerArchive, prepareIndexedRunnerArchive, publishIndexedArchiveHead, readIndexedArchiveHead } from "./indexed-authority-archive.js";
import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js";
import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { runnerdRecoveryInternals } from "./runnerd-codex-transport.js";
it("reopens the exact indexed controller after cold rotation without reading older epochs", async () => {
const root = mkdtempSync(resolve(tmpdir(), "indexed-cold-rotation-"));
const controller = resolve(root, "control-plane"), runner = resolve(root, "runner");
mkdirSync(controller, { mode: 0o700 }); mkdirSync(runner, { mode: 0o700 });
const identity = { runnerInstanceId: "runner-test", environmentLeaseId: "environment-test", normalizedSessionId: "session-test", runId: "run-old", turnId: "turn-old", itemId: "item-old" };
const desired = { ...identity, runId: "run-new", turnId: "turn-new", itemId: "item-new" };
const store = await SqliteAuthorityStore.open({ path: resolve(controller, "authority.sqlite"), binding: JSON.stringify(identity), create: true });
try {
const core = await DurablePrpControlPlane.open({ stateDirectory: controller, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: store });
const state = structuredClone(core.store.state) as unknown as Record<string, unknown>;
await store.close();
const path = resolve(runner, "runner-state.sqlite");
const db = new DatabaseSync(path);
try {
chmodSync(path, 0o600);
writeFileSync(`${path}.lifetime`, "", { mode: 0o600 });
db.exec("CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB)");
db.prepare("INSERT INTO store_binding VALUES (1,1,'runner/session')").run();
const bytes = Buffer.from(JSON.stringify({ ...identity, schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended", nextSourceSeq: 1, ackedSourceSeq: 0 }));
db.prepare("INSERT INTO current_state VALUES ('runner',12,?,?)").run(bytes, createHash("sha256").update(bytes).digest());
} finally { db.close(); }
writeFileSync(resolve(runner, "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/session" }), { mode: 0o600 });
expect(await runnerdRecoveryInternals.rotateLocalAuthorityEpoch(root, state, desired)).toEqual(state);
const head = readIndexedArchiveHead(resolve(root, "authority-epochs"));
expect(head).not.toBeNull();
// The legacy scan would select this newer, invalid archive first.
const unrelated = resolve(root, "authority-epochs", "unrelated", "control-plane");
mkdirSync(unrelated, { recursive: true, mode: 0o700 });
writeFileSync(resolve(unrelated, "control-plane-state.json"), "not valid authority", { mode: 0o600 });
expect(await runnerdRecoveryInternals.latestArchivedControlPlaneState(root, desired)).toEqual(state);
await expect(runnerdRecoveryInternals.latestArchivedControlPlaneState(root, { ...desired, normalizedSessionId: "wrong-session" })).rejects.toThrow("archive_conflict");
expect(await runnerdRecoveryInternals.rotateLocalAuthorityEpoch(root, state, desired)).toEqual(state);
} finally { await store.close(); rmSync(root, { recursive: true, force: true }); }
});
it("routes interrupted cold rotation directly despite unavailable old archives", () => {
const root = mkdtempSync(resolve(tmpdir(), "indexed-archive-head-"));
const archive = resolve(root, `epoch-${"a".repeat(24)}`);
mkdirSync(archive, { mode: 0o700 });
try {
expect(readIndexedArchiveHead(root)).toBeNull();
publishIndexedArchiveHead(archive);
// Unrelated retained epochs must never be enumerated or read to resume.
for (let index = 0; index < 128; index++) {
symlinkSync(resolve(root, "unavailable-volume", String(index)), resolve(root, `old-${index}`));
}
expect(readIndexedArchiveHead(root)).toBe(archive);
const next = resolve(root, `epoch-${"b".repeat(24)}`);
mkdirSync(next, { mode: 0o700 });
publishIndexedArchiveHead(next);
rmSync(archive, { recursive: true });
expect(readIndexedArchiveHead(root)).toBe(next);
rmSync(next, { recursive: true });
expect(() => readIndexedArchiveHead(root)).toThrow();
// Never treat a broken pointer as legacy state and fall back to old epochs.
rmSync(resolve(root, "current-archive.json"));
symlinkSync(resolve(root, "missing"), resolve(root, "current-archive.json"));
expect(() => readIndexedArchiveHead(root)).toThrow("archive_unsafe");
} finally { rmSync(root, { recursive: true, force: true }); }
});
it.each([
[1, "12"], [2, "12"], [2, "9223372036854775807"],
[2, "r:59365b32-c44d-46f2-8550-a7b5bba1bd68"],
] as const)("recovers an interrupted schema %s SQLite archive at revision %s without scanning or copying its history", async (schema, generation) => {
const root = mkdtempSync(resolve(tmpdir(), "indexed-archive-"));
const runner = resolve(root, "runner"), archive = resolve(root, "archive");
mkdirSync(runner, { mode: 0o700 }); mkdirSync(archive, { mode: 0o700 });
mkdirSync(resolve(root, "control-plane"), { mode: 0o700 });
const path = resolve(runner, "runner-state.sqlite");
const db = new DatabaseSync(path);
chmodSync(path, 0o600);
writeFileSync(`${path}.lifetime`, "", { mode: 0o600 });
db.exec("CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB); CREATE TABLE receipts(key TEXT PRIMARY KEY,bytes BLOB)");
db.prepare("INSERT INTO store_binding VALUES (1,?,'runner/session')").run(schema);
if (schema === 2) db.exec("CREATE TABLE receipt_prepare(singleton INTEGER PRIMARY KEY,state_key TEXT,generation INTEGER,body BLOB,digest BLOB)");
const bytes = Buffer.from(JSON.stringify({ lifecycle: "suspended", nextSourceSeq: 2, ackedSourceSeq: 1 }));
db.prepare("INSERT INTO current_state VALUES ('runner',?,?,?)").run(generation.startsWith("r:") ? generation : BigInt(generation), bytes, createHash("sha256").update(bytes).digest());
db.prepare("INSERT INTO receipts VALUES ('ancient',?)").run(Buffer.from("original outcome"));
db.close();
if (schema === 2) {
const shards = resolve(runner, "runner-state.sqlite.receipts");
mkdirSync(shards, { mode: 0o700 });
writeFileSync(resolve(shards, "partition-fixture"), "immutable receipt evidence", { mode: 0o600 });
}
const routing = resolve(runner, "runner-state.sqlite.routing");
mkdirSync(routing, { mode: 0o700 });
writeFileSync(resolve(routing, "routing-fixture"), "immutable routing evidence", { mode: 0o600 });
writeFileSync(resolve(runner, "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/session" }), { mode: 0o600 });
try {
await prepareIndexedRunnerArchive(runner, archive);
await expect(finishIndexedRunnerArchive(runner, archive)).rejects.toThrow("controller_missing");
renameSync(resolve(root, "control-plane"), resolve(archive, "control-plane"));
// Simulate loss of the controller after the DB rename and before locator publication.
renameSync(path, resolve(archive, "runner-state.sqlite"));
expect(await finishIndexedRunnerArchive(runner, archive)).toBe(true);
expect(existsSync(resolve(runner, "runner-state.json"))).toBe(false);
expect(existsSync(resolve(archive, "runner-state.sqlite.routing", "routing-fixture"))).toBe(true);
expect(existsSync(resolve(runner, "runner-state.sqlite.routing"))).toBe(false);
if (schema === 2) {
expect(existsSync(resolve(archive, "runner-state.sqlite.receipts", "partition-fixture"))).toBe(true);
expect(existsSync(resolve(runner, "runner-state.sqlite.receipts"))).toBe(false);
}
expect(await finishIndexedRunnerArchive(runner, archive)).toBe(true);
expect(await readIndexedLocalState(resolve(archive, "runner-state.json"))).toMatchObject({ generation, state: { lifecycle: "suspended", outbox: [] } });
const restored = new DatabaseSync(resolve(archive, "runner-state.sqlite"), { readOnly: true });
try { expect(Buffer.from(restored.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome"); }
finally { restored.close(); }
} finally { rmSync(root, { recursive: true, force: true }); }
});
it("fences standalone controller archive against an open storage owner and stale authority", async () => {
const root = mkdtempSync(resolve(tmpdir(), "indexed-controller-archive-"));
const controller = resolve(root, "controller"), target = resolve(root, "archived");
mkdirSync(controller, { mode: 0o700 });
const identity = { runnerInstanceId: "runner-test", environmentLeaseId: "environment-test", normalizedSessionId: "session-test", runId: "run-old", turnId: "turn-old", itemId: "item-old" };
const store = await SqliteAuthorityStore.open({ path: resolve(controller, "authority.sqlite"), binding: JSON.stringify(identity), create: true });
try {
const core = await DurablePrpControlPlane.open({ stateDirectory: controller, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: store });
const storedState = (await store.load())!.state;
const digest = createHash("sha256").update(authorityJson(storedState)).digest("hex");
const command = ["archive-controller", "--directory", controller, "--destination", target, "--digest", digest];
await expect(nativeStorageCommand(command)).rejects.toThrow("open connection");
await store.close();
await expect(nativeStorageCommand([...command.slice(0, -1), "0".repeat(64)])).rejects.toThrow("authority changed");
expect(existsSync(controller)).toBe(true);
await nativeStorageCommand(command);
expect(existsSync(controller)).toBe(false);
expect(existsSync(resolve(target, "authority.sqlite"))).toBe(true);
const reopened = await SqliteAuthorityStore.open({ path: resolve(target, "authority.sqlite"), binding: JSON.stringify(identity), create: false });
try { expect((await reopened.load())?.state).toEqual(storedState); } finally { await reopened.close(); }
} finally { await store.close(); rmSync(root, { recursive: true, force: true }); }
});
@@ -0,0 +1,117 @@
import { randomUUID } from "node:crypto";
import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, openSync, readSync, renameSync, writeFileSync } from "node:fs";
import { basename, dirname, resolve } from "node:path";
import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js";
import { nativeStorageCommand } from "../control-plane/native-storage-command.js";
import { authorityGeneration } from "../control-plane/durable-authority-store.js";
const files = ["runner-state.sqlite", "runner-state.sqlite-wal", "runner-state.sqlite-shm", "runner-state.sqlite.receipts", "runner-state.sqlite.routing", "runner-state.json"];
type Transfer = { schema: "paperclip.runner.indexed-archive.v1"; generation: string; stateSha256: string; files: string[] };
function privateDirectory(path: string): void {
const stat = lstatSync(path);
if (!stat.isDirectory() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_indexed_archive_unsafe");
}
export function syncArchiveDirectory(path: string): void {
if (process.platform === "win32") return;
const fd = openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
try { fsyncSync(fd); } finally { closeSync(fd); }
}
function privateRegular(path: string): void {
const stat = lstatSync(path);
if (!stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_indexed_archive_unsafe");
}
/** Publish before moving the active controller directory. This is a bounded
* routing hint; the caller must still validate the archived authority binding. */
export function publishIndexedArchiveHead(archive: string): void {
const root = dirname(archive);
privateDirectory(root);
privateDirectory(archive);
const name = basename(archive);
if (!/^epoch-[a-f0-9]{24}$/.test(name)) throw new Error("native_indexed_archive_invalid");
const target = resolve(root, "current-archive.json");
if (existsSync(target)) privateRegular(target);
const temporary = `${target}.${randomUUID()}.tmp`;
writeFileSync(temporary, JSON.stringify({ schema: "paperclip.runner.indexed-archive-head.v1", archive: name }), { mode: 0o600, flag: "wx", flush: true });
renameSync(temporary, target);
syncArchiveDirectory(root);
}
/** Once a head exists, missing/corrupt current routing fails closed. It must
* never fall back to choosing an older archive by timestamp or scanning. */
export function readIndexedArchiveHead(archivesRoot: string): string | null {
const path = resolve(archivesRoot, "current-archive.json");
// lstat distinguishes an absent pointer from a dangling symlink.
try { lstatSync(path); } catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
}
privateDirectory(archivesRoot);
privateRegular(path);
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > 1024) throw new Error("native_indexed_archive_invalid");
const bytes = Buffer.alloc(stat.size);
let offset = 0;
while (offset < bytes.length) {
const count = readSync(fd, bytes, offset, bytes.length - offset, offset);
if (!count) throw new Error("native_indexed_archive_changed");
offset += count;
}
const value = JSON.parse(bytes.toString()) as { schema?: unknown; archive?: unknown };
if (value.schema !== "paperclip.runner.indexed-archive-head.v1" || typeof value.archive !== "string" || !/^epoch-[a-f0-9]{24}$/.test(value.archive)) throw new Error("native_indexed_archive_invalid");
const archive = resolve(archivesRoot, value.archive);
privateDirectory(archive);
return archive;
} finally { closeSync(fd); }
}
function readTransfer(archive: string): Transfer | null {
const path = resolve(archive, "runner-transfer.json");
if (!existsSync(path)) return null;
privateDirectory(archive);
privateRegular(path);
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > 4096) throw new Error("native_indexed_archive_unsafe");
const bytes = Buffer.alloc(stat.size);
let offset = 0;
while (offset < bytes.length) {
const count = readSync(fd, bytes, offset, bytes.length - offset, offset);
if (!count) throw new Error("native_indexed_archive_changed");
offset += count;
}
const value = JSON.parse(bytes.toString()) as Transfer;
if (typeof value.generation !== "string") throw new Error("native_indexed_archive_invalid");
try { authorityGeneration(value.generation); } catch { throw new Error("native_indexed_archive_invalid"); }
if (value.schema !== "paperclip.runner.indexed-archive.v1" || value.generation === "0" ||
!/^[a-f0-9]{64}$/.test(value.stateSha256) || !Array.isArray(value.files) ||
value.files.length < 2 || value.files.length > files.length || new Set(value.files).size !== value.files.length ||
!value.files.includes("runner-state.sqlite") || !value.files.includes("runner-state.json") || value.files.some(file => !files.includes(file))) throw new Error("native_indexed_archive_invalid");
return value;
} finally { closeSync(fd); }
}
/** Only called after the existing caller proves a suspended, exclusively owned
* runner. The intent survives a crash between renames of SQLite companion files. */
export async function prepareIndexedRunnerArchive(runner: string, archive: string, runnerBinary?: string): Promise<void> {
privateDirectory(runner);
privateDirectory(archive);
const snapshot = await readIndexedLocalState(resolve(runner, "runner-state.json"), { runnerBinary });
if (snapshot.state.lifecycle !== "suspended") throw new Error("native_indexed_archive_unsettled");
await nativeStorageCommand(["archive-prepare", "--directory", runner, "--destination", archive,
"--generation", snapshot.generation, "--digest", snapshot.stateDigest], runnerBinary);
}
/** The controller directory must already have moved durably. The native owner
* completes the fixed renames while holding both store lifetime fences. */
export async function finishIndexedRunnerArchive(runner: string, archive: string, runnerBinary?: string): Promise<boolean> {
if (!readTransfer(archive)) return false;
await nativeStorageCommand(["archive-finish", "--directory", runner, "--destination", archive], runnerBinary);
return true;
}
@@ -0,0 +1,76 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { expect, it, vi } from "vitest";
import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
// Real native transport/process/storage; the provider is credential-free.
// The small connection limit exercises the normal production re-auth path.
it.each([false, true])("rekeys repeatedly during one active provider turn (command epochs: %s)", async (commandEpochs) => {
const root = await mkdtemp(join(tmpdir(), "indexed-channel-rekey-"));
const callsPath = join(root, "calls.log");
let authority: SqliteAuthorityStore | undefined;
const cores: DurablePrpControlPlane[] = [];
const open = DurablePrpControlPlane.open.bind(DurablePrpControlPlane);
const spy = vi.spyOn(DurablePrpControlPlane, "open").mockImplementation(async options => {
const core = await open({ ...options, secureChannelFrameLimit: 16, ...(commandEpochs ? { commandEpochLimit: 8 } : {}) });
cores.push(core); return core;
});
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--record-process-start", "--call-log", callsPath, "--require-lightweight-history"],
stateDirectory: root, runnerReconnectGraceMs: 10_000,
authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "channel-rekey", create: true }),
});
let consumer: Promise<void> | undefined, failure: unknown;
try {
const opened = await bundle.transport.request("thread/start", { cwd: root, dynamicTools: [] });
consumer = (async () => {
for await (const notification of bundle.transport.notifications()) {
const port = bundle.transport.normalizedDelivery!()!;
if (notification.paperclipDelivery) await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: notification.paperclipDelivery,
driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [] });
}
})().catch(error => { failure = error; });
await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Keep the same turn active across encrypted connections." }] });
const owner = bundle.evidence(), core = cores[0]!, identity = structuredClone(core.store.state.identity);
const firstCommand = structuredClone(core.store.state.commands[0]!);
const epochs = new Set<string | undefined>();
const threadId = (opened.thread as { id: string }).id;
for (let index = 0; index < 64; index++) {
await bundle.transport.request("thread/read", { threadId, includeTurns: false });
if (failure) throw failure;
epochs.add(core.store.state.indexedState?.controllerEpoch);
expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false });
}
if (commandEpochs) {
expect(epochs.size).toBeGreaterThanOrEqual(7);
expect(await core.queueCommand(firstCommand.type, firstCommand.payload, firstCommand.commandId)).toMatchObject({ commandId: firstCommand.commandId, status: firstCommand.status });
}
expect(core.store.state.connectionCount).toBeGreaterThanOrEqual(4);
expect(core.store.state.identity).toEqual(identity);
expect(core.store.state.malformedFrames).toBe(0);
const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/);
expect(calls.filter(call => call === "process-start")).toHaveLength(1);
expect(calls.filter(call => call === "thread/start")).toHaveLength(1);
expect(calls.filter(call => call === "turn/start")).toHaveLength(1);
expect(calls).not.toContain("turn/interrupt");
expect(calls).not.toContain("thread/resume");
const first = (await authority!.readEvents(identity.runId, "0", 1, 1024 * 1024)).records[0]!;
expect(await authority!.getRecord(identity.runId, "event", first.id)).toEqual(first);
} finally {
await bundle.transport.close(); await consumer; await authority?.close(); spy.mockRestore();
await rm(root, { recursive: true, force: true });
}
}, 90_000);
@@ -0,0 +1,81 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { expect, it } from "vitest";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
// Explicit protocol scale qualification, with real runner/fake-provider
// processes and real storage. This does not stand in for Product E2E or a soak.
it.skipIf(process.env.PAPERCLIP_HISTORY_CONTINUATIONS !== "1000")("continues one provider through 1,000 indexed run epochs", async () => {
const directory = await mkdtemp(join(tmpdir(), "indexed-continuations-"));
let authority: SqliteAuthorityStore | undefined;
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(directory, "fake-state.json"), "--durable-turn-ids"],
stateDirectory: directory,
prpIdentity: { runnerInstanceId: "history-runner", environmentLeaseId: "history-lease", normalizedSessionId: "history-session", runId: "history-run-0", turnId: "history-turn-0", itemId: "history-item-0" },
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
authorityStoreFactory: async (_identity, root) => authority ??= await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: "history-continuations", create: true }),
});
bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] }));
let completed: (() => void) | null = null;
let consumerFailure: unknown;
let consumer: Promise<void> | undefined;
const started = Date.now();
const durations: number[] = [];
try {
await bundle.transport.request("initialize", {});
const opened = await bundle.transport.request("thread/start", {
cwd: directory,
dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }],
completionContract: { revision: "history-contract", criterionIds: ["objective"] },
});
const owner = bundle.evidence();
consumer = (async () => {
for await (const notification of bundle.transport.notifications()) {
const port = bundle.transport.normalizedDelivery!()!;
if (notification.paperclipDelivery) await port.commit({
expectedRevision: port.load()?.revision ?? 0,
raw: notification.paperclipDelivery,
driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [],
});
if (notification.method === "turn/completed") completed?.();
}
})().catch((error) => { consumerFailure = error; completed?.(); });
bundle.transport.setBeforeRunRotation!(async () => { if (consumerFailure) throw consumerFailure; });
for (let index = 0; index < 1000; index++) {
const before = Date.now();
if (index) await bundle.transport.attachRun!({ runId: `history-run-${index}`, turnId: `history-turn-${index}`, itemId: `history-item-${index}` });
let timer: ReturnType<typeof setTimeout> | undefined;
const terminal = new Promise<void>((resolve, reject) => { completed = resolve; timer = setTimeout(() => reject(new Error(`turn ${index} timed out`)), 20_000); });
try {
await bundle.transport.request("turn/start", { input: [{ type: "text", text: `Continue the same task: ${index}` }] });
await terminal;
if (consumerFailure) throw consumerFailure;
} finally { clearTimeout(timer); }
expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false });
const current = await authority!.load();
expect(Buffer.byteLength(JSON.stringify(current!.state))).toBeLessThan(256 * 1024);
durations.push(Date.now() - before);
}
const sorted = [...durations].sort((a, b) => a - b);
const report = { schema: "paperclip.indexed-continuations-qualification.v1", turns: durations.length, elapsedMs: Date.now() - started, turnP95Ms: sorted[949], first100MeanMs: durations.slice(0,100).reduce((a,b)=>a+b)/100, last100MeanMs: durations.slice(-100).reduce((a,b)=>a+b)/100, sameRunnerAndProvider: true, provider: "credential-free fake Codex process", opened };
if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, JSON.stringify(report, null, 2), { mode: 0o600 });
} finally {
await bundle.transport.close();
await consumer;
await authority?.close();
expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
await rm(directory, { recursive: true, force: true });
}
}, 30 * 60_000);
@@ -0,0 +1,110 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { advanceSourceCursor, type SourceCursor } from "../control-plane/event-epochs.js";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { expect, it, vi } from "vitest";
import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
it.each([undefined, 4])("crosses raw event epochs with normalized limit %s while preserving output and provider ownership", async (normalizedEventEpochLimit) => {
const root = await mkdtemp(join(tmpdir(), "indexed-event-epochs-"));
const callsPath = join(root, "calls.log");
const identity = { runnerInstanceId: "epoch-runner", environmentLeaseId: "epoch-lease", normalizedSessionId: "epoch-session",
runId: "epoch-run", turnId: "epoch-turn", itemId: "epoch-item" };
let authority: SqliteAuthorityStore | undefined, core: DurablePrpControlPlane | undefined;
const open = DurablePrpControlPlane.open.bind(DurablePrpControlPlane);
const spy = vi.spyOn(DurablePrpControlPlane, "open").mockImplementation(async options => core = await open({ ...options, eventEpochLimit: 8, commandEpochLimit: 8, secureChannelFrameLimit: 64, normalizedEventEpochLimit }));
const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--soak-output-on-steer", "--record-process-start", "--call-log", callsPath, "--require-lightweight-history"],
stateDirectory: root, prpIdentity: identity,
authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"event-epochs",create:true}),
});
const driver = new CodexAppServerDriver({ taskEnvelope: {schema:"paperclip.skillless_task.v1",objective:"Keep producing output across event epochs.",
completionContract:{revision:"event-epochs-v1",criteria:[{id:"output",requirement:"Deliver every output body exactly."}]},constraints:[],expectedResultSchema:"paperclip.run_result.v1"},
runnerInstanceId:identity.runnerInstanceId,approvalPolicy:"never",includeCollaborationModeInstructions:false,
environment:{PATH:process.env.PATH ?? "/usr/bin:/bin",HOME:"/isolated/home",CODEX_HOME:"/isolated/codex-home",LANG:"C.UTF-8"},transportFactory:()=>bundle.transport,requireProviderSessionIdentity:true });
let session: Awaited<ReturnType<typeof driver.openSession>> | undefined, consumer: Promise<void> | undefined, failure: unknown;
const epochs = new Set<string | undefined>(), outputs = new Map<string, string>(), ids = new Set<string>();
let completed = 0;
let cursor: SourceCursor = { sourceSeq: 0 };
const normalizedEpochs = new Set<string | undefined>();
try {
session = await driver.openSession({runId:identity.runId,normalizedSessionId:identity.normalizedSessionId,workingDirectory:root});
consumer = (async () => { for await (const event of session!.events()) {
expect(ids.has(event.sourceEventId)).toBe(false); ids.add(event.sourceEventId);
cursor = advanceSourceCursor(cursor, event);
normalizedEpochs.add(event.sourceEpoch);
if (event.eventType === "output.body.chunk") {
const id = String((event.payload.body as Record<string, unknown>).bodyId);
outputs.set(id, (outputs.get(id) ?? "") + String(event.payload.text));
}
if (event.eventType === "item.completed" && event.itemId?.startsWith("soak-output-")) {
const body = event.payload.outputBody as Record<string, unknown>;
expect(outputs.get(String(body.bodyId))).toBe(`${event.itemId}: ${"s".repeat(8192)}`);
completed++;
}
await session!.acknowledgeEvent!(event);
} })().catch(error => {failure=error;});
const turn = await session.startTurn({message:{text:"Hold this turn across namespace changes."}});
const owner = bundle.evidence();
for (let index=1;index<=16;index++) {
await session.steer!({turnId:turn.turnId,message:{text:`Output ${index}`},correlationId:`epoch-steer-${index}`});
await vi.waitFor(() => { if(failure) throw failure; expect(completed).toBe(index); },{timeout:20_000,interval:10});
epochs.add(core!.store.state.indexedState?.sourceEpoch);
expect(bundle.evidence()).toMatchObject({runnerPid:owner.runnerPid,providerPid:owner.providerPid,runnerExited:false});
if(index%3===0) core!.disconnectActiveRunner();
}
expect(epochs.size).toBeGreaterThanOrEqual(5);
if (normalizedEventEpochLimit === undefined) expect(normalizedEpochs.size).toBe(1);
else expect(normalizedEpochs.size).toBeGreaterThan(10);
expect(core!.store.state.identity).toEqual(identity);
const calls=(await readFile(callsPath,"utf8")).trim().split(/\r?\n/);
expect(calls.filter(call=>call==="process-start")).toHaveLength(1);
expect(calls.filter(call=>call==="thread/start")).toHaveLength(1);
expect(calls.filter(call=>call==="turn/start")).toHaveLength(1);
expect(calls).not.toContain("turn/interrupt"); expect(calls).not.toContain("thread/resume");
await session.close({reason:"event epoch qualification complete"}); await consumer; session=undefined;
if(failure) throw failure;
} finally { await session?.close({reason:"event epoch cleanup"}); await consumer; await bundle.transport.close(); await authority?.close(); spy.mockRestore(); await rm(root,{recursive:true,force:true}); }
},90_000);
it("keeps indexed replay and renewed event namespaces through warm run attachment", async () => {
const root=await mkdtemp(join(tmpdir(),"indexed-event-warm-"));
let authority:SqliteAuthorityStore|undefined, core:DurablePrpControlPlane|undefined, consumer:Promise<void>|undefined, failure:unknown;
const open=DurablePrpControlPlane.open.bind(DurablePrpControlPlane);
const spy=vi.spyOn(DurablePrpControlPlane,"open").mockImplementation(async options=>core=await open({...options,eventEpochLimit:4,commandEpochLimit:8}));
const bundle=createCapabilityRunnerdCodexTransport({runnerBinary:runnerTestBinary(),
codexCommand:resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),codexArgs:["--state-file",join(root,"fake.json"),"--durable-turn-ids"],
stateDirectory:root,lifecyclePolicy:{mode:"warm",idleTimeoutMs:60_000},
authorityStoreFactory:async (_identity,directory)=>authority??=await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"event-epochs-warm",create:true}),
});
bundle.transport.setServerRequestHandler(async()=>({success:true,contentItems:[]}));
let completed=0;
try {
await bundle.transport.request("thread/start",{cwd:root,dynamicTools:[{name:"get_task_context",description:"Read the active task.",inputSchema:{type:"object",properties:{},additionalProperties:false}}],completionContract:{revision:"epoch-warm-v1",criterionIds:["objective"]}});
const owner=bundle.evidence();
consumer=(async()=>{for await(const notification of bundle.transport.notifications()) {
const port=bundle.transport.normalizedDelivery!()!;
if(notification.paperclipDelivery) await port.commit({expectedRevision:port.load()?.revision??0,raw:notification.paperclipDelivery,driver:{},events:[]});
if(notification.method==="turn/completed") completed++;
}})().catch(error=>{failure=error;});
for(let round=1;round<=3;round++) {
await bundle.transport.request("turn/start",{input:[{type:"text",text:`Run ${round}`}]});
await vi.waitFor(()=>{if(failure) throw failure;expect(completed).toBe(round);expect(core!.store.state.indexedState?.sourceEpoch).toBeTruthy();},{timeout:20_000,interval:10});
expect(bundle.evidence()).toMatchObject({runnerPid:owner.runnerPid,providerPid:owner.providerPid,runnerExited:false});
if(round<3) await bundle.transport.attachRun!({runId:`warm-run-${round}`,turnId:`warm-turn-${round}`,itemId:`warm-item-${round}`});
}
if(failure) throw failure;
} finally {await bundle.transport.close();await consumer;await authority?.close();spy.mockRestore();await rm(root,{recursive:true,force:true});}
},90_000);
@@ -0,0 +1,130 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { createHash } from "node:crypto";
import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { expect, it } from "vitest";
import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
const mib = Number(process.env.PAPERCLIP_HISTORY_PIPELINE_MIB ?? 0);
it.skipIf(![64, 10240].includes(mib))("ingests large ordinary provider output through runner, controller, normalizer and durable event sink", async () => {
const root = await mkdtemp(join(tmpdir(), "indexed-history-pipeline-"));
let authority: SqliteAuthorityStore | undefined;
const sink = await SqliteAuthorityStore.open({ path: join(root, "normalized.sqlite"), binding: "pipeline-sink", create: true });
const identity = { runnerInstanceId: "pipeline-runner", environmentLeaseId: "pipeline-lease", normalizedSessionId: "pipeline-session", runId: "pipeline-run", turnId: "pipeline-turn", itemId: "pipeline-item" };
const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--history-growth-mib", String(mib)],
stateDirectory: root, prpIdentity: identity,
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "history-pipeline", create: true }),
});
const driver = new CodexAppServerDriver({ taskEnvelope: { schema: "paperclip.skillless_task.v1", objective: "Generate ordinary output to qualify durable history.",
completionContract: { revision: "pipeline-v1", criteria: [{ id: "history", requirement: "Deliver all output." }] }, constraints: [], expectedResultSchema: "paperclip.run_result.v1" },
runnerInstanceId: identity.runnerInstanceId, approvalPolicy: "never", includeCollaborationModeInstructions: false,
environment: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: "/isolated/home", CODEX_HOME: "/isolated/codex-home", LANG: "C.UTF-8" },
transportFactory: () => bundle.transport, requireProviderSessionIdentity: true });
let session: Awaited<ReturnType<typeof driver.openSession>> | undefined;
let generation = "0", events = 0, normalizedBytes = 0, peakCurrentBytes = 0, historyItems = 0, bodyBytes = 0;
let body: { id: string; offset: number; hash: ReturnType<typeof createHash> } | null = null;
const hash = createHash("sha256"), started = Date.now();
const report = async (status: string, extra = {}) => {
if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, JSON.stringify({ schema: "paperclip.indexed-history-pipeline.v1", status, provider: "credential-free fake Codex", mib, events, historyItems, bodyBytes, normalizedBytes, peakCurrentBytes, elapsedMs: Date.now() - started, ...extra }, null, 2), { mode: 0o600 });
};
try {
session = await driver.openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: root });
const commit = async (event: import("../protocol/replay-contract.js").PrpEvent) => {
generation = await sink.commit({ expectedGeneration: generation, state: { lastSourceSeq: event.sourceSeq }, records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record<string, unknown> }] });
};
session.setEventCommitter!(commit);
const consumeUntilTerminal = async () => {
for await (const event of session!.events()) {
const bytes = JSON.stringify(event); hash.update(bytes); normalizedBytes += Buffer.byteLength(bytes); events++;
await commit(event);
await session!.acknowledgeEvent!(event);
if (event.eventType === "output.body.chunk") {
const reference = event.payload.body as Record<string, unknown>;
const chunk = String(event.payload.text), offset = Number(event.payload.offset);
if (offset === 0) { expect(body).toBeNull(); body = { id: String(reference.bodyId), offset: 0, hash: createHash("sha256") }; }
expect(body?.id).toBe(reference.bodyId);
expect(body?.offset).toBe(offset);
expect(createHash("sha256").update(chunk).digest("hex")).toBe(event.payload.sha256);
body!.hash.update(chunk); body!.offset += Buffer.byteLength(chunk); bodyBytes += Buffer.byteLength(chunk);
}
const outputId = event.eventType === "tool.execution.completed" ? event.payload.executionId : event.itemId;
if (["item.completed", "tool.execution.completed"].includes(event.eventType) && typeof outputId === "string" && outputId.startsWith("history-")) {
const reference = event.payload.outputBody as Record<string, unknown>;
expect(body?.offset).toBe(256 * 1024);
expect(body!.hash.digest("hex")).toBe(reference.sha256);
expect(body?.id).toBe(reference.bodyId);
body = null; historyItems++;
}
if (events % 128 === 0) {
peakCurrentBytes = Math.max(peakCurrentBytes, Buffer.byteLength(JSON.stringify((await authority!.load())!.state)));
expect(peakCurrentBytes).toBeLessThan(8 * 1024 * 1024);
await report("running");
}
if (["session.failed", "turn.failed"].includes(event.eventType)) throw new Error(`pipeline failed: ${JSON.stringify(event.payload)}`);
if (event.eventType === "turn.completed") return;
}
throw new Error("history stream ended before completion");
};
const consume = consumeUntilTerminal();
// Keep any asynchronous consumer failure observed while turn admission runs.
void consume.catch(() => {});
await session.startTurn({ message: { text: "Generate the qualification output." } });
await consume;
expect(historyItems).toBe(mib * 4);
expect(bodyBytes).toBe(mib * 1024 * 1024);
await session.flushEventDelivery!();
const beforeContinuation = bundle.evidence();
const continuationStarted = performance.now();
await session.attachRun!({ runId: "pipeline-continuation" });
const resumed = consumeUntilTerminal(); void resumed.catch(() => {});
await session.startTurn({ message: { text: "Continue after the retained output history." } });
await resumed;
const continuationMs = performance.now() - continuationStarted;
expect(bundle.evidence().providerPid).toBe(beforeContinuation.providerPid);
expect(bundle.evidence().runnerPid).toBe(beforeContinuation.runnerPid);
await session.flushEventDelivery!();
const before = performance.now();
const snapshots = await Promise.all([authority!.load(), readIndexedLocalState(join(root, "runner/runner-state.json")), readIndexedLocalState(join(root, "runner/codex-provider-state.json"))]);
const resumeReadMs = performance.now() - before;
const currentBytes = snapshots.map(snapshot => Buffer.byteLength(JSON.stringify(snapshot!.state)));
expect(currentBytes.every(bytes => bytes < 8 * 1024 * 1024)).toBe(true);
let physicalBytes = 0, files = 0;
async function inventory(directory: string): Promise<void> {
for (const entry of await readdir(directory, { withFileTypes: true })) {
const path = join(directory, entry.name);
if (entry.isDirectory()) await inventory(path);
else if (entry.isFile()) {
// SQLite may unlink a transient WAL/SHM after readdir during a
// checkpoint. Persistent files disappearing are still an error.
const metadata = await stat(path).catch(error => {
if ((error as NodeJS.ErrnoException).code === "ENOENT" && /\.sqlite-(wal|shm)$/.test(path)) return null;
throw error;
});
if (metadata) { physicalBytes += metadata.size; files++; }
}
}
}
await inventory(root);
expect(physicalBytes).toBeGreaterThan(mib * 1024 * 1024);
const evidence = bundle.evidence();
await session.close({ reason: "qualification complete" }); session = undefined;
await bundle.transport.close(); await authority!.close(); authority = undefined;
await sink.close(); await rm(root, { recursive: true, force: true });
await report("passed", { physicalBytes, files, currentBytes, resumeReadMs, continuationMs, normalizedSha256: hash.digest("hex"), runner: evidence });
} catch (error) { await report("failed", { error: error instanceof Error ? error.message : String(error) }); throw error; }
finally { await session?.close({ reason: "history qualification complete" }); await bundle.transport.close(); await authority?.close(); await sink.close(); await rm(root, { recursive: true, force: true }); }
}, 12 * 60 * 60_000);
@@ -0,0 +1,86 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import { mkdtemp, rename, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { performance } from "node:perf_hooks";
import { expect, it } from "vitest";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
const hours = 72;
// Real elapsed-time qualification. A held fake-provider turn exercises runner
// leases and wall-clock limits without paid inference. This does not qualify
// a hosted provider/sandbox TTL or substitute for crash/resource-failure lanes.
it.skipIf(process.env.PAPERCLIP_HISTORY_SOAK_HOURS !== String(hours))("keeps one indexed active turn alive for 72 real hours", async () => {
const directory = await mkdtemp(join(tmpdir(), "indexed-history-soak-"));
let authority: SqliteAuthorityStore | undefined;
const identity = { runnerInstanceId: "soak-runner", environmentLeaseId: "soak-lease", normalizedSessionId: "soak-session", runId: "soak-run", turnId: "soak-turn", itemId: "soak-item" };
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: runnerTestBinary(),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(directory, "fake-state.json"), "--durable-turn-ids", "--hold-turn"],
stateDirectory: directory, prpIdentity: identity,
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 600_000 },
authorityStoreFactory: async (_identity, root) => authority ??= await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: "history-soak", create: true }),
});
bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] }));
let consumer: Promise<void> | undefined;
let consumerFailure: unknown;
let terminal = false;
let startedAt = new Date().toISOString(), started = performance.now();
let qualified = false;
let probes = 0, peakCurrentBytes = 0, maxProbeMs = 0;
const report = async (status: string, error?: unknown) => {
if (!process.env.PAPERCLIP_HISTORY_REPORT) return;
const path = process.env.PAPERCLIP_HISTORY_REPORT;
const value = { schema: "paperclip.indexed-active-turn-soak.v1", status, startedAt, updatedAt: new Date().toISOString(), elapsedMs: performance.now()-started, requiredHours: hours, probes, peakCurrentBytes, maxProbeMs, provider: "credential-free fake Codex process", runnerPid: bundle.evidence().runnerPid, providerPid: bundle.evidence().codexPid, ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}) };
await writeFile(`${path}.tmp`, JSON.stringify(value,null,2), { mode: 0o600 });
await rename(`${path}.tmp`, path);
};
try {
await bundle.transport.request("initialize", {});
const opened = await bundle.transport.request("thread/start", { cwd: directory, dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }], completionContract: { revision: "soak-contract", criterionIds: ["objective"] } }) as { thread: { id: string } };
const owner = bundle.evidence();
consumer = (async () => {
for await (const notification of bundle.transport.notifications()) {
const port = bundle.transport.normalizedDelivery!()!;
if (notification.paperclipDelivery) await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: notification.paperclipDelivery, driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [] });
if (notification.method === "turn/completed") terminal = true;
}
})().catch(error => { consumerFailure = error; });
await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Keep this turn active until explicitly interrupted." }] });
startedAt = new Date().toISOString(); started = performance.now();
await report("running");
while (performance.now()-started < hours*60*60_000) {
const before = performance.now();
await bundle.transport.request("thread/read", { threadId: opened.thread.id, includeTurns: false });
probes++; maxProbeMs = Math.max(maxProbeMs,performance.now()-before);
if (consumerFailure) throw consumerFailure;
expect(terminal).toBe(false);
expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false });
const current = await authority!.load();
const bytes = Buffer.byteLength(JSON.stringify(current!.state));
peakCurrentBytes = Math.max(peakCurrentBytes,bytes);
expect(bytes).toBeLessThan(256*1024);
await report("running");
await new Promise(resolve => setTimeout(resolve, Math.min(30_000, Math.max(0,hours*60*60_000-(performance.now()-started)))));
}
qualified = true;
await report("settling");
} catch (error) { await report("failed",error); throw error; }
finally {
try {
await bundle.transport.close(); await consumer; await authority?.close();
expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
if (qualified) await report("passed");
} catch (error) { await report("cleanup_failed",error); throw error; }
finally { await rm(directory,{recursive:true,force:true}); }
}
}, (hours+1)*60*60_000);
@@ -0,0 +1,145 @@
import { resolveRunnerCargoTestBinary } from "../../test/cargo-test-binary.js";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { appendFile, cp, mkdtemp, open, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { migrateLegacySessionAuthority, type PreparedLocalSession } from "../control-plane/legacy-authority-activation.js";
import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
import { readDurableControlPlaneState } from "../control-plane/authority-locator.js";
import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js";
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js";
const execute = promisify(execFile);
const runnerBinary = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "paperclip-runnerd");
const providerBinary = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server");
const identity = { runnerInstanceId: "migration-runner", environmentLeaseId: "migration-lease", normalizedSessionId: "migration-session", runId: "migration-run", turnId: "migration-turn", itemId: "migration-item" };
async function fileDigest(path: string) {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path, { highWaterMark: 64 * 1024 })) hash.update(chunk);
return hash.digest("hex");
}
it("migrates a real legacy runner/provider through every activation crash boundary, then continues the same provider session", async () => {
const root = await mkdtemp(join(tmpdir(), "legacy-session-migration-"));
const original = join(root, "original");
const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary, codexCommand: providerBinary,
codexArgs: ["--state-file", join(root, "fake-state.json"), "--durable-turn-ids"], stateDirectory: original, prpIdentity: identity });
bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] }));
let successor: ReturnType<typeof createCapabilityRunnerdCodexTransport> | undefined;
let authority: SqliteAuthorityStore | undefined;
try {
await bundle.transport.request("initialize", {});
const opened = await bundle.transport.request("thread/start", { cwd: root,
dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }],
completionContract: { revision: "migration-contract", criterionIds: ["objective"] } });
await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Complete before conversion." }] });
for await (const notification of bundle.transport.notifications()) if (notification.method === "turn/completed") break;
await bundle.transport.close();
expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
const owner = bundle.evidence();
const assertExclusiveFence = async () => {
for (const pid of [owner.runnerPid, owner.codexPid]) {
if (!pid) throw new Error("missing exact fixture process owner");
try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; throw error; }
throw new Error("fixture owner remains alive");
}
};
await assertExclusiveFence();
const controllerBytes = await readFile(join(original, "control-plane/control-plane-state.json"));
const runnerBytes = await readFile(join(original, "runner/runner-state.json"));
const providerBytes = await readFile(join(original, "runner/codex-provider-state.json"));
const originalController = JSON.parse(controllerBytes.toString());
const runner = JSON.parse(runnerBytes.toString());
const boundaries = ["source-change", "intent", "prepared", "controller-commit", "publish:runner-state.sqlite", "publish:runner-state.sqlite.receipts", "publish:runner-state.sqlite.routing", "publish:runner-state.sqlite.lifetime", "publish:runner-state.sqlite.lock", "publish:codex-provider-state.sqlite", "publish:codex-provider-state.sqlite.receipts", "publish:codex-provider-state.sqlite.routing", "publish:codex-provider-state.sqlite.lifetime", "publish:codex-provider-state.sqlite.lock", "publish:runner-state.json", "publish:codex-provider-state.json", "controller-locator", "active"];
for (const [index, boundary] of boundaries.entries()) {
const directory = join(root, `case-${index}`);
await cp(original, directory, { recursive: true });
const controllerPath = join(directory, "control-plane/control-plane-state.json");
const runnerPath = join(directory, "runner/runner-state.json");
let expectedAck = originalController.ackedSourceSeq;
if (boundary === "active") {
// Storage qualification: inject valid historical diagnostic frames
// after the real fixture settles. This is synthesized history, not
// 2,000 model calls. Then migrate >192 MiB and resume real processes.
const file = await open(controllerPath, "w", 0o600);
const { committedEvents, ...metadata } = originalController;
expectedAck += 2_000;
await file.write(`${JSON.stringify({ ...metadata, ackedSourceSeq: expectedAck }).slice(0, -1)},"committedEvents":[`);
let separator = "";
for (const event of committedEvents) { await file.write(`${separator}${JSON.stringify(event)}`); separator = ","; }
const padding = "x".repeat(100 * 1024);
for (let seq = originalController.ackedSourceSeq + 1; seq <= expectedAck; seq++) {
const sourceEventId = `migration-growth-${seq}`;
const payload = { schema: "paperclip.prp.event.v1", schemaVersion: 1, ...identity, sourceSeq: seq, sourceEventId,
sourceInstanceId: identity.runnerInstanceId, sourceKind: "runner", eventType: "harness.diagnostic", priority: 1,
emittedAt: "2026-09-28T00:00:00.000Z", payload: { code: "qualification-history", text: padding } };
await file.write(`,${JSON.stringify({ sourceSeq: seq, sourceEventId, eventType: "harness.diagnostic", priority: 1, deliveryCount: 1, logicalEffectCount: 1,
envelope: { protocol: "paperclip.runner", version: 1, kind: "event", ...identity, payload } })}`);
}
await file.write("]}"); await file.close();
expect((await stat(controllerPath)).size).toBeGreaterThan(192 * 1024 * 1024);
await writeFile(runnerPath, JSON.stringify({ ...runner, ackedSourceSeq: expectedAck, nextSourceSeq: expectedAck + 1 }), { mode: 0o600 });
}
const originalControllerDigest = await fileDigest(controllerPath), originalRunnerDigest = await fileDigest(runnerPath);
authority = await SqliteAuthorityStore.open({ path: join(directory, "control-plane/authority.sqlite"), binding: `migration-${index}`, create: true });
const prepareLocal = async (destination: string): Promise<PreparedLocalSession> => {
const args = ["storage", "stage-legacy", "--state-dir", join(directory, "runner"), "--destination", destination, "--fence-id", "migration-fence",
"--connect-url", "ws://127.0.0.1:3000/api/runner/v1/connect/migration-run", "--runner-version", "qualification", "--runner-digest", `sha256:${"a".repeat(64)}`,
"--max-outbox-bytes", String(runner.maxOutboxBytes), "--p0-reserve-bytes", String(runner.p0ReserveBytes),
"--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId, "--session-id", identity.normalizedSessionId,
"--run-id", identity.runId, "--turn-id", identity.turnId, "--item-id", identity.itemId];
const { stdout } = await execute(runnerBinary, args, { maxBuffer: 128 * 1024, timeout: 30_000 });
return JSON.parse(stdout) as PreparedLocalSession;
};
const options = { root: directory, identity, authority, fenceId: "migration-fence", assertExclusiveFence, prepareLocal };
await expect(migrateLegacySessionAuthority({ ...options, onBoundary: async (name) => { if (name === boundary || (boundary === "source-change" && name === "prepared")) throw new Error(`crash:${boundary}`); } })).rejects.toThrow(`crash:${boundary}`);
if (boundary === "source-change") {
await appendFile(join(directory, "control-plane/control-plane-state.json"), "\n");
await expect(migrateLegacySessionAuthority(options)).rejects.toThrow("source_changed");
expect((await authority.load())!.state.phase).toBe("prepared");
await authority.close(); authority = undefined;
continue;
}
if (boundary !== "active") await expect(DurablePrpControlPlane.open({ stateDirectory: join(directory, "control-plane"), identity, expectedRunnerVersion: "test", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: authority })).rejects.toThrow("migration_pending");
await authority.close();
authority = await SqliteAuthorityStore.open({ path: join(directory, "control-plane/authority.sqlite"), binding: `migration-${index}`, create: false });
await migrateLegacySessionAuthority({ ...options, authority });
const migrated = await readDurableControlPlaneState(join(directory, "control-plane"));
expect(migrated.identity).toEqual(identity);
expect(migrated.ackedSourceSeq).toBe(expectedAck);
expect(await fileDigest(join(directory, "indexed-migration/legacy/control-plane-state.json"))).toBe(originalControllerDigest);
expect(await fileDigest(join(directory, "indexed-migration/legacy/runner-state.json"))).toBe(originalRunnerDigest);
expect(await readFile(join(directory, "indexed-migration/legacy/codex-provider-state.json"))).toEqual(providerBytes);
for (const name of ["runner-state", "codex-provider-state"]) {
expect((await stat(join(directory, "runner", `${name}.sqlite.routing`))).isDirectory()).toBe(true);
await expect(stat(join(directory, "indexed-migration/runner", `${name}.sqlite.routing`))).rejects.toMatchObject({ code: "ENOENT" });
}
expect((await readIndexedLocalState(join(directory, "runner/runner-state.json"))).state).toMatchObject({ ...identity, lifecycle: "suspended" });
const firstCommand = originalController.commands.find((command: {status:string}) => command.status === "completed");
expect((await authority.getRecord(identity.runId, "command", firstCommand.commandId))!.body).toEqual(firstCommand);
if (index === boundaries.length - 1) {
await authority.close(); authority = undefined;
successor = createCapabilityRunnerdCodexTransport({ runnerBinary, codexCommand: providerBinary,
codexArgs: ["--state-file", join(root, "fake-state.json"), "--durable-turn-ids"],
stateDirectory: directory, prpIdentity: { ...identity, runId: "migration-next-run", turnId: "migration-next-turn", itemId: "migration-next-item" },
authorityStoreFactory: async (_identity, currentDirectory) => authority = await SqliteAuthorityStore.open({ path: join(currentDirectory, "authority.sqlite"), binding: `migration-${index}`, create: true }) });
successor.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] }));
const resumed = await successor.transport.request("thread/read", {});
expect((resumed.thread as Record<string, unknown>).id).toBe((opened.thread as Record<string, unknown>).id);
await successor.transport.request("turn/start", { input: [{ type: "text", text: "Continue after indexed migration." }] });
for await (const notification of successor.transport.notifications()) if (notification.method === "turn/completed") break;
await successor.transport.close();
expect(successor.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
}
await authority.close(); authority = undefined;
}
} finally {
await successor?.transport.close(); await bundle.transport.close(); await authority?.close();
await rm(root, { recursive: true, force: true });
}
}, 120_000);
@@ -1,3 +1,4 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
// The retained-settlement maintenance suite below moved out of
// runnerd-codex-transport.test.ts. Vitest schedules whole files onto
// workers, so that single 8.9k-line file serialized ~400s of tests and was
@@ -51,7 +52,12 @@ import {
// Explicit private-artifact test lane; production/default dist is never changed.
const defaultCapabilityRunnerdBinary = () =>
process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ??
qualifiedCapabilityRunnerdBinary();
resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
function maintenanceFixtureBackendName(fixtureId: string): string {
return `maintenance-test-${fixtureId}`;
@@ -138,10 +144,7 @@ it.each([
const activated = join(directory, "activated");
const home = join(directory, "source-home");
await mkdir(home);
const fakeCodex = resolve(
import.meta.dirname,
"../../runner/target/debug/fake-codex-app-server",
);
const fakeCodex = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server");
const bin = join(directory, "provider-bin");
if (bareCodex) {
await mkdir(bin);
@@ -270,10 +273,10 @@ it.each([
.update(await readFile(fixtureRunner))
.digest("hex")}`,
});
builder.queueCommand("turn.stop", {
(await builder.queueCommand("turn.stop", {
reason: "interrupted original close",
});
if (!missingHome) builder.queueCommand("runner.suspend", {});
}));
if (!missingHome) (await builder.queueCommand("runner.suspend", {}));
// Match the retained production split: runner-owned unacknowledged
// output plus another full provider-owned prefix behind the old suspend.
const runnerFile = join(original, "runner/runner-state.json");
@@ -815,10 +818,10 @@ it.each([
.update(await readFile(fixtureRunner)).digest("hex")}`,
onCommittedEvent: appendEvent,
});
const snapshot = replayCore.queueCommand("session.snapshot", {});
const stop = replayCore.queueCommand("turn.stop", {
const snapshot = (await replayCore.queueCommand("session.snapshot", {}));
const stop = (await replayCore.queueCommand("turn.stop", {
reason: "startup-fence regression only",
});
}));
let replayHandle: ReturnType<typeof durableControlPlane.spawnRunner> | null = null;
let replayAssertionFailed = false;
try {
@@ -828,7 +831,7 @@ it.each([
connectUrl: replayCore.connectUrl,
stateDirectory: join(copy, "runner"),
identity,
ticket: replayCore.issueBootstrapTicket(),
ticket: (await replayCore.issueBootstrapTicket()),
maxOutboxBytes: runnerState.maxOutboxBytes,
p0ReserveBytes: runnerState.p0ReserveBytes,
maxRuntimeMs: 2_000,
@@ -846,9 +849,9 @@ it.each([
hasRuntimeContext: false,
}),
});
await vi.waitFor(() => {
await vi.waitFor(async () => {
for (const queued of [snapshot, stop]) {
const command = replayCore.getCommand(queued.commandId);
const command = (await replayCore.getCommand(queued.commandId));
expect(command?.status).toBe("failed");
expect(command?.result).toMatchObject({
result: {
@@ -1,3 +1,4 @@
import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js";
import {
cp,
mkdir,
@@ -35,6 +36,9 @@ import { executeNativeSession } from "../native-session-runtime.js";
import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js";
// Preserve the real implementation even if a timed-out fixture is still unwinding.
const openUnmockedControlPlane = DurablePrpControlPlane.open.bind(DurablePrpControlPlane);
import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
@@ -90,7 +94,12 @@ import {
// Explicit private-artifact test lane; production/default dist is never changed.
const defaultCapabilityRunnerdBinary = () =>
process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ??
qualifiedCapabilityRunnerdBinary();
resolveRunnerCargoTestBinaryOrDefault(
resolve(import.meta.dirname, "../../runner"),
"debug",
"paperclip-runnerd",
qualifiedCapabilityRunnerdBinary,
);
async function expectTurnStarted(
notifications: AsyncIterator<{ method: string }>,
@@ -705,7 +714,9 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)(
codexCommand: fakeCodex,
codexArgs: fakeCodexArgs(stateDirectory, "--split-event-burst"),
stateDirectory,
closeGraceMs: 2_000,
// Only the expiry case requires a short deadline. The success case must
// leave room for real journal fsyncs under a parallel storage test load.
closeGraceMs: mode === "within_budget" ? 10_000 : 2_000,
controlPlaneRegistration: async (authority) => {
core = authority;
await authority.start();
@@ -832,7 +843,7 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)(
await rm(stateDirectory, { recursive: true, force: true });
}
},
15_000,
30_000,
);
it("infers a remote provider turn until its own terminal event is durable", () => {
@@ -2106,6 +2117,16 @@ it("rehydrates canonical session goals into Codex goal notifications", () => {
).toEqual({ revision: 7, threadId: "thread-1", workingNow: false });
});
it("preserves the command output channel and execution binding through the facade", () => {
const payload = { kind: "commandExecution", providerMethod: "item/commandExecution/outputDelta", itemId: "exec-1", text: "stdout" };
expect(runnerdCanonicalNotificationMethod("item.delta", payload)).toBe("item/commandExecution/outputDelta");
expect(rehydrateRunnerdDeltaNotification(payload, "thread", "turn")).toMatchObject({
threadId: "thread", turnId: "turn", itemId: "exec-1", delta: "stdout", kind: "commandExecution",
});
expect(runnerdCanonicalNotificationMethod("item.delta", { ...payload, providerMethod: "paperclip/runResult" })).toBeUndefined();
expect(runnerdCanonicalNotificationMethod("item.delta", { kind: "agentMessage", text: "prose" })).toBe("item/agentMessage/delta");
});
it("routes canonical session goals back through the Codex notification facade", () => {
expect(
runnerdCanonicalNotificationMethod("session.goal.updated", {
@@ -2201,10 +2222,7 @@ it("recovers provider readiness from an already-committed journal without replay
).toEqual(persistedReady);
});
const fakeCodex = resolve(
import.meta.dirname,
"../../runner/target/debug/fake-codex-app-server",
);
const fakeCodex = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server");
function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] {
return [
@@ -2325,13 +2343,13 @@ it("keeps a quiet active Codex turn in the same process across connection lease
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-renew-active-"));
const callsPath = join(stateDirectory, "calls.log");
const cores: DurablePrpControlPlane[] = [];
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane")
.mockImplementation(function(options: ConstructorParameters<typeof OriginalCore>[0]) {
const core = new OriginalCore({ ...options, connectionLeaseTtlMs: 60_000 });
const openCore = openUnmockedControlPlane;
const coreSpy = vi.spyOn(durableControlPlane.DurablePrpControlPlane, "open")
.mockImplementation(async function(options: Parameters<typeof openCore>[0]) {
const core = await openCore({ ...options, connectionLeaseTtlMs: 60_000 });
cores.push(core);
return core;
} as unknown as typeof OriginalCore);
});
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex,
codexArgs: fakeCodexArgs(stateDirectory, "--hold-turn", "--record-process-start", "--call-log", callsPath),
@@ -2556,13 +2574,23 @@ it("controls a Codex session goal end to end through durable PRP v2", async () =
});
}, 30_000);
it.each([false, true])("binds goal turns through the full Codex harness (autonomous continuation: %s)", async (autocontinue) => {
it.each([{ autocontinue: false, indexed: false }, { autocontinue: true, indexed: false }, { autocontinue: true, indexed: true }])("binds goal turns through the full Codex harness (autonomous continuation: $autocontinue, indexed: $indexed)", async ({ autocontinue, indexed }) => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-goal-harness-"));
let authority: SqliteAuthorityStore | null = null;
const sink = indexed ? await SqliteAuthorityStore.open({ path: join(stateDirectory, "normalized.sqlite"), binding: "normalized-goal-test", create: true }) : null;
let sinkGeneration = "0";
const append = async (event: PrpEvent) => {
if (sink) sinkGeneration = await sink.commit({ expectedGeneration: sinkGeneration, state: { lastSourceSeq: event.sourceSeq }, records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record<string, unknown> }] });
};
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: defaultCapabilityRunnerdBinary(),
codexCommand: fakeCodex,
codexArgs: fakeCodexArgs(stateDirectory, "--goal-autostart", ...(autocontinue ? ["--goal-autocontinue"] : [])),
stateDirectory,
...(indexed ? {
prpIdentity: { runnerInstanceId: "runner-codex", environmentLeaseId: "environment-goal", runId: "run-goal-harness-autostart", normalizedSessionId: "normalized-goal-harness-autostart", turnId: "turn-goal", itemId: "item-goal" },
authorityStoreFactory: async (_identity: unknown, directory: string) => authority = await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "goal-harness-indexed", create: true }),
} : {}),
});
const driver = new CodexAppServerDriver({
taskEnvelope: {
@@ -2593,11 +2621,13 @@ it.each([false, true])("binds goal turns through the full Codex harness (autonom
normalizedSessionId: "normalized-goal-harness-autostart",
workingDirectory: tmpdir(),
});
if (indexed) session.setEventCommitter!(append);
const observed: Array<{ eventType: string }> = [];
const turnStarted = Promise.race([
(async () => {
for await (const event of session!.events()) {
observed.push(event);
if (indexed) { await append(event); await session!.acknowledgeEvent!(event); }
if (event.eventType === "turn.started" && event.turnId === (autocontinue ? "provider-goal-turn-2" : "provider-goal-turn-1")) return event;
if (event.eventType === "session.failed") {
throw new Error(`goal autostart failed: ${JSON.stringify(event.payload)}`);
@@ -2625,6 +2655,8 @@ it.each([false, true])("binds goal turns through the full Codex harness (autonom
} finally {
await session?.close();
await bundle.transport.close();
await (authority as SqliteAuthorityStore | null)?.close();
await sink?.close();
await rm(stateDirectory, { recursive: true, force: true });
}
expect(bundle.evidence()).toMatchObject({
@@ -3439,10 +3471,7 @@ it("does not retry a real memoized transport close whose suspension proof is una
}));
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: defaultCapabilityRunnerdBinary(),
codexCommand: resolve(
import.meta.dirname,
"../../runner/target/debug/fake-codex-app-server",
),
codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),
codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")],
stateDirectory,
closeGraceMs: 400,
@@ -4273,13 +4302,13 @@ it.each(["held-ack", "lost-ack", "rejected-attach"] as const)(
const commitEntered = new Promise<void>((resolveEntered) => {
enteredCommit = resolveEntered;
});
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
const openCore = openUnmockedControlPlane;
const coreSpy = vi
.spyOn(durableControlPlane, "DurablePrpControlPlane")
.mockImplementation(function (
options: ConstructorParameters<typeof OriginalCore>[0],
.spyOn(durableControlPlane.DurablePrpControlPlane, "open")
.mockImplementation(async function (
options: Parameters<typeof openCore>[0],
) {
const core = new OriginalCore({
const core = await openCore({
...options,
onCommittedEvent: async (event) => {
await options.onCommittedEvent?.(event);
@@ -4314,7 +4343,7 @@ it.each(["held-ack", "lost-ack", "rejected-attach"] as const)(
});
cores.push(core);
return core;
} as unknown as typeof OriginalCore);
});
const launch = durableControlPlane.spawnRunner;
const launchSpy = vi
.spyOn(durableControlPlane, "spawnRunner")
@@ -4570,16 +4599,16 @@ it.each([false, true])(
const callsPath = join(stateDirectory, "calls.log");
const cores: DurablePrpControlPlane[] = [];
const handles: ReturnType<typeof durableControlPlane.spawnRunner>[] = [];
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
const openCore = openUnmockedControlPlane;
const coreSpy = vi
.spyOn(durableControlPlane, "DurablePrpControlPlane")
.mockImplementation(function (
options: ConstructorParameters<typeof OriginalCore>[0],
.spyOn(durableControlPlane.DurablePrpControlPlane, "open")
.mockImplementation(async function (
options: Parameters<typeof openCore>[0],
) {
const core = new OriginalCore(options);
const core = await openCore(options);
cores.push(core);
return core;
} as unknown as typeof OriginalCore);
});
const launch = durableControlPlane.spawnRunner;
const launchSpy = vi
.spyOn(durableControlPlane, "spawnRunner")
@@ -4651,8 +4680,8 @@ it.each([false, true])(
const getCommand = core.getCommand.bind(core);
observerSpy = vi
.spyOn(core, "getCommand")
.mockImplementation((commandId) => {
const command = getCommand(commandId);
.mockImplementation(async (commandId) => {
const command = await getCommand(commandId);
if (
command?.type === "run.attach" &&
core.store.state.completedWarmTransition?.command.commandId !==
@@ -4929,20 +4958,20 @@ it.each([
},
};
};
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
const openCore = openUnmockedControlPlane;
const coreSpy = vi
.spyOn(durableControlPlane, "DurablePrpControlPlane")
.mockImplementation(function (
options: ConstructorParameters<typeof OriginalCore>[0],
.spyOn(durableControlPlane.DurablePrpControlPlane, "open")
.mockImplementation(async function (
options: Parameters<typeof openCore>[0],
) {
const core = new OriginalCore(options);
const core = await openCore(options);
cores.push(core);
if (recovering && snapshotFault) {
const getCommand = core.getCommand.bind(core);
snapshotObserverSpy = vi
.spyOn(core, "getCommand")
.mockImplementation((id) => {
const command = getCommand(id);
.mockImplementation(async (id) => {
const command = await getCommand(id);
if (
command?.type !== "session.snapshot" ||
command.status !== "completed"
@@ -4968,7 +4997,7 @@ it.each([
});
}
return core;
} as unknown as typeof OriginalCore);
});
const launch = durableControlPlane.spawnRunner;
const launchSpy = vi
.spyOn(durableControlPlane, "spawnRunner")
@@ -1,3 +1,9 @@
import { isProviderProcessGeneration, provesProviderGenerationTransition } from "../control-plane/process-generation.js";
import { prepareIndexedRunnerArchive, finishIndexedRunnerArchive, publishIndexedArchiveHead, readIndexedArchiveHead, syncArchiveDirectory } from "./indexed-authority-archive.js";
import { readDurableControlPlaneState, readDurableControlPlaneSnapshot, type ExternalAuthorityReader } from "../control-plane/authority-locator.js";
import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js";
import { defaultStorageRunnerBinary, nativeStorageCommand } from "../control-plane/native-storage-command.js";
import { authorityJson, INDEXED_DURABILITY_CAPABILITY, type DurableAuthorityStore } from "../control-plane/durable-authority-store.js";
import { RunnerdTraceFrameIndex } from "./runnerd-trace-frame-index.js";
import { waitForWarmAttachmentReadiness } from "./warm-attachment-readiness.js";
import { codexExecutableReadOnlyRoots } from "../drivers/codex/codex-security-config.js";
@@ -40,6 +46,7 @@ import type {
} from "../contracts/durable-recovery.js";
import type { NativeRunIdentity } from "../contracts/types.js";
import type { PrpEvent } from "../protocol/replay-contract.js";
import { RAW_DELIVERY_FRAME_COMPLETE } from "../control-plane/normalized-delivery.js";
import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
import type {
HarnessRuntimeRequestResolution,
@@ -149,20 +156,9 @@ export function withCodexCollaborationRuntimeInstructions(
return `${base}\n\n${CODEX_COLLABORATION_RUNTIME_INSTRUCTIONS}`;
}
function readControlPlaneState(directory: string): Record<string, unknown> {
const path = resolve(directory, "control-plane-state.json");
const metadata = lstatSync(path);
if (
metadata.isSymbolicLink() ||
!metadata.isFile() ||
metadata.size > 64 * 1024 * 1024
) {
throw new Error("native_runner_control_plane_state_unsafe");
}
return record(JSON.parse(readFileSync(path, "utf8")));
}
const readControlPlaneState = readDurableControlPlaneState;
function readRunnerState(path: string): Record<string, unknown> {
async function readRunnerState(path: string): Promise<Record<string, unknown>> {
const metadata = lstatSync(path);
if (
metadata.isSymbolicLink() ||
@@ -171,7 +167,9 @@ function readRunnerState(path: string): Record<string, unknown> {
) {
throw new Error("native_runner_authority_rotation_state_unsafe");
}
return record(JSON.parse(readFileSync(path, "utf8")));
const value = record(JSON.parse(readFileSync(path, "utf8")));
if (["paperclip.runner.durable.state.indexed.v1", "paperclip.runner.codex-provider-state.indexed.v1"].includes(String(value.schema))) return (await readIndexedLocalState(path)).state;
return value;
}
function controlPlaneIdentity(
@@ -217,6 +215,9 @@ function assertRealDirectory(path: string): void {
}
function quarantineLocalRuntimeState(root: string, reason: unknown): never {
// An operator backup or an interrupted archive is recoverable contention,
// not evidence of corruption. Preserve the active routing for a retry.
if (reason instanceof Error && reason.message.includes("storage_pressure:")) throw reason;
assertRealDirectory(root);
const quarantine = resolve(
dirname(root),
@@ -233,21 +234,39 @@ function quarantineLocalRuntimeState(root: string, reason: unknown): never {
function authorityArchiveDirectory(
root: string,
identity: DurableRecoveryIdentity,
indexed = false,
): string {
const digest = createHash("sha256")
.update(JSON.stringify(identity))
// Indexed stores canonicalize object keys. Archive routing must survive
// reopening the same identity in a different JSON property order.
.update(indexed ? authorityJson(identity) : JSON.stringify(identity))
.digest("hex")
.slice(0, 24);
return resolve(root, "authority-epochs", `epoch-${digest}`);
}
function latestArchivedControlPlaneState(
async function latestArchivedControlPlaneState(
root: string,
desired: DurableRecoveryIdentity,
): Record<string, unknown> | null {
readAuthority?: ExternalAuthorityReader,
): Promise<Record<string, unknown> | null> {
const archivesRoot = resolve(root, "authority-epochs");
if (!existsSync(archivesRoot)) return null;
assertRealDirectory(archivesRoot);
const indexedArchive = readIndexedArchiveHead(archivesRoot);
if (indexedArchive !== null) {
const directory = resolve(indexedArchive, "control-plane");
assertRealDirectory(directory);
const state = await readControlPlaneState(directory, readAuthority);
const identity = controlPlaneIdentity(state);
if (
authorityArchiveDirectory(root, identity, true) !== indexedArchive ||
identity.runnerInstanceId !== desired.runnerInstanceId ||
identity.environmentLeaseId !== desired.environmentLeaseId ||
identity.normalizedSessionId !== desired.normalizedSessionId
) throw new Error("native_runner_authority_archive_conflict");
return state;
}
const candidates = readdirSync(archivesRoot, { withFileTypes: true })
.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
.map((entry) => resolve(archivesRoot, entry.name, "control-plane"))
@@ -262,7 +281,7 @@ function latestArchivedControlPlaneState(
.sort((left, right) => right.modifiedAt - left.modifiedAt);
for (const candidate of candidates) {
assertRealDirectory(candidate.directory);
const state = readControlPlaneState(candidate.directory);
const state = await readControlPlaneState(candidate.directory, readAuthority);
const identity = controlPlaneIdentity(state);
if (
identity.runnerInstanceId === desired.runnerInstanceId &&
@@ -275,11 +294,12 @@ function latestArchivedControlPlaneState(
return null;
}
function rotateLocalAuthorityEpoch(
async function rotateLocalAuthorityEpoch(
root: string,
controlPlaneState: Record<string, unknown>,
desired: DurableRecoveryIdentity,
): Record<string, unknown> {
runnerBinary?: string,
): Promise<Record<string, unknown>> {
const priorIdentity = controlPlaneIdentity(controlPlaneState);
if (
priorIdentity.runnerInstanceId !== desired.runnerInstanceId ||
@@ -293,9 +313,16 @@ function rotateLocalAuthorityEpoch(
}
const runnerDirectory = resolve(root, "runner");
const runnerStatePath = resolve(runnerDirectory, "runner-state.json");
const archive = authorityArchiveDirectory(root, priorIdentity);
const indexedArchive = existsSync(resolve(runnerDirectory, "runner-state.sqlite")) ||
existsSync(resolve(root, "authority-epochs", "current-archive.json"));
const archive = authorityArchiveDirectory(root, priorIdentity, indexedArchive);
const archivedControlPlane = resolve(archive, "control-plane");
const archivedRunnerState = resolve(archive, "runner-state.json");
const activeControlPlane = resolve(root, "control-plane");
if (existsSync(resolve(archive, "runner-transfer.json")) && existsSync(archivedControlPlane)) {
if (existsSync(activeControlPlane)) throw new Error("native_runner_authority_archive_conflict");
await finishIndexedRunnerArchive(runnerDirectory, archive, runnerBinary);
}
const runnerStateSource = existsSync(runnerStatePath)
? runnerStatePath
: archivedRunnerState;
@@ -303,7 +330,7 @@ function rotateLocalAuthorityEpoch(
throw new Error("native_runner_authority_rotation_state_unavailable");
}
assertRealDirectory(runnerDirectory);
const runnerState = readRunnerState(runnerStateSource);
const runnerState = await readRunnerState(runnerStateSource);
if (
runnerState.runnerInstanceId !== priorIdentity.runnerInstanceId ||
runnerState.environmentLeaseId !== priorIdentity.environmentLeaseId ||
@@ -327,15 +354,33 @@ function rotateLocalAuthorityEpoch(
mkdirSync(archive, { mode: 0o700 });
}
assertRealDirectory(archive);
const activeControlPlane = resolve(root, "control-plane");
syncArchiveDirectory(archivesRoot);
const indexed = existsSync(resolve(runnerDirectory, "runner-state.sqlite"));
if (indexed && existsSync(runnerStatePath)) {
await prepareIndexedRunnerArchive(runnerDirectory, archive, runnerBinary);
publishIndexedArchiveHead(archive);
}
if (existsSync(activeControlPlane)) {
assertRealDirectory(activeControlPlane);
if (existsSync(archivedControlPlane)) {
throw new Error("native_runner_authority_archive_conflict");
}
renameSync(activeControlPlane, archivedControlPlane);
const locatorPath = resolve(activeControlPlane, "control-plane-state.json");
const locator = lstatSync(locatorPath).size <= 8192 ? JSON.parse(readFileSync(locatorPath, "utf8")) : null;
if (locator?.schema === "paperclip.runner.authority-locator.v1" && locator.location?.kind === "sqlite") {
const snapshot = await readDurableControlPlaneSnapshot(activeControlPlane);
if (!snapshot.storedSha256 || authorityJson(snapshot.state) !== authorityJson(controlPlaneState)) {
throw new Error("native_runner_authority_archive_conflict");
}
await nativeStorageCommand(["archive-controller", "--directory", activeControlPlane, "--destination", archivedControlPlane,
"--digest", snapshot.storedSha256], runnerBinary);
} else renameSync(activeControlPlane, archivedControlPlane);
syncArchiveDirectory(root);
syncArchiveDirectory(archive);
}
if (existsSync(runnerStatePath)) {
if (existsSync(resolve(archive, "runner-transfer.json"))) {
await finishIndexedRunnerArchive(runnerDirectory, archive, runnerBinary);
} else if (existsSync(runnerStatePath)) {
if (existsSync(archivedRunnerState)) {
throw new Error("native_runner_authority_archive_conflict");
}
@@ -392,7 +437,7 @@ async function rotateExternalAuthorityEpoch(
throw new Error("native_runner_authority_archive_conflict");
}
const archivedIdentity = controlPlaneIdentity(
readControlPlaneState(archivedControlPlane),
await readControlPlaneState(archivedControlPlane),
);
if (!recoveryIdentityMatches(archivedIdentity, priorIdentity)) {
throw new Error("native_runner_authority_archive_conflict");
@@ -552,21 +597,21 @@ type ProviderDrainState =
ReturnType<typeof providerDrainStateFromSnapshot> | "unreadable" | null;
async function awaitProviderDrainBarrier(input: {
readProviderState: () => ProviderDrainState;
readProviderState: () => ProviderDrainState | Promise<ProviderDrainState>;
semanticResultsSettled: () => boolean;
commands: () => readonly {
commandId: string;
status: string;
result?: unknown;
}[];
queueDrain: (commandId: string) => void;
pump: () => void;
queueDrain: (commandId: string) => void | Promise<void>;
pump: () => void | Promise<void>;
deadline: number;
pollIntervalMs?: number;
}): Promise<boolean> {
let receiptConfirmed = false;
while (Date.now() < input.deadline) {
input.pump();
await input.pump();
// A callback is not part of the provider FIFO until its result is durably
// queued and completed. Never certify a temporarily empty prefix while
// that admitted old-authority result is still being produced.
@@ -576,7 +621,7 @@ async function awaitProviderDrainBarrier(input: {
);
continue;
}
const state = input.readProviderState();
const state = await input.readProviderState();
// Remote roots still require the exact runner-owned receipt. Their
// checkpoint separately verifies provider settlement on the remote host.
if (
@@ -594,9 +639,9 @@ async function awaitProviderDrainBarrier(input: {
continue;
}
const commandId = `command_close_drain_${randomUUID().replaceAll("-", "")}`;
input.queueDrain(commandId);
await input.queueDrain(commandId);
while (Date.now() < input.deadline) {
input.pump();
await input.pump();
const command = input
.commands()
.find((candidate) => candidate.commandId === commandId);
@@ -713,10 +758,10 @@ async function awaitRunnerSuspensionBarrier(input: {
type: string;
status: string;
}[];
queueSuspend: (commandId: string) => void;
queueSuspend: (commandId: string) => void | Promise<void>;
readRunnerState: () => Promise<Record<string, unknown>>;
runnerHasExited: () => Promise<boolean>;
pump: () => void;
pump: () => void | Promise<void>;
deadline: number;
pollIntervalMs?: number;
}): Promise<boolean> {
@@ -742,10 +787,10 @@ async function awaitRunnerSuspensionBarrier(input: {
const commandId =
existing?.commandId ??
`command_close_suspend_${randomUUID().replaceAll("-", "")}`;
if (!existing) input.queueSuspend(commandId);
if (!existing) await input.queueSuspend(commandId);
while (Date.now() < input.deadline) {
input.pump();
await input.pump();
const command = input
.commands()
.find((candidate) => candidate.commandId === commandId);
@@ -1075,6 +1120,9 @@ export interface CapabilityRunnerdProcessEvidence {
}
export interface CapabilityRunnerdCodexTransportOptions {
/** Fresh indexed authority or an already activated authority; legacy stays on its compatibility reader. */
authorityStoreFactory?: (identity: DurableRecoveryIdentity, directory: string) => Promise<DurableAuthorityStore | undefined>;
readAuthorityState?: ExternalAuthorityReader;
provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx";
opencodePermissionMode?: NativeOpenCodePermissionMode;
acpxAgent?: QualifiedAcpxAgent;
@@ -1338,6 +1386,12 @@ export function runnerdCanonicalNotificationMethod(
if (eventType === "session.goal.snapshot" && payload.goal === null) {
return undefined;
}
// Tool output is its own provider channel. Treating it as assistant prose
// loses the execution binding and pollutes the agent message history.
if (eventType === "item.delta" && payload.kind === "commandExecution") {
return payload.providerMethod === "item/commandExecution/outputDelta"
? "item/commandExecution/outputDelta" : undefined;
}
return (
{
"turn.started": "turn/started",
@@ -1792,6 +1846,7 @@ export function rehydrateRunnerdItemNotification(
type: rawItem.type ?? rawParams.kind,
status: rawItem.status ?? rawParams.status,
text: rawItem.text ?? rawParams.text,
...(rawParams.outputBody ? { outputBody: rawParams.outputBody } : {}),
...(phase === undefined ? {} : { phase }),
...(channel === undefined ? {} : { channel }),
},
@@ -2500,9 +2555,9 @@ async function settleRetainedRunnerdSessionOwned(
before.provider.activeProviderTurnId !== null &&
before.provider.activeProviderTurnId !== undefined
) {
core.queueCommand("turn.stop", {
(await core.queueCommand("turn.stop", {
reason: "exact retained authority cleanup",
});
}));
}
await core.start();
await authorize();
@@ -2515,7 +2570,7 @@ async function settleRetainedRunnerdSessionOwned(
connectUrl: core.connectUrl,
stateDirectory: resolve(root, "runner"),
identity: input.identity,
ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),
ticket: (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)),
maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
maxRuntimeMs: 30_000,
@@ -2575,7 +2630,7 @@ async function settleRetainedRunnerdSessionOwned(
const provider = providerDrainStateFromSnapshot(state.provider);
if (!terminalOnly && !terminalQueued && provider.providerSettled) {
if (!drainQueued) {
core.queueCommand("runner.drain", {}, undefined, true);
(await core.queueCommand("runner.drain", {}, undefined, true));
drainQueued = true;
} else if (
provider.pendingEventCount === 0 &&
@@ -2584,7 +2639,7 @@ async function settleRetainedRunnerdSessionOwned(
(command) => command.status === "pending",
)
) {
core.queueCommand("runner.suspend", {}, undefined, true);
(await core.queueCommand("runner.suspend", {}, undefined, true));
terminalQueued = true;
}
}
@@ -2759,10 +2814,10 @@ async function settleRetainedRunnerdSessionOwned(
pids.add(identity.processId);
}
if (
!Number.isSafeInteger(before.provider.providerProcessGeneration) ||
settled.provider.providerProcessGeneration !==
Number(before.provider.providerProcessGeneration) +
(epochRestoresProvider ? 1 : 0)
!isProviderProcessGeneration(before.provider.providerProcessGeneration) ||
(epochRestoresProvider
? !provesProviderGenerationTransition(before.provider.providerProcessGeneration, settled.provider.providerProcessGeneration, settled.provider.processGenerationTransition)
: settled.provider.providerProcessGeneration !== before.provider.providerProcessGeneration)
)
throw maintenanceDenied();
const provider = providerDrainStateFromSnapshot(settled.provider);
@@ -3306,6 +3361,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#adoptedRunnerAuthenticated = false;
#pump: NodeJS.Timeout | null = null;
#eventSourceSeq = 0;
#eventSourceEpoch?: string;
#eventIdentity: DurableRecoveryIdentity | null = null;
#pendingWarmRecoveryCompletion: {
transitionId: string;
@@ -3559,7 +3615,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
);
while (Date.now() < recoveryDeadline) {
this.#throwIfFailed();
this.#pumpEvents();
await this.#pumpEvents();
terminal = [...(this.#core?.store.state.committedEvents ?? [])]
.reverse()
.find((event) => {
@@ -3595,7 +3651,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
}
this.#recoveryTurnBindingPending = false;
this.#pumpEvents();
await this.#pumpEvents();
return {
thread: {
id: this.#threadId,
@@ -3673,6 +3729,27 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
return this.#queue;
}
normalizedDelivery() {
return this.#core?.normalizedDelivery() ?? null;
}
#beforeRunRotation: (() => Promise<void>) | null = null;
setBeforeRunRotation(settle: () => Promise<void>): void { this.#beforeRunRotation = settle; }
async #settleNormalizedDelivery(): Promise<void> {
const core = this.#core;
if (!core?.indexedPersistence) return;
const deadline = Date.now() + 30_000;
for (;;) {
await this.#pumpEvents();
await this.#beforeRunRotation?.();
const consumer = core.store.state.indexedState?.normalizedDelivery;
if (consumer && consumer.raw.sourceEpoch === core.store.state.indexedState?.sourceEpoch && consumer.raw.sourceSeq === core.store.state.ackedSourceSeq && (consumer.raw.sourceSeq === 0 || consumer.raw.ordinal === RAW_DELIVERY_FRAME_COMPLETE) && consumer.pending.length === 0) return;
if (Date.now() >= deadline) throw new Error("storage_pressure: normalized delivery has not quiesced before run rotation");
await new Promise<void>((resolve) => setTimeout(resolve, 5));
}
}
setServerRequestHandler(handler: CodexServerRequestHandler): void {
this.#handler = handler;
}
@@ -3750,6 +3827,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
throw new Error("native_runner_prp_run_rotation_unavailable");
}
await this.#awaitWarmRunAttachmentReady();
await this.#settleNormalizedDelivery();
const prior = core.store.state.identity;
const desired: DurableRecoveryIdentity = {
...prior,
@@ -3786,20 +3864,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#authorizedTools,
this.options.resumeCompletionContract,
);
this.#runAttachTemplate = structuredClone(runAttachTemplate);
if (core.indexedPersistence) (runAttachTemplate as Record<string, unknown>).durability = INDEXED_DURABILITY_CAPABILITY;
this.#runAttachTemplate = structuredClone(runAttachTemplate);
const payload = {
...runAttachTemplate,
paperclipNextAuthority: { identity: desired, connection },
};
core.queueCommand("run.attach", payload, commandId, true);
(await core.queueCommand("run.attach", payload, commandId, true));
await this.#waitCommand("run.attach", commandId);
const attached = core.getCommand(commandId);
const attached = (await core.getCommand(commandId));
if (attached?.status !== "completed") {
throw new Error("native_runner_prp_run_rotation_failed");
}
activationStarted = true;
core.rotateRunIdentity(desired, runAttachTemplate);
(await core.rotateRunIdentity(desired, runAttachTemplate));
await registration?.activate?.();
if (registration?.failure) {
void registration.failure.catch((error: unknown) => {
@@ -3826,6 +3905,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
this.#eventIdentity = structuredClone(desired);
this.#eventSourceSeq = 0;
this.#eventSourceEpoch = undefined;
this.#deferredTurnStartEvents = [];
this.#durableTurnId = desired.turnId;
await previousRelease?.();
@@ -3935,24 +4015,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
async #readDurableRunnerState(): Promise<Record<string, unknown>> {
if (this.options.readRunnerState) return this.options.readRunnerState();
return record(
JSON.parse(
readFileSync(
resolve(this.#root, "runner", "runner-state.json"),
"utf8",
),
),
);
return readRunnerState(resolve(this.#root, "runner", "runner-state.json"));
}
#providerDrainState():
| {
pendingEventCount: number;
activeProviderTurnId: string | null;
providerSettled: boolean;
}
| "unreadable"
| null {
async #providerDrainState(): Promise<ProviderDrainState> {
if (this.options.runnerFilesystemRoot !== undefined) return null;
const provider = this.options.provider ?? "codex";
const filename =
@@ -3973,7 +4039,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
};
}
try {
const state = record(JSON.parse(readFileSync(statePath, "utf8")));
const state = await readRunnerState(statePath);
return providerDrainStateFromSnapshot(state);
} catch {
return "unreadable";
@@ -3981,7 +4047,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
async #stopActiveProviderTurnBeforeSuspend(deadline: number): Promise<void> {
const state = this.#providerDrainState();
const state = await this.#providerDrainState();
const core = this.#core;
const inferredActiveProviderTurnId =
state === null &&
@@ -4001,12 +4067,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
return;
}
const commandId = `command_close_stop_${randomUUID().replaceAll("-", "")}`;
core.queueCommand(
(await core.queueCommand(
"turn.stop",
{ reason: "transport closing after durable run terminal" },
commandId,
true,
);
));
while (Date.now() < deadline) {
this.#pumpEventsSafely();
const command = core.store.state.commands.find(
@@ -4040,8 +4106,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
readProviderState: () => this.#providerDrainState(),
semanticResultsSettled: () => core.semanticToolResultsSettled(),
commands: () => core.store.state.commands,
queueDrain: (commandId) => {
core.queueCommand("runner.drain", {}, commandId, true);
queueDrain: async (commandId) => {
(await core.queueCommand("runner.drain", {}, commandId, true));
},
pump: () => this.#pumpEventsSafely(),
deadline: Date.now() + timeoutMs,
@@ -4057,7 +4123,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#diagnostic(
"provider suffix did not prove durable drain before bounded runner suspension: " +
JSON.stringify({
providerState: this.#providerDrainState(),
providerState: await this.#providerDrainState(),
semanticResultsSettled: core.semanticToolResultsSettled(),
drainStatus: lastDrain?.status ?? null,
retainedEventsDrained:
@@ -4162,8 +4228,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
suspensionRequired = true;
runnerSuspended = await awaitRunnerSuspensionBarrier({
commands: () => this.#core?.store.state.commands ?? [],
queueSuspend: (commandId) => {
this.#core?.queueCommand("runner.suspend", {}, commandId, true);
queueSuspend: async (commandId) => {
(await this.#core?.queueCommand("runner.suspend", {}, commandId, true));
},
readRunnerState: async () => {
const state = await this.#readDurableRunnerState();
@@ -4242,7 +4308,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
// provider state. Quiesce the authenticated route, then probe its
// independently verified durable state before releasing the process owner;
// an incomplete or identity-conflicting state remains fail-closed.
const finalProviderState = this.#providerDrainState();
const finalProviderState = await this.#providerDrainState();
const runnerSettled =
runnerSuspended &&
providerDrained &&
@@ -4309,7 +4375,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const dynamicTools = Array.isArray(params.dynamicTools)
? params.dynamicTools.map(record)
: [];
const core = new DurablePrpControlPlane({
const core = await DurablePrpControlPlane.open({
authorityStore: await this.options.authorityStoreFactory?.(identity, resolve(this.#root, "control-plane")),
stateDirectory: resolve(this.#root, "control-plane"),
identity,
expectedRunnerVersion: runnerArtifact.version,
@@ -4637,10 +4704,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
// Preserve the first generation's provider attachment seed independently
// of bounded command history. The in-memory copy serves a live warm
// continuation; the control-plane copy serves a controller/runner resume.
if (core.indexedPersistence) (runAttachTemplate as Record<string, unknown>).durability = INDEXED_DURABILITY_CAPABILITY;
this.#runAttachTemplate = structuredClone(runAttachTemplate);
core.persistRunAttachTemplate(runAttachTemplate);
core.queueCommand("run.prepare", runAttachTemplate);
core.queueCommand("session.open", { reuse: "same_session" });
(await core.persistRunAttachTemplate(runAttachTemplate));
(await core.queueCommand("run.prepare", runAttachTemplate));
(await core.queueCommand("session.open", { reuse: "same_session" }));
const registration = this.options.controlPlaneRegistration
? await this.options.controlPlaneRegistration(core)
: null;
@@ -4652,6 +4720,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#controlPlaneRelease = registration.release;
}
const handle = spawnRunner({
indexedDurability: core.indexedPersistence,
connection: registration?.connection ?? {
mode: "connect",
connectUrl: registration?.connectUrl ?? core.connectUrl,
@@ -4659,7 +4728,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
stateDirectory:
this.options.runnerStateDirectory ?? resolve(this.#root, "runner"),
identity,
ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),
ticket: (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)),
maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
maxRuntimeMs: 0,
@@ -4768,7 +4837,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
);
let candidateRunnerState =
localStateOwner && existsSync(localRunnerStatePath)
? readRunnerState(localRunnerStatePath)
? await readRunnerState(localRunnerStatePath)
: null;
const hasRunnerWarmBoundary =
candidateRunnerState?.warmTransition !== undefined ||
@@ -4777,10 +4846,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
let controlPlaneState: Record<string, unknown> | null;
try {
controlPlaneState = existsSync(controlPlaneStatePath)
? readControlPlaneState(controlPlaneDirectory)
? await readControlPlaneState(controlPlaneDirectory, this.options.readAuthorityState)
: null;
} catch (error) {
if (localProvider && localStateOwner && !hasRunnerWarmBoundary) {
if (localProvider && localStateOwner && !hasRunnerWarmBoundary && !this.options.authorityStoreFactory) {
quarantineLocalRuntimeState(this.#root, error);
}
throw error;
@@ -4901,19 +4970,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
if (!localProvider) {
throw new Error("PRP provider resume state is unavailable");
}
const archivedState = latestArchivedControlPlaneState(
const archivedState = await latestArchivedControlPlaneState(
this.#root,
desiredIdentity,
this.options.readAuthorityState,
);
if (!archivedState) {
throw new Error("PRP provider resume state is unavailable");
}
if (localStateOwner) {
try {
controlPlaneState = rotateLocalAuthorityEpoch(
controlPlaneState = await rotateLocalAuthorityEpoch(
this.#root,
archivedState,
desiredIdentity,
this.options.runnerBinary,
);
} catch (error) {
quarantineLocalRuntimeState(this.#root, error);
@@ -4941,10 +5012,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
if (localStateOwner) {
try {
controlPlaneState = rotateLocalAuthorityEpoch(
controlPlaneState = await rotateLocalAuthorityEpoch(
this.#root,
controlPlaneState,
desiredIdentity,
this.options.runnerBinary,
);
} catch (error) {
quarantineLocalRuntimeState(this.#root, error);
@@ -5032,7 +5104,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
opencodeExecutable!,
)
: undefined;
const core = new DurablePrpControlPlane({
const core = await DurablePrpControlPlane.open({
authorityStore: await this.options.authorityStoreFactory?.(identity, controlPlaneDirectory),
stateDirectory: controlPlaneDirectory,
identity,
expectedRunnerVersion: runnerArtifact.version,
@@ -5091,8 +5164,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
],
});
}
this.#runAttachTemplate = structuredClone(runAttachTemplate);
core.queueCommand("run.attach", runAttachTemplate);
if (core.indexedPersistence) (runAttachTemplate as Record<string, unknown>).durability = INDEXED_DURABILITY_CAPABILITY;
this.#runAttachTemplate = structuredClone(runAttachTemplate);
(await core.queueCommand("run.attach", runAttachTemplate));
}
const committedEvents = core.store.state.committedEvents;
const runAttachment = recoveredRunAttachment(core.store.state);
@@ -5104,7 +5178,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
? `command_resume_probe_${randomUUID().replaceAll("-", "")}`
: null;
if (recoveryProbeCommandId !== null) {
core.queueCommand("runner.drain", {}, recoveryProbeCommandId);
(await core.queueCommand("runner.drain", {}, recoveryProbeCommandId));
}
// A controller retry can open the exact authority after run.attach has
// already reached a durable outcome. Re-observe that command instead of
@@ -5117,6 +5191,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
? committedEvents[runAttachment.providerIdentityEventIndex]!.sourceSeq -
1
: core.store.state.ackedSourceSeq;
if (core.indexedPersistence) {
const consumer = core.normalizedDelivery()!.load();
this.#eventSourceSeq = Math.max(0, (consumer?.raw.sourceSeq ?? 0) - 1);
this.#eventSourceEpoch = consumer?.raw.sourceEpoch;
for (const pending of (consumer?.driver.pendingRequests ?? []) as Array<{ request: { requestId: string } }>) {
this.#bridgedRuntimeInputs.set(pending.request.requestId, { durableTurnId: this.#durableTurnId });
}
}
const adoptedProviderIdentityIndex =
latestProviderIdentityEventIndex(committedEvents);
if (
@@ -5268,21 +5350,22 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const bootstrapTicket = adoptedRunner
? null
: warmRecovery
? core.issueWarmTransitionBootstrapTicket(
? (await core.issueWarmTransitionBootstrapTicket(
{
transitionId: warmRecovery.transitionId,
runnerState: warmRecovery.runnerState,
},
RUNNER_BOOTSTRAP_TICKET_TTL_MS,
)
: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS);
))
: (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS));
if (warmRecovery) {
await this.options.authorizeWarmTransitionRecovery?.("before_spawn");
}
const handle = adoptedRunner
? null
: spawnRunner({
connection: registration?.connection ?? {
indexedDurability: core.indexedPersistence,
connection: registration?.connection ?? {
mode: "connect",
connectUrl: registration?.connectUrl ?? core.connectUrl,
},
@@ -5356,12 +5439,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
await new Promise<void>((resolveWait) => setTimeout(resolveWait, 10));
}
if (core.getCommand(warmRecovery.commandId)?.status !== "completed") {
if ((await core.getCommand(warmRecovery.commandId))?.status !== "completed") {
throw new Error("native_runner_warm_transition_result_unproven");
}
this.#durableTurnId = desiredIdentity.turnId;
this.#eventIdentity = structuredClone(desiredIdentity);
this.#eventSourceSeq = 0;
this.#eventSourceEpoch = undefined;
this.#deferredTurnStartEvents = [];
if (oldTransitionRegistration && newTransitionRegistration) {
await oldTransitionRegistration.release();
@@ -5592,7 +5676,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
});
while (!providerTurnStarted() && Date.now() < deadline) {
this.#throwIfFailed();
this.#pumpEvents();
await this.#pumpEvents();
if (providerTurnStarted()) break;
if (await this.#runnerHasExited())
throw new Error("runnerd exited before provider turn startup");
@@ -5655,7 +5739,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
);
}
} else {
core.queueCommand(type, payload, commandId, true);
(await core.queueCommand(type, payload, commandId, true));
}
await this.#waitCommand(type, commandId);
}
@@ -5668,9 +5752,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const core = this.#core;
if (core === null) throw new Error("PRP provider thread is not started");
const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`;
core.queueCommand(type, payload, commandId, true);
(await core.queueCommand(type, payload, commandId, true));
await this.#waitCommand(type, commandId, deadline);
const command = core.getCommand(commandId);
const command = (await core.getCommand(commandId));
if (command?.status !== "completed" || command.type !== type) {
throw new Error(`PRP command ${type} omitted its durable result`);
}
@@ -5730,7 +5814,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const deadline = Date.now() + 30_000;
while (Date.now() < deadline) {
this.#throwIfFailed();
this.#pumpEvents();
await this.#pumpEvents();
if (
this.#threadId.length > 0 &&
(this.#checkpointProviderIdentityExpectation !== null ||
@@ -5759,7 +5843,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
? this.#core?.store.state.commands.find(
(candidate) => candidate.type === type,
)
: this.#core?.getCommand(commandId);
: (await this.#core?.getCommand(commandId));
if (command?.status === "completed") return;
if (command !== undefined && command.status !== "pending") {
throw new Error(
@@ -5776,7 +5860,29 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
#pumpEvents(): void {
#eventPumpPending: Promise<void> | null = null;
#queueDelivery(event: DurableRecoveryCommittedEvent, ordinal: number, notification: CodexRpcNotification): void {
const port = this.normalizedDelivery();
if (!port) { this.#queue.push(notification); return; }
const raw = this.#core?.store.state.indexedState?.normalizedDelivery?.raw;
if (raw && raw.sourceEpoch === event.sourceEpoch && (event.sourceSeq < raw.sourceSeq || (event.sourceSeq === raw.sourceSeq && ordinal <= raw.ordinal))) return;
this.#queue.push({ ...notification, paperclipDelivery: { epoch: port.epoch, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), sourceSeq: event.sourceSeq, ordinal } });
}
#queueRawBoundary(event: DurableRecoveryCommittedEvent): void {
if (this.#core?.indexedPersistence) this.#queueDelivery(event, RAW_DELIVERY_FRAME_COMPLETE, { method: "paperclip/rawConsumed", params: {} });
}
#pumpEvents(): Promise<void> {
// Timer, drain, and command waiters share one bounded history read.
// A slow database must not accumulate one query every five milliseconds.
return this.#eventPumpPending ??= this.#readAndPumpEvents().finally(() => {
this.#eventPumpPending = null;
});
}
async #readAndPumpEvents(): Promise<void> {
const core = this.#core;
// The controller may activate its new epoch before attachRun observes the
// confirmed handoff. Never consume either epoch with the other's cursor.
@@ -5787,7 +5893,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
)
return;
this.#flushPendingTraceRehydrations();
const events = core.store.state.committedEvents;
// The database inbox absorbs downstream backpressure. Only a bounded
// delivery prefix may be materialized into the driver notification queue.
const consumed = core.store.state.indexedState?.normalizedDelivery?.raw;
if (core.indexedPersistence && (this.#eventSourceEpoch !== consumed?.sourceEpoch || this.#eventSourceSeq - (consumed?.sourceSeq ?? 0) >= 128)) return;
const readIdentity = structuredClone(this.#eventIdentity);
const events = await core.readCommittedEvents(this.#eventSourceSeq, 128, this.#eventSourceEpoch);
if (!this.#eventIdentity || !recoveryIdentityMatches(readIdentity, this.#eventIdentity) || !recoveryIdentityMatches(readIdentity, core.store.state.identity)) return;
for (;;) {
const deferredEvent =
!this.#recoveryTurnBindingPending &&
@@ -5797,7 +5909,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const event =
deferredEvent ??
events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq);
if (event === undefined) return;
if (event === undefined) {
if (core.indexedPersistence && this.#deferredTurnStartEvents.length === 0) {
const transition = await core.readEventEpochTransition(this.#eventSourceEpoch);
const raw = core.store.state.indexedState?.normalizedDelivery?.raw;
if (transition && raw && this.#eventSourceSeq === transition.finalOrdinal && raw.sourceEpoch === this.#eventSourceEpoch && raw.sourceSeq === transition.finalOrdinal && raw.ordinal === RAW_DELIVERY_FRAME_COMPLETE) {
// Publish one explicit boundary only after every preceding frame is
// durably reduced. The next pump waits for this cursor to commit.
this.#eventSourceEpoch = transition.nextEpoch;
this.#eventSourceSeq = 0;
this.#queue.push({ method: "paperclip/rawEpoch", params: {}, paperclipDelivery: { epoch: core.store.state.identity.runId, sourceEpoch: transition.nextEpoch, sourceSeq: 0, ordinal: 0 } });
}
}
return;
}
const fromDeferredQueue = deferredEvent !== undefined;
if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) {
throw new Error(
@@ -5878,6 +6003,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
event.eventType === "session.resumed"
) {
this.#applyProviderIdentityEvent(event);
if (core.indexedPersistence && !fromDeferredQueue && this.#deferredTurnStartEvents.some((earlier) => earlier.sourceSeq < event.sourceSeq)) {
// Identity can unblock thread/read while older frames are waiting
// for its turn binding. Its consumer cursor must still follow those
// frames, even though the transport needs the identity immediately.
this.#deferredTurnStartEvents.push(structuredClone(event));
continue;
}
this.#queueRawBoundary(event);
continue;
}
if (event.eventType === "harness.diagnostic") {
@@ -5924,7 +6057,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
? event.envelope.turnId
: this.#durableTurnId,
});
void this.#handler({
const bridgedRequest = {
id: requestId,
method,
params,
@@ -5932,12 +6065,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
sourceEventId: event.sourceEventId,
sourceEventType: event.eventType,
},
}).catch((error) => {
};
if (core.indexedPersistence) this.#queueDelivery(event, 1, { method: "paperclip/runtimeRequest", params: { request: bridgedRequest } });
else void this.#handler(bridgedRequest).catch((error) => {
this.#failTransport(
error instanceof Error ? error : new Error(String(error)),
);
});
}
this.#queueRawBoundary(event);
continue;
}
if (
@@ -5950,6 +6086,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
).requestId;
if (typeof requestId === "string")
this.#bridgedRuntimeInputs.delete(requestId);
this.#queueRawBoundary(event);
continue;
}
const sessionUpdatePayload = record(eventPayload);
@@ -5970,7 +6107,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
: canonicalMethod
? expandRunnerdCanonicalNotifications(canonicalMethod, eventPayload)
: [];
let deliveryOrdinal = 0;
for (const payload of notifications) {
deliveryOrdinal++;
const method = payload.method;
if (typeof method !== "string") continue;
const rawParams = record(payload.params);
@@ -6080,7 +6219,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
}
}
this.#queue.push({
this.#queueDelivery(event, deliveryOrdinal, {
method,
params,
...(this.options.environment?.PAPERCLIP_PROVIDER_TRACE_PATH
@@ -6093,6 +6232,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
: {}),
});
}
this.#queueRawBoundary(event);
if (this.options.environment?.PAPERCLIP_PROVIDER_TRACE_PATH) {
const pending = {
sourceEventId: event.sourceEventId,
@@ -6277,9 +6417,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#pendingDriverTraceInterpretations = driverRetry;
}
#pumpEventsSafely(): void {
async #pumpEventsSafely(): Promise<void> {
try {
this.#pumpEvents();
await this.#pumpEvents();
} catch (error) {
this.#failTransport(
new Error(
@@ -6477,7 +6617,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
let recoveredHandle: RunnerProcessHandle;
try {
recoveredHandle = restart(
this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),
(await this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)),
);
} catch (error) {
lastDetail = error instanceof Error ? error.message : String(error);
@@ -6544,6 +6684,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
#failTransport(error: Error): void {
if (this.#failure === null && error.message.includes("Durable authority commit is indeterminate")) this.#diagnostic(error.message);
if (this.#failure !== null || this.#closed) return;
this.#failure = error;
this.#rejectFailureSignal(error);
@@ -6587,15 +6728,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
export function defaultCapabilityRunnerdBinary(): string {
const staged = resolve(
packageRoot,
`dist/bin/paperclip-runnerd${executableSuffix}`,
);
if (existsSync(staged)) return staged;
return resolve(
packageRoot,
`runner/target/debug/paperclip-runnerd${executableSuffix}`,
);
return defaultStorageRunnerBinary();
}
/** Starts an authenticated durable PRP authority, runnerd, and Codex provider transport. */
@@ -6633,6 +6766,8 @@ export const runnerdRecoveryInternals = Object.freeze({
releaseRunnerProcessOwnership,
runnerCloseDeadlines,
rotatedRunAttachPayload,
latestArchivedControlPlaneState,
rotateLocalAuthorityEpoch,
rotateExternalAuthorityEpoch,
turnStartCommandResultValid,
turnStartNotificationDisposition,
@@ -1,3 +1,4 @@
import { advanceSourceCursor, decodeSourceCursor, encodeSourceCursor, type SourceCursor } from "./control-plane/event-epochs.js";
import { randomUUID } from "node:crypto";
import type {
@@ -904,6 +905,7 @@ async function consumeTurn(
const consumer = (async () => {
let eventCount = 0;
let highestContiguousSourceSeq = 0;
let highestContiguousSourceEpoch: string | undefined;
let governedResult: PrpStructuredRunResult | null = null;
let semanticResultProposal: PrpStructuredRunResult | null = null;
let sessionGoalObserved = sessionGoal !== undefined;
@@ -934,6 +936,7 @@ async function consumeTurn(
event,
eventCount,
highestContiguousSourceSeq,
highestContiguousSourceEpoch,
governedResult: result,
};
};
@@ -966,12 +969,11 @@ async function consumeTurn(
const receipt = await controlPlane.appendEvent(event, {
signal: appendAbort.signal,
});
await session.acknowledgeEvent?.(event);
if (stopConsumer) throw new Error("native event consumer stopped");
eventCount += receipt.disposition === "committed" ? 1 : 0;
highestContiguousSourceSeq = Math.max(
highestContiguousSourceSeq,
receipt.highestContiguousSourceSeq,
);
highestContiguousSourceSeq = receipt.highestContiguousSourceSeq;
highestContiguousSourceEpoch = receipt.highestContiguousSourceEpoch;
const eventGoal = goalFromEvent(event);
const goalChanged = eventGoal !== undefined &&
goalLifecycleFingerprint(eventGoal) !== goalLifecycleFingerprint(previousGoal);
@@ -1127,6 +1129,7 @@ async function consumeTurn(
event,
eventCount,
highestContiguousSourceSeq,
highestContiguousSourceEpoch,
governedResult:
governedResult ??
(goalStatus(eventGoal) === "complete" ? semanticResultProposal : null) ??
@@ -1146,6 +1149,7 @@ async function consumeTurn(
event,
eventCount,
highestContiguousSourceSeq,
highestContiguousSourceEpoch,
governedResult:
governedResult ??
(goalStatus(latestSessionGoal) === "complete" ? semanticResultProposal : null) ??
@@ -1173,6 +1177,7 @@ async function consumeTurn(
event,
eventCount,
highestContiguousSourceSeq,
highestContiguousSourceEpoch,
governedResult,
};
}
@@ -1481,6 +1486,15 @@ function checkpointCursor(cursor: string | null | undefined): number {
return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0;
}
/** This is an allocation high-water mark, not a contiguous delivery ACK.
* Retained ledgers can have holes, so never reuse a larger persisted ordinal.
* Crossing namespaces still requires the exact predecessor transition. */
function advanceRecoveryCursor(cursor: SourceCursor, event: PrpEvent): SourceCursor {
if (cursor.sourceEpoch !== event.sourceEpoch) return advanceSourceCursor(cursor, event);
if (event.sourceEpochTransition || !Number.isSafeInteger(event.sourceSeq) || event.sourceSeq <= cursor.sourceSeq) throw new Error("native_recovery_replay_did_not_advance");
return { sourceSeq: event.sourceSeq, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}) };
}
async function reconcileRecoveryCursor(input: {
controlPlane: ControlPlanePort;
checkpoint: PersistedNativeSession;
@@ -1488,38 +1502,16 @@ async function reconcileRecoveryCursor(input: {
sourceInstanceId: string;
signal: AbortSignal;
}): Promise<PersistedNativeSession> {
const checkpointHighWater = checkpointCursor(input.checkpoint.cursor);
let afterSourceSeq = checkpointHighWater;
let persistedHighWater = checkpointHighWater;
while (true) {
const replay = await input.controlPlane.replayEvents(
{
runId: input.runId,
sourceInstanceId: input.sourceInstanceId,
afterSourceSeq,
limit: 1_000,
},
{ signal: input.signal },
);
let cursor = decodeSourceCursor(input.checkpoint.cursor);
for (;;) {
const replay = await input.controlPlane.replayEvents({ runId: input.runId, sourceInstanceId: input.sourceInstanceId,
afterSourceSeq: cursor.sourceSeq, sourceEpoch: cursor.sourceEpoch, limit: 1_000 }, { signal: input.signal });
input.signal.throwIfAborted();
if (replay.events.length === 0) break;
const pageHighWater = replay.events.reduce(
(highest, event) => Math.max(highest, event.sourceSeq),
afterSourceSeq,
);
if (pageHighWater <= afterSourceSeq) {
throw new Error("native_recovery_replay_did_not_advance");
}
persistedHighWater = Math.max(persistedHighWater, pageHighWater);
afterSourceSeq = pageHighWater;
if (!replay.events.length) break;
for (const event of replay.events) cursor = advanceRecoveryCursor(cursor, event);
}
if (
persistedHighWater === checkpointHighWater &&
input.checkpoint.cursor === String(checkpointHighWater)
) {
return input.checkpoint;
}
return { ...input.checkpoint, cursor: String(persistedHighWater) };
const encoded = encodeSourceCursor(cursor);
return input.checkpoint.cursor === encoded ? input.checkpoint : { ...input.checkpoint, cursor: encoded };
}
async function replayCheckpointedTurnTerminal(input: {
@@ -1529,63 +1521,22 @@ async function replayCheckpointedTurnTerminal(input: {
priorTerminalTurnIds: readonly string[];
expectedTurnId?: string | null;
}): Promise<{ terminal: PrpEvent; hasPriorResultProposal: boolean } | null> {
let afterSourceSeq = 0;
const terminals: PrpEvent[] = [];
const latestResultProposalByTurn = new Map<string, number>();
let cursor: SourceCursor = { sourceSeq: 0 };
let latest: { terminal: PrpEvent; hasPriorResultProposal: boolean } | null = null;
const priorTerminalTurnIds = new Set(input.priorTerminalTurnIds);
while (true) {
const replay = await input.controlPlane.replayEvents({
runId: input.runId,
sourceInstanceId: input.sourceInstanceId,
afterSourceSeq,
limit: 1_000,
});
if (replay.events.length === 0) {
// Disposition recovery owns the newest durable provider terminal. An
// older task turn may also have a valid proposal, but selecting it would
// finalize stale work and strand the actual recovery terminal.
const terminal =
[...terminals].sort(
(left, right) => right.sourceSeq - left.sourceSeq,
)[0] ?? null;
const proposalSequence =
latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0;
return terminal === null
? null
: {
terminal,
hasPriorResultProposal:
proposalSequence > 0 && proposalSequence < terminal.sourceSeq,
};
}
// A result proposal is only relevant to the currently inspected turn.
let proposedTurn: string | undefined;
for (;;) {
const replay = await input.controlPlane.replayEvents({ runId: input.runId, sourceInstanceId: input.sourceInstanceId,
afterSourceSeq: cursor.sourceSeq, sourceEpoch: cursor.sourceEpoch, limit: 1_000 });
if (!replay.events.length) return latest;
for (const event of replay.events) {
if (event.turnId && event.eventType === "run.result.proposed") {
latestResultProposalByTurn.set(
event.turnId,
Math.max(
latestResultProposalByTurn.get(event.turnId) ?? 0,
event.sourceSeq,
),
);
}
if (
event.turnId &&
isTurnTerminal(event) &&
(input.expectedTurnId !== undefined && input.expectedTurnId !== null
? event.turnId === input.expectedTurnId
: !priorTerminalTurnIds.has(event.turnId))
) {
terminals.push(structuredClone(event));
cursor = advanceRecoveryCursor(cursor, event);
if (event.turnId && event.eventType === "run.result.proposed") proposedTurn = event.turnId;
if (event.turnId && isTurnTerminal(event) && (input.expectedTurnId != null ? event.turnId === input.expectedTurnId : !priorTerminalTurnIds.has(event.turnId))) {
latest = { terminal: structuredClone(event), hasPriorResultProposal: proposedTurn === event.turnId };
}
}
const pageHighWater = replay.events.reduce(
(highest, event) => Math.max(highest, event.sourceSeq),
afterSourceSeq,
);
if (pageHighWater <= afterSourceSeq) {
throw new Error("native_recovery_replay_did_not_advance");
}
afterSourceSeq = pageHighWater;
}
}
@@ -1652,21 +1603,22 @@ async function replayProvesEffectFreeInitialAcpxTurn(input: {
}
const targetTurnId = terminalTurns[0]!.turnId;
const events: PrpEvent[] = [];
let afterSourceSeq = 0;
let cursor: SourceCursor = { sourceSeq: 0 };
try {
while (true) {
const replay = await input.controlPlane.replayEvents({
runId: input.runId,
sourceInstanceId: input.sourceInstanceId,
afterSourceSeq,
afterSourceSeq: cursor.sourceSeq,
sourceEpoch: cursor.sourceEpoch,
limit: 1_000,
});
if (replay.events.length === 0) break;
for (const event of replay.events) {
// An incomplete or reordered replay cannot prove absence of work.
if (event.sourceSeq !== afterSourceSeq + 1) return false;
cursor = advanceSourceCursor(cursor, event);
events.push(structuredClone(event));
afterSourceSeq = event.sourceSeq;
if (events.length > 10_000) return false;
}
}
@@ -2035,6 +1987,9 @@ export async function executeNativeSession(
throw error;
}
sessionOriginRunnerInstances.set(session, options.runnerInstanceId);
session.setEventCommitter?.(async (event) => {
await options.controlPlane.appendEvent(event, { signal: AbortSignal.timeout(30_000) });
});
let sessionClosePromise: Promise<void> | null = null;
let sessionQuarantined = false;
const quarantineSession = (reason: string) => {
@@ -2189,6 +2144,7 @@ export async function executeNativeSession(
event: null as PrpEvent | null,
eventCount: 0,
highestContiguousSourceSeq: 0,
highestContiguousSourceEpoch: undefined as string | undefined,
governedResult: null as PrpStructuredRunResult | null,
};
const completionSnapshot =
@@ -2205,6 +2161,7 @@ export async function executeNativeSession(
completedSemanticResultTurnId(completionSnapshot),
}
: null;
if (completed) await session.flushEventDelivery?.();
if (!completed) {
// A recovered driver is authoritative about whether a provider turn is
// still active. In particular, drivers normalize the checkpoint race
@@ -2295,6 +2252,7 @@ export async function executeNativeSession(
eventCount: 0,
highestContiguousSourceSeq:
replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq,
highestContiguousSourceEpoch: replayedDisposition === null ? undefined : recoveryTerminal.sourceEpoch,
governedResult: null,
});
// Event consumption must begin before startTurn so an eager provider cannot
@@ -2582,7 +2540,7 @@ export async function executeNativeSession(
}
}
}
consumed.highestContiguousSourceSeq = Math.max(
if (!consumed.highestContiguousSourceEpoch) consumed.highestContiguousSourceSeq = Math.max(
consumed.highestContiguousSourceSeq,
controlReplay.highestContiguousSourceSeq,
);
@@ -2599,7 +2557,7 @@ export async function executeNativeSession(
accountedControlEventSequences.add(event.sourceSeq);
consumed.eventCount += 1;
}
consumed.highestContiguousSourceSeq = Math.max(
if (!consumed.highestContiguousSourceEpoch) consumed.highestContiguousSourceSeq = Math.max(
consumed.highestContiguousSourceSeq,
receipt.highestContiguousSourceSeq,
);
@@ -2639,6 +2597,7 @@ export async function executeNativeSession(
driverKind: descriptor.name,
nativeEventCount: consumed.eventCount,
highestContiguousSourceSeq: consumed.highestContiguousSourceSeq,
...(consumed.highestContiguousSourceEpoch ? { highestContiguousSourceEpoch: consumed.highestContiguousSourceEpoch } : {}),
};
},
});
@@ -33,6 +33,16 @@ function envelope(
}
describe("provider-neutral events", () => {
it.each([1, 2])("accepts lossless tool output references in PRP v%s without allowing arbitrary fields", version => {
const [event] = canonicalProviderEventsFromCodex("item/completed", { item: {
id: "command-output", type: "commandExecution", command: "qualification", status: "completed", aggregatedOutput: "x".repeat(5000), exitCode: 0,
} });
const reference = { schema: "paperclip.output.body.v1", bodyId: "a".repeat(64), sha256: "a".repeat(64), byteLength: "5000", mediaType: "text/plain; charset=utf-8" };
const input = { ...envelope(event), schema: `paperclip.prp.event.v${version}`, schemaVersion: version, payload: { ...event.payload, outputBody: reference } };
expect(validatePrpEvent(input).ok).toBe(true);
expect(validatePrpEvent({ ...input, payload: { ...input.payload, outputBody: { ...reference, byteLength: 5000 } } }).ok).toBe(false);
expect(validatePrpEvent({ ...input, payload: { ...input.payload, unexpected: true } }).ok).toBe(false);
});
it("preserves Codex notice summaries with legacy and empty-message fallbacks", () => {
for (const method of ["configWarning", "deprecationNotice", "warning"]) {
for (const [params, expected] of [
@@ -78,6 +78,7 @@ export interface TypedEventFamilyCapability {
}
export const CANONICAL_PROVIDER_EVENT_TYPES = [
"output.body.chunk",
"harness.diagnostic",
"plan.updated",
"tool.execution.started",
@@ -0,0 +1,34 @@
import { isAbsolute, resolve } from "node:path";
/** Resolve a Cargo-built test fixture without depending on the shared default
* target directory. Relative CARGO_TARGET_DIR values are relative to Cargo's
* workspace (the runner crate workspace), matching Cargo's build invocation.
*/
export function resolveRunnerCargoTestBinary(
runnerWorkspace: string,
profile: "debug" | "release",
binaryName: string,
): string {
const configuredTarget = process.env.CARGO_TARGET_DIR;
const targetDirectory = configuredTarget
? isAbsolute(configuredTarget)
? configuredTarget
: resolve(runnerWorkspace, configuredTarget)
: resolve(runnerWorkspace, "target");
const executable = `${binaryName}${process.platform === "win32" ? ".exe" : ""}`;
return resolve(targetDirectory, profile, executable);
}
/** Preserve existing staged/default resolution when no isolated Cargo target
* is configured, while honoring CARGO_TARGET_DIR for fixture runs.
*/
export function resolveRunnerCargoTestBinaryOrDefault(
runnerWorkspace: string,
profile: "debug" | "release",
binaryName: string,
defaultBinary: () => string,
): string {
return process.env.CARGO_TARGET_DIR
? resolveRunnerCargoTestBinary(runnerWorkspace, profile, binaryName)
: defaultBinary();
}
@@ -0,0 +1,27 @@
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { visitCommandLines, writeTopLevelNames } from "./file-sync-stream.js";
it("streams a listing beyond 32 MiB without retaining its lines", async () => {
let count = 0;
await visitCommandLines(process.execPath, ["-e", `const fs=require('fs'); for(let i=0;i<10000;i++) fs.writeSync(1,'x'.repeat(4096)+'\\n');`], line => { expect(line.length).toBe(4096); count++; });
expect(count).toBe(10_000);
}, 30_000);
it("bounds an individual entry and propagates child failure", async () => {
await expect(visitCommandLines(process.execPath, ["-e", "process.stdout.write('x'.repeat(70000))"], () => undefined)).rejects.toThrow("too long");
await expect(visitCommandLines(process.execPath, ["-e", "process.exit(7)"], () => undefined)).rejects.toThrow("listing failed");
});
it("writes literal NUL-delimited names without shell expansion or a directory argv", async () => {
const root = await mkdtemp(join(tmpdir(), "daytona-catalog-"));
try {
const source = join(root, "source"), names = join(root, "names"); await mkdir(source);
const entries = ["-C", "a name", "line\nbreak", "$(not-a-command)"];
for (const name of entries) await writeFile(join(source, name), "x");
await writeTopLevelNames(source, names);
expect((await readFile(names, "utf8")).split("\0").filter(Boolean).sort()).toEqual(entries.map(name => `./${name}`).sort());
} finally { await rm(root, { recursive: true, force: true }); }
});
@@ -0,0 +1,37 @@
import { spawn } from "node:child_process";
import { open, opendir } from "node:fs/promises";
/** NUL-delimited disk catalog; filenames never become a growing argv array. */
export async function writeTopLevelNames(directory: string, destination: string): Promise<void> {
const file = await open(destination, "wx", 0o600);
try {
for await (const entry of await opendir(directory, { bufferSize: 128 })) await file.writeFile(`./${entry.name}\0`);
} finally { await file.close(); }
}
/** Stream a tar listing with a per-entry bound instead of a total listing cap. */
export async function visitCommandLines(command: string, args: string[], visit: (line: string) => void): Promise<void> {
const child = spawn(command, args, { env: { ...process.env, COPYFILE_DISABLE: "1", LC_ALL: "C" }, stdio: ["ignore", "pipe", "pipe"] });
let pending = "", diagnosticBytes = 0;
const maxLine = 64 * 1024;
const finished = new Promise<void>((resolve, reject) => {
child.once("error", reject);
child.once("close", code => code === 0 ? resolve() : reject(new Error("Daytona sync archive listing failed")));
});
void finished.catch(() => undefined);
child.stderr.on("data", (chunk: Buffer) => { diagnosticBytes += chunk.length; if (diagnosticBytes > maxLine) child.kill(); });
child.stdout.setEncoding("utf8");
try {
for await (const chunk of child.stdout) {
pending += String(chunk); let end: number;
while ((end = pending.indexOf("\n")) >= 0) {
if (end > maxLine) throw new Error("Daytona sync archive entry listing too long");
visit(pending.slice(0, end)); pending = pending.slice(end + 1);
}
if (pending.length > maxLine) throw new Error("Daytona sync archive entry listing too long");
}
if (pending) visit(pending);
await finished;
if (diagnosticBytes > maxLine) throw new Error("Daytona sync archive listing failed");
} catch (error) { child.kill(); await finished.catch(() => undefined); throw error; }
}
@@ -1,3 +1,4 @@
import { visitCommandLines, writeTopLevelNames } from "./file-sync-stream.js";
import path from "node:path";
import os from "node:os";
import { promises as fs, createReadStream, createWriteStream } from "node:fs";
@@ -164,32 +165,13 @@ async function createHostTarball(input: {
followSymlinks?: boolean;
}): Promise<void> {
const excludeArgs = ["._*", ...(input.exclude ?? [])].flatMap((entry) => ["--exclude", entry]);
const entries = (await fs.readdir(input.localDir)).sort((left, right) => left.localeCompare(right));
if (entries.length === 0) {
// An empty source is valid (blank workspace / empty asset dir). Write a valid
// gzip-compressed empty tar (1024-byte zero EOF marker) so extraction is a
// clean no-op and uses the same transport as non-empty directories.
await fs.writeFile(input.archivePath, await new Promise<Buffer>((resolve, reject) => {
zlib.gzip(Buffer.alloc(1024), (error, compressed) => error ? reject(error) : resolve(compressed));
}));
return;
}
await execFileAsync(
"tar",
[
"-cz",
"--no-xattrs",
...(input.followSymlinks ? ["-h"] : []),
"-f",
input.archivePath,
"-C",
input.localDir,
...excludeArgs,
"--",
...entries,
],
{ env: { ...process.env, COPYFILE_DISABLE: "1" }, maxBuffer: 32 * 1024 * 1024 },
);
const namesPath = `${input.archivePath}.names`;
try {
await writeTopLevelNames(input.localDir, namesPath);
await execFileAsync("tar", ["-cz", "--no-xattrs", ...(input.followSymlinks ? ["-h"] : []),
"-f", input.archivePath, "-C", input.localDir, ...excludeArgs, "--null", "-T", namesPath],
{ env: { ...process.env, COPYFILE_DISABLE: "1" }, maxBuffer: 64 * 1024 });
} finally { await fs.rm(namesPath, { force: true }); }
}
/**
@@ -251,12 +233,8 @@ export function splitLinkEntryOnce(field: string, delimiter: string): { name: st
* targets are inspected; any unparseable line fails closed.
*/
async function assertTarballEntriesConfined(archivePath: string): Promise<void> {
const { stdout } = await execFileAsync("tar", ["-tvf", archivePath], {
env: { ...process.env, COPYFILE_DISABLE: "1" },
maxBuffer: 32 * 1024 * 1024,
});
const lines = stdout.split("\n").filter((line) => line.trim().length > 0);
for (const line of lines) {
await visitCommandLines("tar", ["-tvf", archivePath], (line) => {
if (!line.trim()) return;
const parsed = parseTarVerboseListingLine(line);
if (!parsed) {
throw new Error(`Daytona syncOut refusing tarball with an unparseable entry listing: ${line}`);
@@ -287,7 +265,7 @@ async function assertTarballEntriesConfined(archivePath: string): Promise<void>
);
}
}
}
});
}
async function extractHostTarball(input: { archivePath: string; localDir: string }): Promise<void> {
@@ -305,8 +283,7 @@ async function countHostFiles(root: string, exclude?: string[]): Promise<number>
const excludeSet = new Set(exclude ?? []);
let total = 0;
const walk = async (dir: string): Promise<void> => {
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
for (const entry of entries) {
for await (const entry of await fs.opendir(dir, { bufferSize: 128 })) {
if (excludeSet.has(entry.name)) continue;
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
@@ -1183,14 +1160,14 @@ async function syncOutDirectoryMapping(input: {
// Tar the source in-sandbox (naming top-level entries so no "." self-entry is
// embedded), reproducing the `followSymlinks` → `-h` mapping, then stream the
// single archive back over the native bulk channel.
const namesPath = `${remoteTar}.names`;
const tarScript = [
"set -e",
`cd ${shellQuote(mapping.sourcePath)}`,
"set -- *",
'if [ "$#" -eq 1 ] && [ "$1" = "*" ] && [ ! -e "$1" ] && [ ! -L "$1" ]; then set --; fi',
'for entry in .[!.]* ..?*; do [ -e "$entry" ] || [ -L "$entry" ] || continue; set -- "$@" "$entry"; done',
`if [ "$#" -eq 0 ]; then dd if=/dev/zero of=${shellQuote(remoteTar)} bs=1024 count=1; ` +
`else tar -c --no-xattrs ${mapping.followSymlinks ? "-h " : ""}${excludeFlags} -f ${shellQuote(remoteTar)} -- "$@"; fi`,
].join(" && ");
`trap ${shellQuote(`rm -f -- ${shellQuote(namesPath)}`)} EXIT`,
`find . -mindepth 1 -maxdepth 1 -print0 > ${shellQuote(namesPath)}`,
`tar -c --no-xattrs ${mapping.followSymlinks ? "-h " : ""}${excludeFlags} -f ${shellQuote(remoteTar)} --null -T ${shellQuote(namesPath)}`,
].join("; ");
await assertSandboxCommandOk(sandbox, `sh -c ${shellQuote(tarScript)}`, timeoutSeconds, "syncOut tar");
guardRoundTrips += 1;
+21 -21
View File
@@ -157,12 +157,12 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) {
await delay(10);
}
};
const launch = (core, iteration) =>
const launch = async (core, iteration) =>
spawnRunner({
connectUrl: core.connectUrl,
stateDirectory: runnerDirectory,
identity,
ticket: core.issueBootstrapTicket(60_000),
ticket: (await core.issueBootstrapTicket(60_000)),
maxOutboxBytes: 16 * 1024 * 1024,
p0ReserveBytes: 1024 * 1024,
maxRuntimeMs: 10_000,
@@ -302,7 +302,7 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) {
let core = makeCore();
let handle;
let firstExit;
const prepare = core.queueCommand(
const prepare = (await core.queueCommand(
"run.prepare",
{
provider: {
@@ -329,30 +329,30 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) {
},
},
"startup-prepare",
);
const open = core.queueCommand("session.open", {}, "startup-open");
));
const open = (await core.queueCommand("session.open", {}, "startup-open"));
try {
await core.start();
handle = launch(core, 1);
handle = await launch(core, 1);
// Atomic controller commits publish new snapshots. Observe commands by
// stable ID instead of waiting on the original queueCommand object.
await waitUntil(
() =>
(core.getCommand(open.commandId)?.status ?? "pending") !== "pending",
async () =>
((await core.getCommand(open.commandId))?.status ?? "pending") !== "pending",
);
assert.equal(
core.getCommand(prepare.commandId)?.status,
(await core.getCommand(prepare.commandId))?.status,
"completed",
"prepare_must_succeed",
);
assert.equal(
core.getCommand(open.commandId)?.status,
(await core.getCommand(open.commandId))?.status,
"failed",
"missing_thread_must_fail",
);
assert.ok(
core
.getCommand(open.commandId)
(await core
.getCommand(open.commandId))
?.result?.result?.message?.includes(
`no rollout found for thread id ${missingThread}`,
),
@@ -451,31 +451,31 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) {
])
await copyFile(source, join(snapshots, name));
const firstFailure = structuredClone(
core.getCommand(open.commandId)?.result,
(await core.getCommand(open.commandId))?.result,
);
const ledgerBefore = await readFile(ledger);
const identityLedgerBefore = await readFile(identityLedger);
core = makeCore();
const snapshot = core.queueCommand(
const snapshot = (await core.queueCommand(
"session.snapshot",
{},
"reopened-snapshot",
);
const reopen = core.queueCommand("session.open", {}, "reopened-open");
));
const reopen = (await core.queueCommand("session.open", {}, "reopened-open"));
handle = null;
let secondExit;
try {
await core.start();
handle = launch(core, 2);
handle = await launch(core, 2);
await waitUntil(
() =>
(core.getCommand(snapshot.commandId)?.status ?? "pending") !==
async () =>
((await core.getCommand(snapshot.commandId))?.status ?? "pending") !==
"pending" &&
(core.getCommand(reopen.commandId)?.status ?? "pending") !==
((await core.getCommand(reopen.commandId))?.status ?? "pending") !==
"pending",
);
for (const queued of [snapshot, reopen]) {
const command = core.getCommand(queued.commandId);
const command = (await core.getCommand(queued.commandId));
assert.ok(command, "reopened_command_must_exist");
assert.equal(command.status, "failed");
assert.ok(