diff --git a/packages/paperclip-runner/spec/operation-groups/source.json b/packages/paperclip-runner/spec/operation-groups/source.json index 2d1cecf0f87..571c221d980 100644 --- a/packages/paperclip-runner/spec/operation-groups/source.json +++ b/packages/paperclip-runner/spec/operation-groups/source.json @@ -11,6 +11,7 @@ "session": "Provider-neutral session open, resume, reconciliation, close, and failure.", "turn": "Model turn submission through terminal turn disposition.", "item": "Provider-neutral model/tool item lifecycle.", + "output": "Negotiated, immutable output-body chunks with exact byte offsets and content digests.", "mcp_app": "MCP App discovery, initialization, tool, action, host-context, and teardown evidence.", "runtime_request": "Runtime permission/input request lifecycle.", "interaction": "Issue-thread interaction proposal, materialization, response, delivery, and rejection.", diff --git a/packages/paperclip-runner/spec/paperclip-agent-operation-groups.md b/packages/paperclip-runner/spec/paperclip-agent-operation-groups.md index b6bfa691674..31a65d7a84a 100644 --- a/packages/paperclip-runner/spec/paperclip-agent-operation-groups.md +++ b/packages/paperclip-runner/spec/paperclip-agent-operation-groups.md @@ -6,7 +6,7 @@ Status: canonical explanatory contract for the Paperclip runner V1 surface. This document keeps three independent meanings of **group** separate. PRP families describe wire evidence and controller commands; capability placement decides who owns an operation; behavioral eval groups organize the 106 scenario corpus. None of the three axes can be used as a substitute for another. -The generated totals are **105 PRP events in 31 event families**, **18 controller commands in 7 command families**, **10 control-plane operations**, **48 reconciled semantic operations** (14 always, 34 optional), and **106 scenarios in 16 behavior groups**. +The generated totals are **106 PRP events in 32 event families**, **18 controller commands in 7 command families**, **10 control-plane operations**, **48 reconciled semantic operations** (14 always, 34 optional), and **106 scenarios in 16 behavior groups**. ## Axis 1: PRP v1 event and command families @@ -37,6 +37,7 @@ PRP records ordered, replayable execution evidence. It is not the model's Paperc | `provider` | Redacted provider notices and actionable warnings. | `provider.notice.recorded` | 1 | | `session` | Provider-neutral session open, resume, reconciliation, close, and failure. | `session.starting`
`session.started`
`session.resuming`
`session.resumed`
`session.reconciled`
`session.updated`
`session.closed`
`session.failed` | 8 | | `turn` | Model turn submission through terminal turn disposition. | `turn.submitted`
`turn.accepted`
`turn.started`
`turn.completed`
`turn.failed`
`turn.interrupted`
`turn.cancelled` | 7 | +| `output` | Negotiated, immutable output-body chunks with exact byte offsets and content digests. | `output.body.chunk` | 1 | | `item` | Provider-neutral model/tool item lifecycle. | `item.started`
`item.delta`
`item.completed`
`item.failed` | 4 | | `usage` | Provider/model-attributed usage and accounting boundaries. | `usage.reported` | 1 | | `semantic_tool` | Canonical authorized Paperclip tool input and result evidence. | `semantic_tool.input`
`semantic_tool.result`
`semantic_tool.reconciled` | 3 | diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 317f80e5a13..6cf9fbbb87b 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -4,6 +4,7 @@ import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../ import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; +import { authorityJson } from "../control-plane/durable-authority-store.js"; const result: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", @@ -97,6 +98,26 @@ const driver: HarnessDriver = { }; describe("HarnessDriverBackend", () => { + it.each([false, true])("recovers a committed terminal after delivery ACK and validates its binding (conflict: %s)", async (conflict) => { + class DurableTerminalSession extends FakeHarnessSession { + override async *events(): AsyncIterable { /* Already acknowledged before the crash. */ } + override async snapshot(): Promise { + return { + ...(await super.snapshot()), activeTurnId: null, + semanticResult: { result, fingerprint: authorityJson(result), turnId: "turn-1" }, + durableTerminal: { event: { ...prpEvent(2, "turn.completed", { status: "completed" }), ...(conflict ? { normalizedSessionId: "other-session" } : {}) }, semanticFingerprint: authorityJson(result) }, + }; + } + } + const backend = new HarnessDriverBackend({ ...driver, openSession: async () => new DurableTerminalSession() }); + const session = await backend.openSession({ identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" } }); + if (conflict) await expect(session.snapshot()).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError); + else { + expect((await session.snapshot()).terminal).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + expect(await session.result()).toMatchObject({ result, turnId: "turn-1" }); + } + }); + it.each([false, true])("honors driver steering support even when the transport exposes a steer method (%s)", async supported => { const steer = vi.fn(async () => { throw new Error("provider does not support steering"); }); const session = Object.assign(new FakeHarnessSession(), { steer }); diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 135724c5421..e8a26f2c08a 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -1,3 +1,4 @@ +import { decodeSourceCursor, encodeSourceCursor, normalizedEventId } from "../control-plane/event-epochs.js"; import type { HarnessDriver, HarnessSession, @@ -17,6 +18,7 @@ import type { PrpEvent, PrpTerminalState, } from "../protocol/replay-contract.js"; +import { validatePrpEvent } from "../protocol/replay-contract.js"; const MAX_RECOVERY_TERMINAL_TURNS = 4_096; const MAX_RECOVERY_TERMINAL_BYTES = 8 * 1024 * 1024; @@ -133,6 +135,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { normalizedSessionId: snapshot.identity.sessionId, activeTurnId, lastSourceSequence: parseCursor(snapshot.cursor), + ...(decodeSourceCursor(snapshot.cursor).sourceEpoch ? { lastSourceEpoch: decodeSourceCursor(snapshot.cursor).sourceEpoch } : {}), ...(snapshot.semanticResult === undefined || snapshot.semanticResult === null ? {} @@ -393,6 +396,7 @@ class HarnessNativeSession implements NativeSession { readonly #session: HarnessSession; #terminal: PrpTerminalState | null = null; #explicitlyCancelled = false; + #lastDriverEventId: string | null = null; #protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null; #assertProtocolIntegrity(): void { @@ -412,6 +416,24 @@ class HarnessNativeSession implements NativeSession { try { const snapshot = await this.#session.snapshot(); this.#assertProtocolIntegrity(); + // The indexed driver has an explicit terminal fact even if the previous + // process died after its run-log ACK but before orchestration checkpointed. + const durable = snapshot.durableTerminal; + if (!this.#explicitlyCancelled && this.#terminal === null && durable && snapshot.activeTurnId === null && snapshot.semanticResult) { + const event = durable.event; + if (!validatePrpEvent(event).ok || event.runId !== this.#input.identity.runId || event.normalizedSessionId !== this.#input.identity.sessionId + || event.turnId !== snapshot.semanticResult.turnId || durable.semanticFingerprint !== snapshot.semanticResult.fingerprint + || durable.semanticFingerprint !== canonicalJson(snapshot.semanticResult.result) + || !["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) { + throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); + } + this.#terminal = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: event.eventType === "turn.completed" ? "completed" : event.eventType === "turn.failed" ? "failed" : event.eventType === "turn.interrupted" ? "interrupted" : "cancelled", + runTerminalState: event.eventType === "turn.completed" ? "succeeded" : event.eventType === "turn.failed" ? "failed" : "cancelled", + reportedWorkDisposition: snapshot.semanticResult.result.reportedWorkDisposition, + }; + } return snapshot; } catch (error) { this.#rethrowProtocolIntegrity(error); @@ -504,6 +526,7 @@ class HarnessNativeSession implements NativeSession { this.#assertProtocolIntegrity(); let sourceInstanceId: string | null = null; let lastSourceSequence = 0; + let lastSourceEpoch: string | undefined; let sawTerminal = false; let synthesizedDurableWait = false; let streamFailure: unknown = null; @@ -517,7 +540,8 @@ class HarnessNativeSession implements NativeSession { event.payload.kind === "interrupt_acknowledgement"); if (this.#explicitlyCancelled && !isCancellationEvent) continue; sourceInstanceId = event.sourceInstanceId; - lastSourceSequence = Math.max(lastSourceSequence, event.sourceSeq); + lastSourceSequence = event.sourceSeq; + lastSourceEpoch = event.sourceEpoch; if (event.eventType === "runtime_request.created") { const request = plainRecord(event.payload.request); if ( @@ -577,6 +601,7 @@ class HarnessNativeSession implements NativeSession { reportedWorkDisposition: disposition, }; } + this.#lastDriverEventId = event.sourceEventId; yield structuredClone(event); } } catch (error) { @@ -596,7 +621,8 @@ class HarnessNativeSession implements NativeSession { let governedWaitTurnId: string | undefined; for (const request of observedPendingInputs.values()) { const sourceSeq = - Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + Math.max(lastSourceSequence, snapshot?.lastSourceEpoch === lastSourceEpoch ? snapshot?.lastSourceSequence ?? 0 : 0) + 1; + if (!Number.isSafeInteger(sourceSeq)) throw new Error("native_synthetic_event_epoch_required"); lastSourceSequence = sourceSeq; const requestId = String(request.requestId); const turnId = @@ -606,7 +632,8 @@ class HarnessNativeSession implements NativeSession { typeof request.itemId === "string" ? request.itemId : requestId; yield { schema: "paperclip.prp.event.v1", - sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceEventId: normalizedEventId(sourceInstanceId, this.#input.identity.runId, sourceSeq, lastSourceEpoch), + ...(lastSourceEpoch ? { sourceEpoch: lastSourceEpoch } : {}), sourceSeq, sourceInstanceId, sourceKind: "runner", @@ -635,7 +662,8 @@ class HarnessNativeSession implements NativeSession { } if (governedWaitTurnId) { const sourceSeq = - Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + Math.max(lastSourceSequence, snapshot?.lastSourceEpoch === lastSourceEpoch ? snapshot?.lastSourceSequence ?? 0 : 0) + 1; + if (!Number.isSafeInteger(sourceSeq)) throw new Error("native_synthetic_event_epoch_required"); lastSourceSequence = sourceSeq; synthesizedDurableWait = true; sawTerminal = true; @@ -647,7 +675,8 @@ class HarnessNativeSession implements NativeSession { }; yield { schema: "paperclip.prp.event.v1", - sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceEventId: normalizedEventId(sourceInstanceId, this.#input.identity.runId, sourceSeq, lastSourceEpoch), + ...(lastSourceEpoch ? { sourceEpoch: lastSourceEpoch } : {}), sourceSeq, sourceInstanceId, sourceKind: "runner", @@ -665,6 +694,22 @@ class HarnessNativeSession implements NativeSession { if (streamFailure && !synthesizedDurableWait) throw streamFailure; } + async acknowledgeEvent(event: PrpEvent): Promise { + // Backend-synthesized provider-loss facts have their own durable writer; + // they were never entries in the driver's normalized outbox. + if (this.#lastDriverEventId !== event.sourceEventId) return; + await this.#session.acknowledgeEvent?.(event); + this.#lastDriverEventId = null; + } + + setEventCommitter(commit: (event: PrpEvent) => Promise): void { + this.#session.setEventCommitter?.(commit); + } + + async flushEventDelivery(): Promise { + await this.#session.flushEventDelivery?.(); + } + async startTurn(input: Parameters[0]) { this.#assertProtocolIntegrity(); // Stop can arrive after session publication but before the first turn. @@ -801,7 +846,7 @@ class HarnessNativeSession implements NativeSession { cursor: snapshot.lastSourceSequence === undefined ? null - : String(snapshot.lastSourceSequence), + : encodeSourceCursor({ sourceSeq: snapshot.lastSourceSequence, sourceEpoch: snapshot.lastSourceEpoch }), semanticResult: this.#explicitlyCancelled ? null : (snapshot.semanticResult?.result ?? null), @@ -849,7 +894,7 @@ export function createHarnessDriverBackend( function parseCursor(cursor: string | null | undefined): number | undefined { if (cursor === undefined || cursor === null || cursor === "") return undefined; - const parsed = Number(cursor); + const parsed = decodeSourceCursor(cursor).sourceSeq; return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined; } diff --git a/packages/paperclip-runner/src/catalog/operation-groups-doc.test.ts b/packages/paperclip-runner/src/catalog/operation-groups-doc.test.ts index 9e3393b9ebb..8e279fa8139 100644 --- a/packages/paperclip-runner/src/catalog/operation-groups-doc.test.ts +++ b/packages/paperclip-runner/src/catalog/operation-groups-doc.test.ts @@ -68,7 +68,7 @@ describe("canonical operation-groups specification", () => { expect(input.source.behaviorGroups).toHaveLength(16); expect(input.scenarios).toHaveLength(106); expect(new Set(input.scenarios.map((scenario) => scenario.id)).size).toBe(106); - expect(input.eventTypes).toHaveLength(105); + expect(input.eventTypes).toHaveLength(106); expect(input.commandTypes).toHaveLength(18); }); diff --git a/packages/paperclip-runner/src/catalog/operation-groups-doc.ts b/packages/paperclip-runner/src/catalog/operation-groups-doc.ts index 5f2fcf6514a..c0da43e10a3 100644 --- a/packages/paperclip-runner/src/catalog/operation-groups-doc.ts +++ b/packages/paperclip-runner/src/catalog/operation-groups-doc.ts @@ -99,7 +99,7 @@ export interface OperationGroupsInput { catalogIndexSource: string; } -const EXPECTED_EVENT_COUNT = 105; +const EXPECTED_EVENT_COUNT = 106; const EXPECTED_COMMAND_COUNT = 18; const EXPECTED_SCENARIO_COUNT = 106; const EXPECTED_BEHAVIOR_GROUP_COUNT = 16; diff --git a/packages/paperclip-runner/src/contracts/control-plane-port.ts b/packages/paperclip-runner/src/contracts/control-plane-port.ts index d8773168683..c4d845df624 100644 --- a/packages/paperclip-runner/src/contracts/control-plane-port.ts +++ b/packages/paperclip-runner/src/contracts/control-plane-port.ts @@ -17,8 +17,9 @@ export interface OpenControlPlaneRunInput { } export interface AppendedEventReceipt { - cursor: number; + cursor: number | string; highestContiguousSourceSeq: number; + highestContiguousSourceEpoch?: string; disposition: "committed" | "duplicate"; } @@ -42,12 +43,14 @@ export interface ReplayControlPlaneEventsInput { runId: string; sourceInstanceId: string; afterSourceSeq: number; + sourceEpoch?: string; limit: number; } export interface ReplayedControlPlaneEvents { events: PrpEvent[]; highestContiguousSourceSeq: number; + highestContiguousSourceEpoch?: string; } export interface FinalizeControlPlaneOperationOptions { diff --git a/packages/paperclip-runner/src/contracts/durable-recovery.ts b/packages/paperclip-runner/src/contracts/durable-recovery.ts index 6b51c94db65..14f4ed11c89 100644 --- a/packages/paperclip-runner/src/contracts/durable-recovery.ts +++ b/packages/paperclip-runner/src/contracts/durable-recovery.ts @@ -92,6 +92,7 @@ export interface DurableRecoveryCoreCommand { schema: "paperclip.prp.command.v1" | "paperclip.prp.command.v2"; commandId: string; controllerSeq: number; + controllerEpoch?: string; type: string; issuedAt: string; payload: Record; @@ -106,6 +107,7 @@ export interface DurableRecoveryCoreCommand { } export interface DurableRecoveryCommittedEvent { + sourceEpoch?: string; sourceSeq: number; sourceEventId: string; eventType: string; diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 3400b419c9c..eed4137dd1f 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -470,6 +470,8 @@ export interface PersistedHarnessSession { activeTurnId?: string | null; semanticResult?: PersistedHarnessSemanticResult | null; terminalTurns?: PersistedHarnessTurnTerminal[]; + /** Indexed reducer fact committed with its normalized delivery outbox. */ + durableTerminal?: { event: PrpEvent; semanticFingerprint: string | null }; codexUsageBaseline?: { baseline: Record; latest: Record }; /** A result-less terminal task may spend this fail-closed one-shot recovery allowance. */ dispositionOnlyRecoveryConsumed?: boolean; @@ -479,6 +481,7 @@ export interface PersistedHarnessSession { goal?: HarnessThreadGoal | null; lineage?: HarnessThreadLineageEntry[]; lastSourceSequence?: number; + lastSourceEpoch?: string; /** Tagged provider identity used to reject cross-profile recovery. */ providerIdentity?: PersistedHarnessProviderIdentity; /** Narrow escape hatch for a durable response-wake when a provider cannot reload its prior native session. */ @@ -501,6 +504,10 @@ export interface HarnessSession { displayId?: string | null; }; events(): AsyncIterable; + /** Called only after the exact normalized event is durable in the run log. */ + acknowledgeEvent?(event: PrpEvent): Promise; + setEventCommitter?(commit: (event: PrpEvent) => Promise): void; + flushEventDelivery?(): Promise; attachRun?(input: { runId: string }): Promise | void; startTurn(input: { message: NativeUserMessage; diff --git a/packages/paperclip-runner/src/contracts/native-execution.ts b/packages/paperclip-runner/src/contracts/native-execution.ts index c23c2f9ab18..9253111347b 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.ts @@ -232,6 +232,7 @@ export interface NativeSessionExecutionResult { driverVersion: string; nativeEventCount: number; highestContiguousSourceSeq: number; + highestContiguousSourceEpoch?: string; usage: Record | null; /** The active durable goal reached a safe turn boundary for run rollover. */ goalRolloverRequired?: boolean; diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 856d04975af..abb33addb6f 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -144,6 +144,9 @@ export interface NativeSession { /** Relinquish controller authority without suspending provider execution. */ detachControllerForRestart?(): Promise; events(input?: { afterCursor?: string | null }): AsyncIterable; + acknowledgeEvent?(event: PrpEvent): Promise; + setEventCommitter?(commit: (event: PrpEvent) => Promise): void; + flushEventDelivery?(): Promise; startTurn(input: { message: NativeUserMessage; /** Set by orchestration only after successful provider-session recovery. */ diff --git a/packages/paperclip-runner/src/control-plane/authority-generation.test.ts b/packages/paperclip-runner/src/control-plane/authority-generation.test.ts new file mode 100644 index 00000000000..61d85a42d72 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/authority-generation.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { authorityDirectlyFollows, authorityGeneration, authorityInteger, validateAuthorityCommit, validateAuthorityWorkPage } from "./durable-authority-store.js"; + +const first = "r:00000000-0000-4000-8000-000000000001"; +const second = "r:00000000-0000-4000-8000-000000000002"; +const maximum = "9223372036854775807"; + +describe("exact authority revisions", () => { + it("accepts legacy exact boundaries and opaque revisions without converting them to numbers", () => { + for (const value of ["0", "9007199254740993", maximum, first]) expect(authorityGeneration(value)).toBe(value); + expect(authorityDirectlyFollows({ generation: first, committedFrom: maximum, state: {} }, maximum)).toBe(true); + expect(authorityDirectlyFollows({ generation: second, committedFrom: first, state: {} }, first)).toBe(true); + validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [] }); + validateAuthorityWorkPage("process-owner", "", 128, first); + }); + + it("requires exact immediate predecessor evidence for migration activation", () => { + const latest = { generation: second, committedFrom: first, state: {} }; + expect(authorityDirectlyFollows(latest, maximum)).toBe(false); + expect(authorityDirectlyFollows(latest, second)).toBe(false); + expect(authorityDirectlyFollows({ generation: first, state: {} }, maximum)).toBe(false); + expect(authorityDirectlyFollows({ generation: "2", state: {} }, first)).toBe(false); + expect(authorityDirectlyFollows({ generation: first, committedFrom: first, state: {} }, first)).toBe(false); + expect(authorityDirectlyFollows({ generation: "9007199254740993", state: {} }, "9007199254740992")).toBe(true); + expect(authorityDirectlyFollows({ generation: maximum, state: {} }, maximum)).toBe(false); + }); + + it.each(["-1", "01", "9223372036854775808", "r:missing", first.toUpperCase(), first.replace("-4000-", "-1000-")])("rejects invalid revision %s", value => { + expect(() => authorityGeneration(value)).toThrow("invalid_authority"); + }); + + it("keeps event ordering numeric and separate from opaque state revisions", () => { + expect(authorityInteger(maximum)).toBe(9_223_372_036_854_775_807n); + expect(() => authorityInteger(first)).toThrow("invalid_authority"); + expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [ + { epoch: "epoch-1", kind: "event", id: "event-1", sequence: second, body: {} }, + ] })).toThrow("invalid_authority"); + }); + + it("reserves zero for effects which have exact identities without ordering", () => { + const receipt = { epoch: "run-1", kind: "effect" as const, id: "effect", sequence: "0", body: {} }; + expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [receipt] })).not.toThrow(); + for (const kind of ["command", "event"] as const) { + expect(() => validateAuthorityCommit({ expectedGeneration: first, state: {}, records: [{ ...receipt, kind }] })).toThrow("invalid_authority"); + } + }); +}); diff --git a/packages/paperclip-runner/src/control-plane/authority-locator.test.ts b/packages/paperclip-runner/src/control-plane/authority-locator.test.ts new file mode 100644 index 00000000000..c93f1d0f22d --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/authority-locator.test.ts @@ -0,0 +1,28 @@ +import { closeSync, mkdtempSync, openSync, rmSync, truncateSync, writeSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { expect, it } from "vitest"; +import { readDurableControlPlaneState } from "./authority-locator.js"; +import { DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES } from "./durable-prp-control-plane.js"; + +it("preserves the existing legacy reader window when adding indexed locators", async () => { + const root = mkdtempSync(resolve(tmpdir(), "legacy-authority-window-")); + const path = resolve(root, "control-plane-state.json"); + try { + const fd = openSync(path, "wx", 0o600); + try { + writeSync(fd, '{"schema":"paperclip.runner.durable.control-plane-state.v1","history":"'); + const chunk = Buffer.alloc(1024 * 1024, "x"); + for (let index = 0; index < 65; index++) writeSync(fd, chunk); + writeSync(fd, '"}'); + } finally { closeSync(fd); } + // This valid compatibility file exceeds the accidental 64 MiB locator + // reader cap, but is still inside the controller's original 192 MiB bound. + const state = await readDurableControlPlaneState(root); + expect((state.history as string).length).toBe(65 * 1024 * 1024); + // A sparse oversized file is only a metadata-rejection fixture, not growth + // qualification: the reader must reject it before allocating its body. + truncateSync(path, DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES + 1); + await expect(readDurableControlPlaneState(root)).rejects.toThrow("state_unsafe"); + } finally { rmSync(root, { recursive: true, force: true }); } +}, 15_000); diff --git a/packages/paperclip-runner/src/control-plane/authority-locator.ts b/packages/paperclip-runner/src/control-plane/authority-locator.ts new file mode 100644 index 00000000000..a301fa42dcb --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/authority-locator.ts @@ -0,0 +1,80 @@ +import { constants, lstatSync, readSync, openSync, fstatSync, closeSync } from "node:fs"; +import { basename, resolve } from "node:path"; +import { SqliteAuthorityStore } from "./sqlite-authority-store.js"; +import { assertNoPendingLegacyMigration } from "./legacy-migration-gate.js"; +import { DurableAuthorityStoreError, MAX_LEGACY_AUTHORITY_STATE_BYTES, type AuthoritySnapshot } from "./durable-authority-store.js"; +import { materializeCurrentAuthority } from "./current-authority-evidence.js"; +import { authorityJson } from "./durable-authority-store.js"; +import { createHash } from "node:crypto"; + +export interface AuthorityLocation { + kind: "sqlite" | "postgres"; + binding: string; + file?: string; + epoch?: string; +} +export interface AuthorityLocator { + schema: "paperclip.runner.authority-locator.v1"; + location: AuthorityLocation; +} +export type ExternalAuthorityReader = (location: AuthorityLocation) => Promise; + +export function parseAuthorityLocator(value: unknown): AuthorityLocator | null { + const record = value as Partial | null; + if (!record || record.schema !== "paperclip.runner.authority-locator.v1") return null; + const location = record.location; + if (!location || !["sqlite", "postgres"].includes(location.kind) || typeof location.binding !== "string" || location.binding.length > 4096 || + (location.kind === "sqlite" && (typeof location.file !== "string" || location.file !== basename(location.file) || !/^[a-zA-Z0-9_.-]+\.sqlite$/.test(location.file))) || + (location.kind === "postgres" && (typeof location.epoch !== "string" || !/^[a-zA-Z0-9_.:-]{1,240}$/.test(location.epoch)))) { + throw new DurableAuthorityStoreError("invalid_authority", "invalid authority locator"); + } + return record as AuthorityLocator; +} + +/** Legacy files are bounded compatibility input; indexed locators load one row. */ +export async function readDurableControlPlaneState(directory: string, readExternal?: ExternalAuthorityReader): Promise> { + return (await readDurableControlPlaneSnapshot(directory, readExternal)).state; +} + +/** The materialized view and its stored digest are different representations. + * Archive fences compare the latter, after checking the former still matches + * the authority the caller admitted. External stores own their own fences. */ +export async function readDurableControlPlaneSnapshot(directory: string, readExternal?: ExternalAuthorityReader): Promise<{ + state: Record; storedSha256: string | null; +}> { + assertNoPendingLegacyMigration(directory); + const path = resolve(directory, "control-plane-state.json"); + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let value: unknown; + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > MAX_LEGACY_AUTHORITY_STATE_BYTES) throw new Error("native_runner_control_plane_state_unsafe"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error("native_runner_control_plane_state_changed"); + offset += count; + } + const after = fstatSync(fd); + if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("native_runner_control_plane_state_changed"); + value = JSON.parse(bytes.toString("utf8")); + if (parseAuthorityLocator(value) && process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.())) throw new Error("native_runner_control_plane_state_unsafe"); + } finally { closeSync(fd); } + const locator = parseAuthorityLocator(value); + if (!locator) return { state: value as Record, storedSha256: null }; + if (locator.location.kind === "postgres") { + if (!readExternal) throw new DurableAuthorityStoreError("storage_unavailable", "Postgres authority reader is required"); + return { state: (await readExternal(locator.location)).state, storedSha256: null }; + } + const metadata = lstatSync(directory); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) throw new Error("native_runner_control_plane_state_unsafe"); + const store = await SqliteAuthorityStore.open({ binding: locator.location.binding, path: resolve(directory, locator.location.file!), create: false, readOnly: true }); + try { + const snapshot = await store.load(); + if (!snapshot) throw new DurableAuthorityStoreError("storage_unavailable", "activated authority snapshot is missing"); + const state = await materializeCurrentAuthority(snapshot.state, store.getRecord.bind(store)); + if ((await store.load())?.generation !== snapshot.generation) throw new DurableAuthorityStoreError("stale_authority", "authority changed while resolving current evidence"); + return { state, storedSha256: createHash("sha256").update(authorityJson(snapshot.state)).digest("hex") }; + } finally { await store.close(); } +} diff --git a/packages/paperclip-runner/src/control-plane/command-epochs.ts b/packages/paperclip-runner/src/control-plane/command-epochs.ts new file mode 100644 index 00000000000..4ae04244b96 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/command-epochs.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +/** Ordering is local to a negotiated delivery namespace, never the run lifetime. */ +export const COMMAND_EPOCH_CAPABILITY = "transport.command_epochs.v1"; +export const COMMAND_EPOCH_LIMIT = 1_048_576; +const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/; +export interface CommandEpochTransition { + schema: "paperclip.prp.command-epoch.v1"; + runId: string; + transitionId: string; + fromEpoch: string | null; + nextEpoch: string; + finalOrdinal: number; +} +export function isCommandEpoch(value: unknown): value is string { + return typeof value === "string" && uuid.test(value); +} +export function isCommandEpochTransition(value: unknown): value is CommandEpochTransition { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const v = value as CommandEpochTransition; + return Object.keys(v).length === 6 && v.schema === "paperclip.prp.command-epoch.v1" + && typeof v.runId === "string" && v.runId.length > 0 && v.runId.length <= 240 + && isCommandEpoch(v.transitionId) && (v.fromEpoch === null || isCommandEpoch(v.fromEpoch)) + && isCommandEpoch(v.nextEpoch) && v.nextEpoch !== v.fromEpoch + && Number.isSafeInteger(v.finalOrdinal) && v.finalOrdinal > 0; +} +/** Current proof slots from older epochs precede this epoch's ordered suffix. + * There is at most one old settled receipt per slot; no history traversal. */ +export function compareCurrentCommands( + a: { controllerEpoch?: string; controllerSeq: number }, + b: { controllerEpoch?: string; controllerSeq: number }, + epoch: string | undefined, +): number { + if (a.controllerEpoch === b.controllerEpoch) return a.controllerSeq - b.controllerSeq; + return Number(a.controllerEpoch === epoch) - Number(b.controllerEpoch === epoch); +} + + +export function commandEpochCloseId(transition: Pick): string { + return `command-epoch-from-${createHash("sha256").update(JSON.stringify([transition.runId, transition.fromEpoch])).digest("hex")}`; +} diff --git a/packages/paperclip-runner/src/control-plane/current-authority-evidence.test.ts b/packages/paperclip-runner/src/control-plane/current-authority-evidence.test.ts new file mode 100644 index 00000000000..04b185514f8 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/current-authority-evidence.test.ts @@ -0,0 +1,29 @@ +import { expect, it, vi } from "vitest"; +import { materializeCurrentAuthority, referenceCurrentAuthority } from "./current-authority-evidence.js"; +import type { StoredCoreState } from "./durable-prp-control-plane.js"; + +it("resolves only exact current references and rejects missing, conflicting or rebound authority", async () => { + const command = { commandId: "finish-result", controllerSeq: 12, status: "completed", type: "semantic_tool.result", + payload: { operationId: "paperclip_finish", input: { summary: "body".repeat(10000) } }, result: { accepted: true } }; + const active = { commandId: "pending", controllerSeq: 13, status: "pending", type: "turn.start" }; + const state = { identity: { runId: "run-1" }, commands: [command, active], committedEvents: [], + indexedState: { schema: "paperclip.runner.current-authority.v1", pendingSemanticInputIds: [] } } as unknown as StoredCoreState; + const stored = referenceCurrentAuthority(state); + expect(stored.commands).toEqual([active]); + expect(JSON.stringify(stored).length).toBeLessThan(1024); + const receipt = { epoch: "run-1", kind: "command" as const, id: command.commandId, sequence: "12", body: command }; + const read = vi.fn(async () => receipt); + expect(await materializeCurrentAuthority(stored as unknown as Record, read)).toEqual(state); + expect(read).toHaveBeenCalledExactlyOnceWith("run-1", "command", "finish-result"); + for (const bad of [null, { ...receipt, epoch: "foreign" }, { ...receipt, id: "other" }, { ...receipt, sequence: "11" }, { ...receipt, body: { ...command, result: {} } }]) { + await expect(materializeCurrentAuthority(stored as unknown as Record, async () => bad)).rejects.toThrow("exact receipt"); + } + const absent = structuredClone(stored); + delete absent.indexedState!.recoveryEvidence; + await expect(materializeCurrentAuthority(absent as unknown as Record, read)).rejects.toThrow("exact receipt"); + const rebound = structuredClone(stored); + const refs = rebound.indexedState!.recoveryEvidence!.records; + refs["command:other"] = refs["command:terminal:paperclip_finish"]!; + delete refs["command:terminal:paperclip_finish"]; + await expect(materializeCurrentAuthority(rebound as unknown as Record, read)).rejects.toThrow("exact receipt"); +}); diff --git a/packages/paperclip-runner/src/control-plane/current-authority-evidence.ts b/packages/paperclip-runner/src/control-plane/current-authority-evidence.ts new file mode 100644 index 00000000000..877ad9205c8 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/current-authority-evidence.ts @@ -0,0 +1,139 @@ +import { normalizedEpochCloseId } from "./event-epochs.js"; +import { compareCurrentEvents, isEventEpochTransition, eventEpochCloseId } from "./event-epochs.js"; +import { compareCurrentCommands, isCommandEpochTransition, commandEpochCloseId } from "./command-epochs.js"; +import { createHash } from "node:crypto"; +import { authorityJson, DurableAuthorityStoreError, type AuthorityRecord, type DurableAuthorityStore } from "./durable-authority-store.js"; +import type { StoredCoreState } from "./durable-prp-control-plane.js"; + +export interface CurrentAuthorityReference { + epoch: string; + kind: "command" | "event"; + id: string; + sequence: string; + sha256: string; +} +export interface CurrentAuthorityEvidence { + schema: "paperclip.current-evidence.v1"; + records: Record; + commandOrder?: string[]; + eventOrder?: string[]; +} +const digest = (body: unknown) => createHash("sha256").update(authorityJson(body)).digest("hex"); +const lifecycleEvents = new Set(["session.started", "session.resumed", "session.reconciled", "harness.ready", "turn.accepted", "turn.started", "turn.completed", "turn.failed", "run.terminal"]); +const terminalOperations = new Set(["paperclip_finish", "paperclip_block"]); +const object = (v: unknown): Record => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; +function invalid(): never { throw new DurableAuthorityStoreError("invalid_authority", "current recovery evidence differs from its exact receipt"); } +function slot(kind: CurrentAuthorityReference["kind"], body: Record): string | null { + if (kind === "command") { + if (body.status === "pending" || body.status === "indeterminate") return null; + if (typeof body.type !== "string" || body.type.length > 80) invalid(); + if (body.type !== "semantic_tool.result") return `command:${body.type}`; + const operation = String(object(body.payload).operationId); + return terminalOperations.has(operation) ? `command:terminal:${operation}` : null; + } + const operation = String(object(object(object(object(body.envelope).payload).payload).semantic_tool).operationId); + if (terminalOperations.has(operation)) return `event:${body.eventType}:${operation}`; + return lifecycleEvents.has(String(body.eventType)) ? `event:${body.eventType}` : null; +} + +/** The on-disk current row contains active work and a fixed set of typed slots. + * Settled payloads live once in the immutable receipt index. This does not + * certify a finish by itself: server acceptance still validates its structured + * input, operation result, completion contract and business result receipt. */ +export function referenceCurrentAuthority(state: StoredCoreState): StoredCoreState { + const next = structuredClone(state); + const current = next.indexedState!; + const evidence: CurrentAuthorityEvidence = { schema: "paperclip.current-evidence.v1", records: {}, commandOrder: next.commands.map(c => c.commandId), eventOrder: next.committedEvents.map(e => e.sourceEventId) }; + const remember = (kind: CurrentAuthorityReference["kind"], body: Record, id: string, sequence: number) => { + const key = slot(kind, body); + if (!key) return false; + evidence.records[key] = { epoch: next.identity.runId, kind, id, sequence: String(sequence), sha256: digest(body) }; + return true; + }; + next.commands = next.commands.filter(command => !remember("command", command as unknown as Record, command.commandId, command.controllerSeq)); + next.committedEvents = next.committedEvents.filter(event => current.pendingSemanticInputIds.includes(event.sourceEventId) || + !remember("event", { ...event, deliveryCount: 1 }, event.sourceEventId, event.sourceSeq)); + if (Object.keys(evidence.records).length > 40) invalid(); + current.recoveryEvidence = evidence; + current.schema = "paperclip.runner.current-authority.v2"; + return next; +} + +/** Resolves only named current receipts. There is no history scan or fallback + * to a nearby event if a receipt is missing. Callers fence snapshot generation + * before and after this bounded read when opening an active authority. */ +export async function materializeCurrentAuthority(state: Record, read: DurableAuthorityStore["getRecord"]): Promise> { + const current = object(state.indexedState); + if (state.indexedState !== undefined && !["paperclip.runner.current-authority.v1", "paperclip.runner.current-authority.v2"].includes(String(current.schema))) invalid(); + if (current.recoveryEvidence === undefined) { + if (current.schema === "paperclip.runner.current-authority.v2") invalid(); + return state; // Retained v1 current rows and already materialized views. + } + if (current.controllerEpoch !== undefined || current.lastCommandEpochTransition !== undefined) { + const transition = current.lastCommandEpochTransition; + if (!isCommandEpochTransition(transition) || transition.nextEpoch !== current.controllerEpoch || transition.runId !== object(state.identity).runId) invalid(); + for (const id of [`command-epoch-${transition.transitionId}`, commandEpochCloseId(transition)]) { + const receipt = await read(transition.runId, "effect", id); + if (!receipt || receipt.epoch !== transition.runId || receipt.kind !== "effect" || receipt.id !== id || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(transition)) invalid(); + } + } + if (current.sourceEpoch !== undefined || current.lastEventEpochTransition !== undefined) { + const transition = current.lastEventEpochTransition; + if (!isEventEpochTransition(transition) || transition.nextEpoch !== current.sourceEpoch || transition.runId !== object(state.identity).runId) invalid(); + for (const id of [`event-epoch-${transition.transitionId}`, eventEpochCloseId(transition)]) { + const receipt = await read(transition.runId, "effect", id); + if (!receipt || receipt.epoch !== transition.runId || receipt.kind !== "effect" || receipt.id !== id || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(transition)) invalid(); + } + } + const delivery = object(current.normalizedDelivery); + if (delivery.producedEpoch !== undefined || delivery.lastSourceEpochTransition !== undefined) { + const t = delivery.lastSourceEpochTransition; + const identity = object(state.identity); + if (!isEventEpochTransition(t) || t.nextEpoch !== delivery.producedEpoch || t.runId !== identity.runId) invalid(); + for (const id of [`normalized-epoch-${t.transitionId}`, normalizedEpochCloseId(t.runId, String(identity.runnerInstanceId), t.fromEpoch)]) { + const receipt = await read(t.runId, "effect", id); + if (!receipt || receipt.sequence !== "0" || authorityJson(receipt.body) !== authorityJson(t)) invalid(); + } + } + const raw = object(object(current.normalizedDelivery).raw); + if (current.normalizedDelivery && raw.sourceEpoch !== current.sourceEpoch) { + const receipt = await read(String(object(state.identity).runId), "effect", eventEpochCloseId({runId: String(object(state.identity).runId), fromEpoch: (raw.sourceEpoch as string | undefined) ?? null})); + if (!receipt || !isEventEpochTransition(receipt.body) || receipt.body.fromEpoch !== (raw.sourceEpoch ?? null) || Number(raw.sourceSeq) > receipt.body.finalOrdinal) invalid(); + } + const evidence = object(current.recoveryEvidence), entries = Object.entries(object(evidence.records)); + if (current.schema !== "paperclip.runner.current-authority.v2" || evidence.schema !== "paperclip.current-evidence.v1" || !evidence.records || Array.isArray(evidence.records) || entries.length > 40 || + !Array.isArray(state.commands) || !Array.isArray(state.committedEvents)) invalid(); + const result = structuredClone(state) as unknown as StoredCoreState; + const ids = new Set(); + for (const [key, value] of entries) { + const ref = value as CurrentAuthorityReference; + if (!ref || ref.epoch !== result.identity.runId || !["command", "event"].includes(ref.kind) || + typeof ref.id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(ref.id) || + !/^[1-9][0-9]{0,15}$/.test(ref.sequence) || !Number.isSafeInteger(Number(ref.sequence)) || !/^[a-f0-9]{64}$/.test(ref.sha256)) invalid(); + const receipt: AuthorityRecord | null = await read(ref.epoch, ref.kind, ref.id); + if (!receipt || receipt.epoch !== ref.epoch || receipt.kind !== ref.kind || receipt.id !== ref.id || + receipt.sequence !== ref.sequence || digest(receipt.body) !== ref.sha256 || slot(ref.kind, receipt.body) !== key || ids.has(`${ref.kind}:${ref.id}`)) invalid(); + ids.add(`${ref.kind}:${ref.id}`); + if (ref.kind === "command") { + if (receipt.body.commandId !== ref.id || String(receipt.body.controllerSeq) !== ref.sequence || result.commands.some(c => c.commandId === ref.id)) invalid(); + result.commands.push(receipt.body as unknown as StoredCoreState["commands"][number]); + } else { + if (receipt.body.sourceEventId !== ref.id || String(receipt.body.sourceSeq) !== ref.sequence || result.committedEvents.some(e => e.sourceEventId === ref.id)) invalid(); + result.committedEvents.push(receipt.body as unknown as StoredCoreState["committedEvents"][number]); + } + } + result.commands.sort((a, b) => compareCurrentCommands(a, b, result.indexedState?.controllerEpoch)); + result.committedEvents.sort((a, b) => compareCurrentEvents(a, b, result.indexedState?.sourceEpoch)); + for (const [order, records, id] of [ + [evidence.commandOrder, result.commands, "commandId"], [evidence.eventOrder, result.committedEvents, "sourceEventId"], + ] as const) { + if (order === undefined) continue; // retained current-evidence v1 + if (!Array.isArray(order) || order.length !== records.length || new Set(order).size !== order.length) invalid(); + const position = new Map(order.map((key, index) => [key, index])); + if (records.some(record => !position.has((record as unknown as Record)[id]))) invalid(); + records.sort((a, b) => position.get((a as unknown as Record)[id])! - position.get((b as unknown as Record)[id])!); + } + delete result.indexedState!.recoveryEvidence; + result.indexedState!.schema = "paperclip.runner.current-authority.v1"; + return result as unknown as Record; +} diff --git a/packages/paperclip-runner/src/control-plane/durable-authority-store.ts b/packages/paperclip-runner/src/control-plane/durable-authority-store.ts new file mode 100644 index 00000000000..313e4f394ef --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/durable-authority-store.ts @@ -0,0 +1,181 @@ +import type { AuthorityLocation } from "./authority-locator.js"; +/** The execution store is independent of the server and its database driver. + * History is queried by exact identity or cursor, never returned by load(). */ +export const INDEXED_DURABILITY_CAPABILITY = "durability.indexed_state.v1"; +/** Compatibility readers and writers must admit the same legacy journal size. */ +export const MAX_LEGACY_AUTHORITY_STATE_BYTES = 192 * 1024 * 1024; +export const MAX_AUTHORITY_STATE_BYTES = 16 * 1024 * 1024; +export const MAX_AUTHORITY_RECORD_BYTES = 1024 * 1024; +export const MAX_AUTHORITY_COMMIT_RECORDS = 512; + +export interface AuthoritySnapshot { + /** Exact store revision. Compare equality; never order or increment it. */ + generation: string; + /** Exact predecessor when the store uses opaque commit identities. */ + committedFrom?: string; + state: Record; +} + +export interface AuthorityRecord { + epoch: string; + kind: "command" | "event" | "effect"; + id: string; + /** Command/event order belongs to the authenticated run epoch. Effects are + * addressed by exact identity and use zero when the store supports it. */ + sequence: string; + /** Absent is the retained numeric namespace; epochs require store capability. */ + sequenceEpoch?: string; + body: Record; +} + +export interface AuthorityCommit { + expectedGeneration: string; + state: Record; + /** Newly settled receipts/events only. Existing records are immutable. */ + records: readonly AuthorityRecord[]; + work?: readonly AuthorityWorkChange[]; +} + +export interface AuthorityWorkRecord { + collection: "process-owner"; + id: string; + body: Record; + sha256: string; +} +export interface AuthorityWorkChange { + collection: AuthorityWorkRecord["collection"]; + id: string; + expectedSha256: string | null; + /** Null retires current work; its exact retirement proof belongs in records. */ + body: Record | null; +} +export interface AuthorityWorkPage { records: AuthorityWorkRecord[]; nextAfter: string | null } + +export interface AuthorityPage { + records: AuthorityRecord[]; + nextAfter: string | null; +} + +export interface DurableAuthorityStore { + readonly binding: string; + readonly location: AuthorityLocation; + /** Retained v1 stores require a positive ordered effect sequence. */ + readonly unorderedEffectReceipts?: boolean; + readonly commandEpochs?: boolean; + readonly eventEpochs?: boolean; + load(): Promise; + commit(input: AuthorityCommit): Promise; + getRecord(epoch: string, kind: AuthorityRecord["kind"], id: string): Promise; + /** Session-scoped exact effect lookup survives run/authority rotation. */ + getSessionEffect(id: string): Promise; + getWork(collection: AuthorityWorkRecord["collection"], id: string): Promise; + readWorkPage(collection: AuthorityWorkRecord["collection"], after: string, limit: number, expectedGeneration: string): Promise; + readEvents(epoch: string, after: string, limit: number, byteBudget: number, sequenceEpoch?: string): Promise; + close(): Promise; +} + +export class DurableAuthorityStoreError extends Error { + readonly code: "storage_pressure" | "storage_unavailable" | "stale_authority" | "receipt_conflict" | "invalid_authority"; + constructor(code: DurableAuthorityStoreError["code"], message: string) { + super(`${code}: ${message}`); + this.name = "DurableAuthorityStoreError"; + this.code = code; + } +} + +export function authorityInteger(value: string): bigint { + if (!/^(0|[1-9][0-9]{0,18})$/.test(value) || BigInt(value) > 9_223_372_036_854_775_807n) { + throw new DurableAuthorityStoreError("invalid_authority", "invalid exact sequence/generation"); + } + return BigInt(value); +} + +/** Legacy stores expose exact integer revisions. Stores with opaque revisions + * retain predecessor evidence for operations which must prove one transition. */ +export function authorityGeneration(value: string): string { + if (/^r:[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(value)) return value; + authorityInteger(value); + return value; +} + +export function authorityDirectlyFollows(snapshot: AuthoritySnapshot, previous: string): boolean { + authorityGeneration(previous); + authorityGeneration(snapshot.generation); + if (snapshot.committedFrom !== undefined) { + authorityGeneration(snapshot.committedFrom); + return snapshot.committedFrom === previous && snapshot.generation !== previous; + } + // Numeric v1 stores have no revision wraparound and prove their predecessor + // by exact arithmetic. An opaque revision without a commit stamp cannot. + if (snapshot.generation.startsWith("r:") || previous.startsWith("r:")) return false; + return authorityInteger(snapshot.generation) === authorityInteger(previous) + 1n; +} + +/** Stable encoding for receipt comparison across SQLite and Postgres. */ +export function authorityJson(value: unknown): string { + const ancestors = new Set(); + function encode(value: unknown, depth: number): string { + if (depth > 128) throw new DurableAuthorityStoreError("invalid_authority", "authority JSON nesting exceeds 128"); + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number" && Number.isFinite(value)) return JSON.stringify(value); + if (typeof value !== "object" || value === null || (!Array.isArray(value) && ![Object.prototype, null].includes(Object.getPrototypeOf(value)))) { + throw new DurableAuthorityStoreError("invalid_authority", "authority requires JSON values"); + } + if (ancestors.has(value)) throw new DurableAuthorityStoreError("invalid_authority", "cyclic authority JSON"); + ancestors.add(value); + try { + if (Array.isArray(value)) return `[${value.map((item) => encode(item, depth + 1)).join(",")}]`; + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${encode((value as Record)[key], depth + 1)}`).join(",")}}`; + } finally { + ancestors.delete(value); + } + } + return encode(value, 0); +} + +export function validateAuthorityCommit(input: AuthorityCommit): void { + authorityGeneration(input.expectedGeneration); + if (Buffer.byteLength(authorityJson(input.state)) > MAX_AUTHORITY_STATE_BYTES || input.records.length > MAX_AUTHORITY_COMMIT_RECORDS) { + throw new DurableAuthorityStoreError("storage_pressure", "current state or commit exceeds admission capacity"); + } + let bytes = 0; + for (const record of input.records) { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(record.epoch) || !["command", "event", "effect"].includes(record.kind) || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(record.id) || (authorityInteger(record.sequence) === 0n && record.kind !== "effect")) { + throw new DurableAuthorityStoreError("invalid_authority", "invalid receipt identity"); + } + if (record.sequenceEpoch !== undefined && (record.kind === "effect" || !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(record.sequenceEpoch))) { + throw new DurableAuthorityStoreError("invalid_authority", "invalid receipt sequence epoch"); + } + const size = Buffer.byteLength(authorityJson(record.body)); + bytes += size; + if (size > MAX_AUTHORITY_RECORD_BYTES || bytes > MAX_AUTHORITY_STATE_BYTES) { + throw new DurableAuthorityStoreError("storage_pressure", "receipt transaction exceeds admission capacity"); + } + } + const changes = input.work ?? []; + if (changes.length > 128) throw new DurableAuthorityStoreError("storage_pressure", "outstanding-work transaction exceeds capacity"); + const keys = new Set(); + for (const change of changes) { + validateAuthorityWorkKey(change.collection, change.id); + const key = `${change.collection}:${change.id}`; + if (keys.has(key) || (change.expectedSha256 !== null && !/^[a-f0-9]{64}$/.test(change.expectedSha256)) || + (change.body !== null && Buffer.byteLength(authorityJson(change.body)) > 16384)) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work mutation"); + keys.add(key); + } +} + +export function validateAuthorityWorkKey(collection: string, id: string): void { + if (collection !== "process-owner" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(id)) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work identity"); +} +export function validateAuthorityWorkPage(collection: string, after: string, limit: number, generation: string): void { + validateAuthorityWorkKey(collection, after || "first"); authorityGeneration(generation); + if (!Number.isInteger(limit) || limit < 1 || limit > 128) throw new DurableAuthorityStoreError("invalid_authority", "invalid outstanding-work page"); +} + +export function validateAuthorityPage(after: string, limit: number, byteBudget: number, sequenceEpoch?: string): void { + authorityInteger(after); + if (sequenceEpoch !== undefined && !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(sequenceEpoch)) throw new DurableAuthorityStoreError("invalid_authority", "invalid event page namespace"); + if (!Number.isInteger(limit) || limit < 1 || limit > 128 || !Number.isInteger(byteBudget) || byteBudget < MAX_AUTHORITY_RECORD_BYTES || byteBudget > MAX_AUTHORITY_STATE_BYTES) { + throw new DurableAuthorityStoreError("invalid_authority", "invalid event page capacity"); + } +} diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 47f958febd0..81eeb3a0e8a 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -3,6 +3,7 @@ import { createDecipheriv, createHash, createHmac, + randomUUID, } from "node:crypto"; import { chmodSync, @@ -16,6 +17,7 @@ import nodeFs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { describe, expect, it, vi } from "vitest"; @@ -30,6 +32,9 @@ import { type RunnerProcessLaunchSpec, } from "./durable-prp-control-plane.js"; import type { DurableRecoveryIdentity } from "./prp-transport-types.js"; +import { SqliteAuthorityStore } from "./sqlite-authority-store.js"; +import { DurableAuthorityStoreError } from "./durable-authority-store.js"; +import { readDurableControlPlaneState } from "./authority-locator.js"; const identity: DurableRecoveryIdentity = { runnerInstanceId: "runner-test-1", @@ -42,6 +47,438 @@ const identity: DurableRecoveryIdentity = { const expectedRunnerVersion = "0.3.0"; const expectedRunnerDigest = `sha256:${"a".repeat(64)}`; +it("renews encrypted connections repeatedly without resetting durable history or treating rekey as corruption", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-channel-rekey-")); + const authority = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, + authorityStore: authority, secureChannelFrameLimit: 4 }); + let client: AuthenticatedClient | null = null; + try { + await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket()); + const token = client!.leaseToken!; + const channels = new Set(); + let first!: Record; + for (let epoch = 0; epoch < 3; epoch++) { + channels.add(client!.sessionId); + for (let ordinal = 1; ordinal <= 4; ordinal++) { + const raw = semanticInputEvent(epoch * 4 + ordinal); + (raw.payload as Record).eventType = "harness.diagnostic"; + (raw.payload as Record).payload = { code: "rekey" }; + if (epoch === 0 && ordinal === 1) first = raw; + sendSecure(client!, raw); + const reply = await receiveSecure(client!); + if (ordinal < 4) expect(reply?.kind).toBe("ack"); + else expect(reply).toBeNull(); // Accepted cursor survives the lost ACK. + } + expect(core.store.state.ackedSourceSeq).toBe((epoch + 1) * 4); + client = await authenticate(core, token); + expect(client).not.toBeNull(); + expect(client!.welcome.payload).toMatchObject({ ackedSourceSeq: (epoch + 1) * 4 }); + } + expect(channels.size).toBe(3); + sendSecure(client!, first); + expect((await receiveSecure(client!))?.payload).toMatchObject({ ackedSourceSeq: 12 }); + expect(core.store.state.malformedFrames).toBe(0); + expect(core.store.state.identity).toEqual(identity); + expect((await authority.getRecord(identity.runId, "event", "semantic-event-1"))?.body.envelope).toEqual(first); + } finally { + client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await authority.close(); + rmSync(root, { recursive: true, force: true }); + } +}, 15_000); + +it("keeps fresh connection identities and recoverable state after diagnostic counters saturate", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-diagnostic-boundary-")); + const config = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let authority = await SqliteAuthorityStore.open(config); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: authority }); + let client: AuthenticatedClient | null = null; + try { + for (const field of ["connectionCount", "freshBootstraps", "malformedFrames", "replayDeliveries", "duplicateCommandResults"] as const) { + core.store.state[field] = Number.MAX_SAFE_INTEGER; + } + await core.store.save(); await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket()); + const lease = client!.leaseToken!, firstId = client!.welcome.connectionId; + client!.socket.destroy(); + client = await authenticate(core, lease); + expect(client!.welcome.connectionId).not.toBe(firstId); + expect(core.store.state.connectionCount).toBe(Number.MAX_SAFE_INTEGER); + expect(core.store.state.freshBootstraps).toBe(Number.MAX_SAFE_INTEGER); + const raw = semanticInputEvent(); + (raw.payload as Record).eventType = "harness.diagnostic"; + (raw.payload as Record).payload = { code: "retained" }; + for (let attempt = 0; attempt < 2; attempt++) { sendSecure(client!, raw); expect((await receiveSecure(client!))?.kind).toBe("ack"); } + expect(core.store.state.replayDeliveries).toBe(Number.MAX_SAFE_INTEGER); + sendMaskedJson(client!.socket, { schema: "invalid-encrypted-frame" }); + expect(await client!.reader.next()).toBeNull(); + expect(core.store.state.malformedFrames).toBe(Number.MAX_SAFE_INTEGER); + client = null; await core.stop(); await core.drainPendingConnectionProcessing(); await authority.close(); + authority = await SqliteAuthorityStore.open({ ...config, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: authority }); + expect(core.store.state.ackedSourceSeq).toBe(1); + expect((await authority.getRecord(identity.runId, "event", "semantic-event-1"))?.body.envelope).toEqual(raw); + } finally { + client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await authority.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each(["rolled_back", "committed", "unreadable"] as const)("retries resource pressure only after verifying an unchanged durable generation (%s)", async disposition => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-pressure-recovery-")); + const authority = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: authority }); + try { + const before = structuredClone(core.store.state), commit = authority.commit.bind(authority); + const fail = vi.spyOn(authority, "commit").mockImplementationOnce(async input => { + if (disposition === "committed") await commit(input); + throw new DurableAuthorityStoreError("storage_pressure", "injected full disk"); + }); + const read = disposition === "unreadable" ? vi.spyOn(authority, "load").mockRejectedValueOnce(new Error("unavailable read")) : null; + await expect(core.issueBootstrapTicket()).rejects.toThrow("storage_pressure"); + fail.mockRestore(); read?.mockRestore(); + if (disposition === "rolled_back") { + expect(core.store.state).toEqual(before); + const ticket = await core.issueBootstrapTicket(); + expect(ticket).toBeTruthy(); + expect(Object.keys(core.store.state.tickets)).toHaveLength(1); + } else { + await expect(core.issueBootstrapTicket()).rejects.toThrow("indeterminate"); + } + } finally { await core.stop(); await authority.close(); rmSync(root, { recursive: true, force: true }); } +}); + +it("resumes a paged process-owner upgrade after an indeterminate commit without replaying older launch phases", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-owner-upgrade-")); + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(storage); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + try { + const snapshot = (await store.load())!; + const state = structuredClone(snapshot.state); + delete (state.indexedState as Record).processOwnerIndexVersion; + const events = []; + for (let index = 1; index <= 260; index++) { + for (const phase of ["intent", "spawned"]) { + const sourceSeq = events.length + 1; + const raw = semanticInputEvent(); + const payload = raw.payload as Record; + payload.eventType = "harness.diagnostic"; + payload.sourceSeq = sourceSeq; payload.sourceEventId = `upgrade-event-${sourceSeq}`; + payload.payload = { code: "provider_startup_ownership", startup: { schema: "paperclip.provider_startup.v1", + launchId: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`, phase, attemptedProcessGeneration: index, + configurationFingerprint: `sha256:${"a".repeat(64)}`, origin: identity, + ...(phase === "spawned" ? { processId: 10000 + index, processGroupId: 10000 + index } : {}), processTreeRetired: false } }; + events.push({ sourceEventId: payload.sourceEventId, sourceSeq, eventType: "harness.diagnostic", envelope: raw, + priority: 1, deliveryCount: 1, logicalEffectCount: 1 }); + } + } + state.committedEvents = events; state.ackedSourceSeq = events.length; + delete ((state.indexedState as { recoveryEvidence: { eventOrder?: string[] } }).recoveryEvidence).eventOrder; + await store.commit({ expectedGeneration: snapshot.generation, state, records: [] }); + await core.stop(); await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + const commit = store.commit.bind(store); + let pages = 0; + const fault = vi.spyOn(store, "commit").mockImplementation(async input => { + expect(input.work?.length ?? 0).toBeLessThanOrEqual(128); + const generation = await commit(input); + if (++pages === 2) throw new Error("lost upgrade commit reply"); + return generation; + }); + await expect(DurablePrpControlPlane.open({ ...options, authorityStore: store })).rejects.toThrow("lost upgrade commit reply"); + fault.mockRestore(); + expect((await store.load())!.state.committedEvents).toHaveLength(520); + await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + expect(core.store.state.committedEvents).toEqual([]); + const first = await core.readProcessOwners(); + const second = await core.readProcessOwners(first.nextAfter!, 128, first.generation); + const third = await core.readProcessOwners(second.nextAfter!, 128, first.generation); + expect([first.records.length, second.records.length, third.records.length]).toEqual([128, 128, 4]); + for (const row of [...first.records, ...second.records, ...third.records]) expect(row.body.startup).toMatchObject({ phase: "spawned" }); + expect(Buffer.byteLength(JSON.stringify(core.store.state))).toBeLessThan(8192); + } finally { await core.stop(); await store.close(); rmSync(root, { recursive: true, force: true }); } +}, 30_000); + +it("pages unresolved process owners independently of historical events and preserves them after restart", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-owner-history-")); + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(storage); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + let client: AuthenticatedClient | null = null; + try { + await core.start(); client = await authenticate(core, await core.issueBootstrapTicket()); + for (let index = 1; index <= 132; index++) { + const raw = semanticInputEvent(); + const payload = raw.payload as Record; + payload.eventType = "harness.diagnostic"; + payload.sourceEventId = `owner-event-${index}`; payload.sourceSeq = index; + payload.payload = { code: "provider_startup_ownership", startup: { schema: "paperclip.provider_startup.v1", + launchId: `00000000-0000-4000-8000-${String(Math.min(index, 131)).padStart(12, "0")}`, + phase: index === 131 ? "intent" : index === 132 ? "initialization_failed" : "spawned", attemptedProcessGeneration: Math.min(index, 131), + configurationFingerprint: `sha256:${"a".repeat(64)}`, origin: identity, + ...(index <= 130 ? { processId: 10000 + index, processGroupId: 10000 + index } : {}), + ...(index === 132 ? { failedStage: "spawn" } : {}), processTreeRetired: false } }; + sendSecure(client, raw); expect((await receiveSecure(client))?.kind).toBe("ack"); + } + expect(core.store.state.committedEvents).toEqual([]); + expect(Buffer.byteLength(JSON.stringify(core.store.state))).toBeLessThan(8192); + const first = await core.readProcessOwners(); + expect(first.records).toHaveLength(128); + expect((await core.readProcessOwners(first.nextAfter!, 128, first.generation)).records).toHaveLength(2); + client.socket.destroy(); client = null; + await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + expect((await core.readProcessOwners("", 1)).records[0]?.body).toMatchObject({ sourceEventId: "owner-event-1", startup: { processId: 10001, processTreeRetired: false } }); + expect((await store.getRecord(identity.runId, "event", "owner-event-1"))?.body.eventType).toBe("harness.diagnostic"); + expect(await store.getWork("process-owner", "00000000-0000-4000-8000-000000000131")).toBeNull(); + expect((await store.getRecord(identity.runId, "event", "owner-event-132"))?.body.eventType).toBe("harness.diagnostic"); + await core.issueBootstrapTicket(); + await expect(core.readProcessOwners(first.nextAfter!, 128, first.generation)).rejects.toThrow("stale_authority"); + } finally { client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); rmSync(root, { recursive: true, force: true }); } +}, 30_000); + +it("replays the exact normalized outbox across raw ACK, mapping, run-log commit and consumer ACK crashes", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-normalized-delivery-")); + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(storage); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + let client: AuthenticatedClient | null = null; + try { + await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket()); + const raw = semanticInputEvent(); + (raw.payload as Record).eventType = "harness.diagnostic"; + (raw.payload as Record).payload = { code: "raw-consumer-test" }; + sendSecure(client!, raw); + expect((await receiveSecure(client!))?.kind).toBe("ack"); + // The runner's ACK is not the normalized consumer's checkpoint. + expect(core.normalizedDelivery()!.load()).toBeNull(); + const event = { schema: "paperclip.prp.event.v1", sourceEventId: `${identity.runnerInstanceId}:${identity.runId}:1`, sourceSeq: 1, + sourceInstanceId: identity.runnerInstanceId, sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, + eventType: "harness.diagnostic", schemaVersion: 1, priority: 1, emittedAt: "2026-09-28T00:00:00.000Z", payload: { code: "normalized" } } as const; + const consumer = core.normalizedDelivery()!; + // A retained counter at its old hard ceiling must not block unrelated + // semantic identities, and old exact receipts must remain queryable. + core.store.state.indexedState!.driverReceiptSequence = "9223372036854775807"; + await core.store.save(); + await expect(consumer.commit({ expectedRevision: 0, raw: { epoch: identity.runId, sourceSeq: 2, ordinal: 1 }, driver: {}, events: [event] })).rejects.toThrow("passed the committed raw inbox"); + await consumer.commit({ expectedRevision: 0, raw: { epoch: identity.runId, sourceSeq: 1, ordinal: 1 }, driver: { reduced: true }, events: [event], receipts: [ + { collection: "terminal", key: "call-1", value: { accepted: 1 } }, + { collection: "terminal", key: "call-2", value: { accepted: 2 } }, + ] }); + expect(core.store.state.indexedState!.driverReceiptSequence).toBeUndefined(); + await expect(consumer.commit({ expectedRevision: 0, raw: { epoch: identity.runId, sourceSeq: 1, ordinal: 1 }, driver: {}, events: [] })).rejects.toThrow("revision changed"); + client!.socket.destroy(); client = null; + await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + const reopened = core.normalizedDelivery()!; + expect(await reopened.history!.get("terminal", "call-1")).toEqual({ accepted: 1 }); + expect(await reopened.history!.get("terminal", "call-2")).toEqual({ accepted: 2 }); + await expect(reopened.commit({ expectedRevision: reopened.load()!.revision, raw: { epoch: identity.runId, sourceSeq: 1, ordinal: 1 }, driver: {}, events: [], + receipts: [{ collection: "terminal", key: "call-1", value: { accepted: 99 } }] })).rejects.toThrow("receipt_conflict"); + expect(reopened.load()).toMatchObject({ raw: { epoch: identity.runId, sourceSeq: 1, ordinal: 1 }, driver: { reduced: true }, pending: [event] }); + // If the run-log write committed but its receipt was lost, replay these + // original bytes (including timestamp); the run-log's exact key dedupes it. + await expect(reopened.acknowledge({ ...event, payload: { code: "different" } })).rejects.toThrow("differs from pending event"); + expect(reopened.load()!.pending).toEqual([event]); + await reopened.acknowledge(event); + expect(reopened.load()).toMatchObject({ produced: 1, acknowledged: 1, pending: [] }); + await core.stop(); await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + expect(core.normalizedDelivery()!.load()).toMatchObject({ raw: { epoch: identity.runId, sourceSeq: 1, ordinal: 1 }, driver: { reduced: true }, pending: [] }); + } finally { + client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each([64, 256 * 1024])("backpressures real encrypted input while a durable commit is stalled (payload bytes %s)", async payloadBytes => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-ingress-pressure-")); + const store = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store }); + let client: AuthenticatedClient | null = null; + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let incoming = 0, pauses = 0; + const attach = core.attachWireConnection.bind(core); + const wireSpy = vi.spyOn(core, "attachWireConnection").mockImplementation(wire => attach({ + onJson: listener => wire.onJson(value => { incoming++; listener(value); }), + onClose: wire.onClose.bind(wire), close: wire.close.bind(wire), sendJson: wire.sendJson.bind(wire), + pauseRead: () => { pauses++; wire.pauseRead!(); }, resumeRead: () => wire.resumeRead!(), + })); + try { + await core.start(); client = await authenticate(core, await core.issueBootstrapTicket()); + const commit = store.commit.bind(store); + const spy = vi.spyOn(store, "commit").mockImplementationOnce(async input => { await gate; return commit(input); }); + const before = incoming, count = 40; + for (let seq = 1; seq <= count; seq++) { + const envelope = semanticInputEvent(seq); + envelope.payload = { ...(envelope.payload as Record), eventType: "item.delta", priority: 2, payload: { delta: "x".repeat(payloadBytes) } }; + sendSecure(client, envelope); + } + await vi.waitFor(() => expect(pauses).toBeGreaterThan(0)); + expect(incoming - before).toBeLessThanOrEqual(16); + expect(core.store.state.ackedSourceSeq).toBe(0); + release(); + for (let seq = 1; seq <= count; seq++) expect((await receiveSecure(client))?.kind).toBe("ack"); + expect(core.store.state.ackedSourceSeq).toBe(count); + expect(core.activeRunnerConnectionCount()).toBe(1); + expect((await store.getRecord(identity.runId, "event", "semantic-event-1"))?.sequence).toBe("1"); + expect((await store.getRecord(identity.runId, "event", `semantic-event-${count}`))?.sequence).toBe(String(count)); + spy.mockRestore(); + } finally { + release(); client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); + wireSpy.mockRestore(); await store.close(); rmSync(root, { recursive: true, force: true }); + } +}, 30_000); + +it("disconnects a transport without flow control before it can enqueue unbounded closures", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-ingress-admission-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let receive!: (value: unknown) => void, onClose!: (reason: object) => void; + const close = vi.fn((code: number | undefined) => onClose({ code })); + let encoded = 0; + try { + core.attachWireConnection({ onJson: listener => { receive = listener; }, onClose: listener => { onClose = listener; }, sendJson() {}, close }); + for (let index = 0; index < 10_000; index++) receive({ toJSON() { encoded++; return { protocol: "paperclip.runner", version: 1, kind: "test" }; } }); + expect(close).toHaveBeenCalledWith(1013); + expect(encoded).toBe(65); + await core.stop(); await core.drainPendingConnectionProcessing(); + expect(core.store.state.ackedSourceSeq).toBe(0); + } finally { await core.stop(); await core.drainPendingConnectionProcessing(); rmSync(root, { recursive: true, force: true }); } +}); + +it("publishes indexed event cursors only after their storage commit", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-authority-publication-")); + const store = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store }); + let client: AuthenticatedClient | null = null; + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + const waiting = new Promise((resolve) => { entered = resolve; }); + let spy: ReturnType | undefined; + try { + await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket()); + const commit = store.commit.bind(store); + spy = vi.spyOn(store, "commit").mockImplementationOnce(async (input) => { + entered(); + await gate; + return commit(input); + }); + const event = semanticInputEvent(); + const payload = event.payload as Record; + payload.eventType = "harness.diagnostic"; + payload.payload = { code: "publication-test" }; + sendSecure(client!, event); + await waiting; + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.indexedState?.providerEverStarted).toBe(false); + expect((await store.load())?.state.ackedSourceSeq).toBe(0); + release(); + expect((await receiveSecure(client!))?.kind).toBe("ack"); + expect(core.store.state.ackedSourceSeq).toBe(1); + expect((await core.readCommittedEvents(0))[0]?.sourceEventId).toBe("semantic-event-1"); + } finally { + release(); + client?.socket.destroy(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + spy?.mockRestore(); + await store.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("never recreates an activated authority whose current row was lost", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-missing-authority-")); + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(storage); + try { + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store }); + const snapshot = (await store.load())!; + const receipt = { epoch: identity.runId, kind: "event" as const, id: "retained-event", sequence: "1", body: { accepted: true } }; + await store.commit({ expectedGeneration: snapshot.generation, state: snapshot.state, records: [receipt] }); + const locator = readFileSync(core.store.path, "utf8"); + await core.stop(); + await store.close(); + const damaged = new DatabaseSync(storage.path); + try { damaged.exec("DELETE FROM current_state WHERE key='authority'"); } + finally { damaged.close(); } + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + await expect(DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store })).rejects.toThrow("activated current authority is missing"); + expect(await store.load()).toBeNull(); + expect(await store.getRecord(identity.runId, "event", receipt.id)).toEqual(receipt); + expect(readFileSync(core.store.path, "utf8")).toBe(locator); + } finally { + await store.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("keeps indexed command receipts beyond the legacy cap and replays them after restart", async () => { + const commandCount = process.env.PAPERCLIP_HISTORY_QUALIFICATION_COMMANDS === "100000" ? 100_000 : 600; + const started = performance.now(); + const root = mkdtempSync(resolve(tmpdir(), "paperclip-indexed-prp-")); + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(storage); + let core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store, connectionLeaseTtlMs: 3_600_000 }); + let client: AuthenticatedClient | null = null; + try { + await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket()); + expect(client).not.toBeNull(); + for (let index = 1; index <= commandCount; index++) { + const command = await core.queueCommand("session.snapshot", { index, optional: undefined }, `snapshot-${index}`, true); + const frame = await receiveSecure(client!); + expect(frame).toMatchObject({ kind: "command", payload: { commandId: command.commandId, controllerSeq: index } }); + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_result", ...identity, + payload: { commandId: command.commandId, commandType: command.type, controllerSeq: index, status: "completed", result: { index } } }); + await vi.waitFor(async () => expect((await core.getCommand(command.commandId))?.status).toBe("completed"), { interval: 1, timeout: 5_000 }); + } + expect(core.store.state.commands).toHaveLength(1); + const persisted = (await store.load())!.state; + expect(persisted.commands).toEqual([]); + expect(persisted.indexedState).toMatchObject({ schema: "paperclip.runner.current-authority.v2", recoveryEvidence: { + schema: "paperclip.current-evidence.v1", records: { "command:session.snapshot": { epoch: identity.runId, kind: "command", id: `snapshot-${commandCount}` } }, + } }); + const proof = await readDurableControlPlaneState(root); + expect(proof.commands).toEqual(core.store.state.commands); + expect(Buffer.byteLength(JSON.stringify((await store.load())!.state))).toBeLessThan(8192); + const old = await core.getCommand("snapshot-1"); + expect(old?.result).toMatchObject({ result: { index: 1 } }); + client!.socket.destroy(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + await store.close(); + store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store, connectionLeaseTtlMs: 3_600_000 }); + expect(await core.queueCommand("session.snapshot", { index: 1 }, "snapshot-1")).toEqual(old); + await expect(core.queueCommand("session.snapshot", { index: 2 }, "snapshot-1")).rejects.toThrow("replay conflicts"); + expect((await core.queueCommand("session.snapshot", { index: commandCount + 1 }, `snapshot-${commandCount + 1}`)).controllerSeq).toBe(commandCount + 1); + if (commandCount === 100_000) console.info(JSON.stringify({ qualification: "indexed-controller-commands", completedCommands: commandCount, retainedCommands: core.store.state.commands.length, currentBytes: Buffer.byteLength(JSON.stringify(core.store.state)), elapsedMs: Math.round(performance.now() - started) })); + expect(core.store.state.commands).toHaveLength(2); + } finally { + client?.socket.destroy(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + await store.close(); + rmSync(root, { recursive: true, force: true }); + } +}, process.env.PAPERCLIP_HISTORY_QUALIFICATION_COMMANDS === "100000" ? 900_000 : 60_000); + function renewalRequest(client: AuthenticatedClient, expiresAt: number): Record { return { protocol: "paperclip.runner", version: client.welcome.version, @@ -66,7 +503,7 @@ it("renews one authenticated connection for three weeks without replacing its au }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; let expiry = Number((client.welcome.payload as Record).connectionLeaseExpiresAtUnixMs); const leaseId = client.welcome.connectionLeaseId; for (let hour = 0; hour < 21 * 24; hour += 3) { @@ -116,7 +553,7 @@ it.each(["expired", "revoked", "wrong-run", "wrong-connection", "wrong-epoch", " let clock: ReturnType | undefined; try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; const expiry = Number((client.welcome.payload as Record).connectionLeaseExpiresAtUnixMs); const request = renewalRequest(client, expiry); if (fault === "expired") clock = vi.spyOn(Date, "now").mockReturnValue(expiry); @@ -136,7 +573,7 @@ it.each(["expired", "revoked", "wrong-run", "wrong-connection", "wrong-epoch", " }, ); -it("persists the initial warm attachment seed idempotently and rejects replacement", () => { +it("persists the initial warm attachment seed idempotently and rejects replacement", async () => { const root = mkdtempSync( resolve(tmpdir(), "runner-initial-attachment-seed-test-"), ); @@ -155,26 +592,26 @@ it("persists the initial warm attachment seed idempotently and rejects replaceme expectedRunnerVersion, expectedRunnerDigest, }); - core.persistRunAttachTemplate(template); - core.persistRunAttachTemplate(structuredClone(template)); + (await core.persistRunAttachTemplate(template)); + (await core.persistRunAttachTemplate(structuredClone(template))); expect( JSON.parse( readFileSync(resolve(root, "control-plane-state.json"), "utf8"), ).runAttachTemplate, ).toEqual(template); - expect(() => - core.persistRunAttachTemplate({ + await expect(async () => + (await core.persistRunAttachTemplate({ ...template, provider: { ...template.provider, kind: "opencode" }, - }), - ).toThrow("attachment template conflicts"); + })), + ).rejects.toThrow("attachment template conflicts"); } finally { rmSync(root, { recursive: true, force: true }); } }); -it("persists a connection-free warm attachment seed when rotating run identity", () => { +it("persists a connection-free warm attachment seed when rotating run identity", async () => { const root = mkdtempSync(resolve(tmpdir(), "runner-attachment-seed-test-")); const nextIdentity: DurableRecoveryIdentity = { ...identity, @@ -197,7 +634,7 @@ it("persists a connection-free warm attachment seed when rotating run identity", expectedRunnerVersion, expectedRunnerDigest, }); - core.rotateRunIdentity(nextIdentity, template); + (await core.rotateRunIdentity(nextIdentity, template)); const stored = JSON.parse( readFileSync(resolve(root, "control-plane-state.json"), "utf8"), @@ -771,7 +1208,9 @@ function sendMaskedJson(socket: Socket, value: unknown): void { } else if (payload.length <= 0xffff) { header.push(0x80 | 126, payload.length >>> 8, payload.length & 0xff); } else { - throw new Error("Test client frame exceeds the supported size."); + if (payload.length > 1024 * 1024) throw new Error("Test client frame exceeds the supported size."); + header.push(0x80 | 127); + for (let shift = 56n; shift >= 0n; shift -= 8n) header.push(Number((BigInt(payload.length) >> shift) & 0xffn)); } const masked = Buffer.from(payload); for (let index = 0; index < masked.length; index += 1) { @@ -805,6 +1244,8 @@ function authHello( protocolMin: 1, protocolMax: 1, warmTransitionVersion: 1, + durability: "durability.indexed_state.v1", + outputBodies: "history.output_bodies.v1", ...selectedIdentity, runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest, @@ -820,11 +1261,22 @@ async function authenticate( warmTransitionId?: string, withoutWarmCapability = false, protocolMax = 1, + commandEpochs = false, + commandResume?: { controllerEpoch: string | null; lastControllerCommandSeq: number }, + eventResume?: import("./event-epochs.js").EventResume, ): Promise { const { socket, reader } = await upgradeSocket(controlPlane.connectUrl); const material = credentialMaterial(token); const hello = authHello(material.credentialId, selectedIdentity); (hello.payload as Record).protocolMax = protocolMax; + if (commandEpochs) { + (hello.payload as Record).commandEpochs = "transport.command_epochs.v1"; + (hello.payload as Record).resume = commandResume ?? { + controllerEpoch: controlPlane.store.state.indexedState?.controllerEpoch ?? null, + lastControllerCommandSeq: (controlPlane.store.state.indexedState?.nextControllerSeq ?? 1) - 1, + }; + } + if (eventResume) Object.assign(hello.payload as object, { eventEpochs: "transport.event_epochs.v1", eventResume }); (hello.payload as Record).runnerDigest = runnerDigest; if (warmTransitionId !== undefined) (hello.payload as Record).warmTransitionId = @@ -927,8 +1379,8 @@ it.each([ let client: AuthenticatedClient | null = null; try { await core.start(); - const command = core.queueCommand("turn.stop", {}); - const ticket = core.issueBootstrapTicket(); + const command = (await core.queueCommand("turn.stop", {})); + const ticket = (await core.issueBootstrapTicket()); const ticketId = credentialMaterial(ticket).credentialId; authenticating = authenticate(core, ticket); await waiting; @@ -1023,10 +1475,10 @@ it("denies held admission if the previous authenticated owner latches integrity let admission: Promise | undefined; try { await core.start(); - old = await authenticate(core, core.issueBootstrapTicket()); + old = await authenticate(core, (await core.issueBootstrapTicket())); expect(old).not.toBeNull(); - const command = core.queueCommand("turn.stop", {}); - const ticket = core.issueBootstrapTicket(); + const command = (await core.queueCommand("turn.stop", {})); + const ticket = (await core.issueBootstrapTicket()); const ticketId = credentialMaterial(ticket).credentialId; hold = true; admission = authenticate(core, ticket); @@ -1218,7 +1670,7 @@ it.each(["committed", "rejected"] as const)( const retired = vi.fn(); try { await core.start(); - client = (await authenticate(core, core.issueBootstrapTicket()))!; + client = (await authenticate(core, (await core.issueBootstrapTicket())))!; const beforeFrames = admittedFrames; for (const sourceSeq of [1, 2]) { const envelope = semanticInputEvent(sourceSeq); @@ -1305,7 +1757,7 @@ it.each([false, true])( let drained: Promise | undefined; try { await core.start(); - const ticket = core.issueBootstrapTicket(); + const ticket = (await core.issueBootstrapTicket()); const ticketId = credentialMaterial(ticket).credentialId; authenticating = authenticate(core, ticket); await waiting; @@ -1500,7 +1952,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; const before = readFileSync(core.store.path, "utf8"); sendSecure(client, eventAt(4097, true).envelope); if (mode === "all_pending") { @@ -1596,7 +2048,7 @@ describe.sequential("DurablePrpControlPlane", () => { const core = authority!; const client = (await authenticate( core, - core.issueBootstrapTicket(), + (await core.issueBootstrapTicket()), identity, runnerDigest, ))!; @@ -1652,7 +2104,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(client, corruptSemanticInputDigest()); await expect(receiveSecure(client)).resolves.toBeNull(); expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); @@ -1677,15 +2129,15 @@ describe.sequential("DurablePrpControlPlane", () => { expect(core.store.state.ackedSourceSeq).toBe(0); expect(onCommittedEvent).not.toHaveBeenCalled(); expect(onSemanticToolInput).not.toHaveBeenCalled(); - expect(() => - core.rotateRunIdentity({ ...identity, runId: "other-run" }), - ).toThrow(error); + await expect(async () => + (await core.rotateRunIdentity({ ...identity, runId: "other-run" })), + ).rejects.toThrow(error); - const suspend = core.queueCommand( + const suspend = (await core.queueCommand( "runner.suspend", {}, "suspend-after-integrity-fault", - ); + )); const cleanup = (await authenticate(core, client.leaseToken!))!; expect(cleanup.welcome.payload).toMatchObject({ pendingCommands: [ @@ -1744,7 +2196,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(client, semanticInputEvent()); await expect(receiveSecure(client)).resolves.toBeNull(); expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(error); @@ -1782,7 +2234,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(client, semanticInputEvent()); await expect(receiveSecure(client)).resolves.toMatchObject({ kind: "ack", @@ -1901,7 +2353,7 @@ describe.sequential("DurablePrpControlPlane", () => { } else { const client = (await authenticate( core, - core.issueBootstrapTicket(), + (await core.issueBootstrapTicket()), ))!; sendSecure(client, event); await expect(receiveSecure(client)).resolves.toBeNull(); @@ -1911,7 +2363,7 @@ describe.sequential("DurablePrpControlPlane", () => { expect(onSemanticToolInput).not.toHaveBeenCalled(); const legitimate = (await authenticate( core, - core.issueBootstrapTicket(), + (await core.issueBootstrapTicket()), ))!; sendSecure(legitimate, semanticInputEvent()); await expect(receiveSecure(legitimate)).resolves.toMatchObject({ @@ -1951,7 +2403,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const first = (await authenticate(core, core.issueBootstrapTicket()))!; + const first = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(first, semanticInputEvent()); await vi.waitFor(() => expect(onCommittedEvent).toHaveBeenCalledTimes(1)); const replacement = (await authenticate(core, first.leaseToken!))!; @@ -1984,7 +2436,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const client = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(client, corruptSemanticInputDigest()); await expect(receiveSecure(client)).resolves.toBeNull(); expect(onProtocolIntegrityError).not.toHaveBeenCalled(); @@ -2023,7 +2475,7 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await core.start(); - const first = (await authenticate(core, core.issueBootstrapTicket()))!; + const first = (await authenticate(core, (await core.issueBootstrapTicket())))!; sendSecure(first, semanticInputEvent()); await expect(receiveSecure(first)).resolves.toBeNull(); expect(core.store.state.ackedSourceSeq).toBe(0); @@ -2058,7 +2510,7 @@ describe.sequential("DurablePrpControlPlane", () => { const clients: AuthenticatedClient[] = []; try { await core.start(); - const first = (await authenticate(core, core.issueBootstrapTicket()))!; + const first = (await authenticate(core, (await core.issueBootstrapTicket())))!; clients.push(first); expect(first.welcome.version).toBe(1); first.socket.destroy(); @@ -2076,7 +2528,7 @@ describe.sequential("DurablePrpControlPlane", () => { reconnect.socket.destroy(); // A replacement process has no in-memory lease. Its owner issues the // normal one-use bootstrap for the unchanged, validated run authority. - const freshTicket = core.issueBootstrapTicket(); + const freshTicket = (await core.issueBootstrapTicket()); const replacement = (await authenticate( core, freshTicket, @@ -2118,29 +2570,29 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await controlPlane.start(); - const cancel = controlPlane.queueCommand( + const cancel = (await controlPlane.queueCommand( "run.cancel", { reason: "test" }, "command-cancel-1", - ); + )); expect( - controlPlane.queueCommand( + (await controlPlane.queueCommand( "run.cancel", { reason: "test" }, "command-cancel-1", - ), + )), ).toEqual(cancel); - expect(() => - controlPlane.queueCommand( + await expect(async () => + (await controlPlane.queueCommand( "run.cancel", { reason: "different" }, "command-cancel-1", - ), - ).toThrow("command replay conflicts"); - expect(() => controlPlane.queueCommand("unknown.command")).toThrow( + )), + ).rejects.toThrow("command replay conflicts"); + await expect(async () => (await controlPlane.queueCommand("unknown.command"))).rejects.toThrow( "command is invalid", ); - const ticket = controlPlane.issueBootstrapTicket(); + const ticket = (await controlPlane.issueBootstrapTicket()); const first = await authenticate(controlPlane, ticket); expect(first?.leaseToken).toEqual(expect.any(String)); first?.socket.destroy(); @@ -2155,7 +2607,7 @@ describe.sequential("DurablePrpControlPlane", () => { const wrongRun = await authenticate( controlPlane, - controlPlane.issueBootstrapTicket(), + (await controlPlane.issueBootstrapTicket()), { ...identity, runId: "00000000-0000-4000-8000-000000000999", @@ -2184,7 +2636,7 @@ describe.sequential("DurablePrpControlPlane", () => { let leaseToken: string; try { await first.start(); - const client = await authenticate(first, first.issueBootstrapTicket()); + const client = await authenticate(first, (await first.issueBootstrapTicket())); leaseToken = client!.leaseToken!; const validation = validatePrpEvent(semanticInputEvent().payload); expect(validation, JSON.stringify(validation)).toMatchObject({ @@ -2268,7 +2720,7 @@ describe.sequential("DurablePrpControlPlane", () => { let leaseToken: string; try { await first.start(); - const client = await authenticate(first, first.issueBootstrapTicket()); + const client = await authenticate(first, (await first.issueBootstrapTicket())); leaseToken = client!.leaseToken!; sendSecure(client!, semanticInputEvent()); await expect(receiveSecure(client!)).resolves.toBeNull(); @@ -2314,19 +2766,19 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await controlPlane.start(); - const journaled = controlPlane.queueCommand( + const journaled = (await controlPlane.queueCommand( "semantic_tool.result", { callId: "call-1" }, "command-tool-1", - ); - controlPlane.queueCommand( + )); + (await controlPlane.queueCommand( "turn.interrupt", { turnId: "turn-1" }, "command-interrupt-1", - ); + )); const client = await authenticate( controlPlane, - controlPlane.issueBootstrapTicket(), + (await controlPlane.issueBootstrapTicket()), ); expect(client?.welcome.payload).toMatchObject({ pendingCommands: [ @@ -2420,10 +2872,10 @@ describe.sequential("DurablePrpControlPlane", () => { await controlPlane.start(); client = await authenticate( controlPlane, - controlPlane.issueBootstrapTicket(), + (await controlPlane.issueBootstrapTicket()), ); expect(client).not.toBeNull(); - const command = controlPlane.queueCommand( + const command = (await controlPlane.queueCommand( "run.attach", { paperclipNextAuthority: { @@ -2436,7 +2888,7 @@ describe.sequential("DurablePrpControlPlane", () => { }, "command-attach-ack-loss", true, - ); + )); await expect(receiveSecure(client!)).resolves.toMatchObject({ kind: "command", }); @@ -2463,7 +2915,7 @@ describe.sequential("DurablePrpControlPlane", () => { ).toEqual(result.payload); // The result ACK remains unread on this lost connection. Observing the // result must not delete the old receipt or activate its new identity. - controlPlane.rotateRunIdentity(nextIdentity); + (await controlPlane.rotateRunIdentity(nextIdentity)); expect(controlPlane.store.state.identity).toEqual(identity); client!.socket.destroy(); replay = await authenticate(controlPlane, leaseToken, identity); @@ -2530,10 +2982,10 @@ describe.sequential("DurablePrpControlPlane", () => { let successor: AuthenticatedClient | null = null; try { await core.start(); - const unusedTicket = core.issueBootstrapTicket(); - peer = await authenticateVersion(core, core.issueBootstrapTicket()); + const unusedTicket = (await core.issueBootstrapTicket()); + peer = await authenticateVersion(core, (await core.issueBootstrapTicket())); const oldToken = peer!.leaseToken!; - const command = core.queueCommand( + const command = (await core.queueCommand( "run.attach", { paperclipNextAuthority: { @@ -2543,7 +2995,7 @@ describe.sequential("DurablePrpControlPlane", () => { }, "command-transition", true, - ); + )); await expect(receiveSecure(peer!)).resolves.toMatchObject({ kind: "command", }); @@ -2566,10 +3018,10 @@ describe.sequential("DurablePrpControlPlane", () => { .warmTransition as Record; const transitionId = receipt.transitionId as string; expect(core.store.state.identity).toEqual(identity); - expect(() => core.queueCommand("turn.start", {})).toThrow( + await expect(async () => (await core.queueCommand("turn.start", {}))).rejects.toThrow( "exact cached attachment replay", ); - expect(() => core.issueBootstrapTicket()).toThrow( + await expect(async () => (await core.issueBootstrapTicket())).rejects.toThrow( "explicit one-use bootstrap", ); if (mode === "revoked-live-peer") { @@ -2713,12 +3165,12 @@ describe.sequential("DurablePrpControlPlane", () => { }, }), ).toBeNull(); - expect(() => - core.issueWarmTransitionBootstrapTicket({ + await expect(async () => + (await core.issueWarmTransitionBootstrapTicket({ transitionId, runnerState: { ...runnerState, runId: nextIdentity.runId }, - }), - ).toThrow("snapshot"); + })), + ).rejects.toThrow("snapshot"); const originalLease = structuredClone( Object.values(core.store.state.leases).find( (lease) => @@ -2728,32 +3180,32 @@ describe.sequential("DurablePrpControlPlane", () => { if (mode === "revoked-bootstrap") { core.store.state.leases[originalLease.credentialId]!.revokedAt = new Date().toISOString(); - expect(() => - core.issueWarmTransitionBootstrapTicket({ + await expect(async () => + (await core.issueWarmTransitionBootstrapTicket({ transitionId, runnerState, - }), - ).toThrow("not authorized"); + })), + ).rejects.toThrow("not authorized"); } else if (mode === "expired-bootstrap") { vi.spyOn(Date, "now").mockReturnValue( originalLease.expiresAtUnixMs + 1, ); - expect(() => - core.issueWarmTransitionBootstrapTicket({ + await expect(async () => + (await core.issueWarmTransitionBootstrapTicket({ transitionId, runnerState, - }), - ).toThrow("not authorized"); + })), + ).rejects.toThrow("not authorized"); } else { - let ticket = core.issueWarmTransitionBootstrapTicket({ + let ticket = (await core.issueWarmTransitionBootstrapTicket({ transitionId, runnerState, - }); + })); if (mode === "bootstrap-before-result") { expect(core.store.state.warmTransition?.phase).toBe( "awaiting_result", ); - expect(core.getCommand(command.commandId)?.status).toBe("pending"); + expect((await core.getCommand(command.commandId))?.status).toBe("pending"); await core.stop(); core = new DurablePrpControlPlane({ stateDirectory: root, @@ -2816,10 +3268,10 @@ describe.sequential("DurablePrpControlPlane", () => { ), ).toBeNull(); wireSpy.mockRestore(); - ticket = core.issueWarmTransitionBootstrapTicket({ + ticket = (await core.issueWarmTransitionBootstrapTicket({ transitionId, runnerState, - }); + })); } successor = await authenticateVersion( core, @@ -2857,7 +3309,7 @@ describe.sequential("DurablePrpControlPlane", () => { ), ).toBeNull(); if (mode === "bootstrap-before-result") - expect(core.getCommand(command.commandId)?.status).toBe("pending"); + expect((await core.getCommand(command.commandId))?.status).toBe("pending"); sendSecure(successor!, { protocol: "paperclip.runner", version: selectedProtocol, @@ -2868,7 +3320,7 @@ describe.sequential("DurablePrpControlPlane", () => { kind: "command_result_ack", payload: { warmTransition: receipt }, }); - expect(core.getCommand(command.commandId)?.status).toBe("completed"); + expect((await core.getCommand(command.commandId))?.status).toBe("completed"); } } else { successor = await authenticateVersion( @@ -2893,12 +3345,12 @@ describe.sequential("DurablePrpControlPlane", () => { transitionId, ), ).toBeNull(); - const next = core.queueCommand( + const next = (await core.queueCommand( "session.snapshot", {}, "next-snapshot", true, - ); + )); expect( core.store.state.commandDeliveryCounts[next.commandId], ).toBeUndefined(); @@ -2976,7 +3428,7 @@ describe.sequential("DurablePrpControlPlane", () => { vi.spyOn(Date, "now").mockReturnValue( Number(receipt.leaseExpiresAtUnixMs) + 1, ); - core.issueBootstrapTicket(); + (await core.issueBootstrapTicket()); expect(Object.values(core.store.state.leases)).toHaveLength(0); vi.restoreAllMocks(); successor!.socket.destroy(); @@ -2987,7 +3439,7 @@ describe.sequential("DurablePrpControlPlane", () => { expectedRunnerVersion, expectedRunnerDigest, }); - expect(core.getCommand(command.commandId)?.status).toBe("completed"); + expect((await core.getCommand(command.commandId))?.status).toBe("completed"); } } } finally { @@ -3015,9 +3467,9 @@ describe.sequential("DurablePrpControlPlane", () => { let syncSpy: { mockRestore(): void } | undefined; try { await core.start(); - client = await authenticate(core, core.issueBootstrapTicket()); + client = await authenticate(core, (await core.issueBootstrapTicket())); const token = client!.leaseToken!; - const command = core.queueCommand( + const command = (await core.queueCommand( "run.attach", { paperclipNextAuthority: { @@ -3032,7 +3484,7 @@ describe.sequential("DurablePrpControlPlane", () => { }, "fsync-attach", true, - ); + )); await receiveSecure(client!); const result = { protocol: "paperclip.runner", @@ -3069,10 +3521,10 @@ describe.sequential("DurablePrpControlPlane", () => { const disk = readFileSync(core.store.path, "utf8"); expect(JSON.parse(disk).warmTransition.phase).toBe("prepared"); expect(core.store.state.commands[0]?.status).toBe("pending"); - expect(() => core.queueCommand("turn.start", {})).toThrow( + await expect(async () => (await core.queueCommand("turn.start", {}))).rejects.toThrow( "indeterminate; reload", ); - expect(() => core.issueBootstrapTicket()).toThrow( + await expect(async () => (await core.issueBootstrapTicket())).rejects.toThrow( "indeterminate; reload", ); expect(readFileSync(core.store.path, "utf8")).toBe(disk); @@ -3139,11 +3591,11 @@ describe.sequential("DurablePrpControlPlane", () => { let pending: Promise | undefined; try { await core.start(); - first = await authenticate(core, core.issueBootstrapTicket()); + first = await authenticate(core, (await core.issueBootstrapTicket())); const unrelatedToken = first!.leaseToken!; first!.socket.destroy(); - participant = await authenticate(core, core.issueBootstrapTicket()); - const command = core.queueCommand( + participant = await authenticate(core, (await core.issueBootstrapTicket())); + const command = (await core.queueCommand( "run.attach", { paperclipNextAuthority: { @@ -3158,7 +3610,7 @@ describe.sequential("DurablePrpControlPlane", () => { }, "held-proof-attach", true, - ); + )); await receiveSecure(participant!); armed = true; pending = authenticate( @@ -3217,21 +3669,21 @@ describe.sequential("DurablePrpControlPlane", () => { }); try { await controlPlane.start(); - const command = controlPlane.queueCommand( + const command = (await controlPlane.queueCommand( "runner.suspend", {}, "command-suspend-1", - ); + )); const client = await authenticate( controlPlane, - controlPlane.issueBootstrapTicket(), + (await controlPlane.issueBootstrapTicket()), ); - const nextAuthorityCommand = controlPlane.queueCommand( + const nextAuthorityCommand = (await controlPlane.queueCommand( "runner.drain", {}, "command-after-suspend-1", true, - ); + )); expect( controlPlane.store.state.commandDeliveryCounts[command.commandId], ).toBe(1); @@ -3296,3 +3748,180 @@ describe.sequential("DurablePrpControlPlane", () => { } }); }); + + +it.each([identity.runId, "r".repeat(240)])("rotates command namespaces durably, reopens a lost transition and preserves earliest receipts (%s)", async (runId) => { + const epochIdentity = { ...identity, runId }; + const root = mkdtempSync(resolve(tmpdir(), "paperclip-command-epochs-")); + const store = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(epochIdentity), create: true }); + const options = { stateDirectory: root, identity: epochIdentity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store, commandEpochLimit: 4 }; + let core = await DurablePrpControlPlane.open(options), client: AuthenticatedClient | null = null; + const capable = (token: string) => authenticate(core, token, epochIdentity, expectedRunnerDigest, undefined, false, 1, true); + let first: Awaited> | undefined; + try { + await core.start(); client = await capable(await core.issueBootstrapTicket()); + const lease = client!.leaseToken!; + const epochs = new Set(); + for (let index = 0; index < 13; index++) { + let admission = core.queueCommand(index === 0 ? "run.prepare" : "session.snapshot", { index }, `epoch-command-${index}`, true); + // Observe rejection immediately when the first controller is stopped. + const outcome = admission.then(value => ({ value }), error => ({ error })); + if (index > 0 && index % 4 === 0) { + const intent = await receiveSecure(client!); + expect(intent?.kind).toBe("command_epoch_rotate"); + expect(intent?.payload).toMatchObject({ finalOrdinal: 4, fromEpoch: core.store.state.indexedState!.controllerEpoch ?? null }); + expect(core.store.state.indexedState!.commandEpochTransition).toEqual(intent?.payload); + if (index === 4) { + // The receiver committed, but its reply was lost. The controller's + // durable intent must survive restart and authenticate a capable peer. + client!.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); + expect(await outcome).toHaveProperty("error"); + core = await DurablePrpControlPlane.open(options); await core.start(); + expect(await authenticate(core, lease, epochIdentity)).toBeNull(); + const pending = core.store.state.indexedState!.commandEpochTransition!; + client = await authenticate(core, lease, epochIdentity, expectedRunnerDigest, undefined, false, 1, true, + { controllerEpoch: pending.nextEpoch, lastControllerCommandSeq: 0 }); + expect((await receiveSecure(client!))?.payload).toEqual(intent?.payload); + } + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_epoch_committed", ...epochIdentity, payload: intent!.payload }); + await vi.waitFor(() => expect(core.store.state.indexedState?.commandEpochTransition).toBeUndefined(), { interval: 1 }); + if (index === 4) admission = core.queueCommand(index === 0 ? "run.prepare" : "session.snapshot", { index }, `epoch-command-${index}`, true); + } + const command = await admission; + epochs.add(command.controllerEpoch); + expect(command.controllerSeq).toBe(index % 4 + 1); + expect((await receiveSecure(client!))?.payload).toMatchObject({ commandId: command.commandId, controllerSeq: command.controllerSeq }); + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_result", ...epochIdentity, + payload: { commandId: command.commandId, commandType: command.type, controllerSeq: command.controllerSeq, status: "completed", result: { index } } }); + await vi.waitFor(async () => expect((await core.getCommand(command.commandId))?.status).toBe("completed"), { interval: 1 }); + if (index === 0) first = structuredClone((await core.getCommand(command.commandId))!); + } + expect(epochs.size).toBe(4); + expect(core.store.state.commands).toHaveLength(2); + expect(await core.queueCommand(first!.type, first!.payload, first!.commandId)).toEqual(first); + expect(core.store.state.identity).toEqual(epochIdentity); + client!.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); + core = await DurablePrpControlPlane.open(options); + expect(await core.queueCommand(first!.type, first!.payload, first!.commandId)).toEqual(first); + expect(core.store.state.commands.at(-1)?.commandId).toBe("epoch-command-12"); + expect(core.store.state.indexedState!.nextControllerSeq).toBe(2); + await core.start(); + expect(await authenticate(core, lease, epochIdentity, expectedRunnerDigest, undefined, false, 1, true, + { controllerEpoch: null, lastControllerCommandSeq: 1 })).toBeNull(); + expect(await authenticate(core, lease, epochIdentity, expectedRunnerDigest, undefined, false, 1, true, + { controllerEpoch: core.store.state.indexedState!.controllerEpoch!, lastControllerCommandSeq: 0 })).toBeNull(); + } finally { + client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); rmSync(root, { recursive: true, force: true }); + } +}, 60_000); + + +it.each(["runner.suspend", "runner.shutdown"])("does not fence a current connection for an old-epoch %s receipt", async type => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-old-terminal-")); + const store = await SqliteAuthorityStore.open({ path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, authorityStore: store, commandEpochLimit: 4 }); + let client: AuthenticatedClient | null = null; + try { + const old = await core.queueCommand(type, {}, "old-terminal"); + // Retained receipt from an already-reconciled prior process. This fixture + // does not pretend an in-flight terminal command can rotate past cleanup. + const oldResult = { commandId: old.commandId, commandType: type, controllerSeq: old.controllerSeq, status: "completed", result: {} }; + core.store.state.commands[0]!.status = "completed"; + core.store.state.commands[0]!.result = oldResult; + await core.store.save(); await core.start(); + client = await authenticate(core, await core.issueBootstrapTicket(), identity, expectedRunnerDigest, undefined, false, 1, true); + for (let index = 2; index <= 5; index++) { + const waiting = core.queueCommand("session.snapshot", { index }, `current-${index}`, true); + if (index === 5) { + const intent = await receiveSecure(client!); + expect(intent?.kind).toBe("command_epoch_rotate"); + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_epoch_committed", ...identity, payload: intent!.payload }); + } + const current = await waiting; + expect((await receiveSecure(client!))?.payload).toMatchObject({ commandId: current.commandId }); + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_result", ...identity, + payload: { commandId: current.commandId, commandType: current.type, controllerSeq: current.controllerSeq, status: "completed", result: {} } }); + await vi.waitFor(async () => expect((await core.getCommand(current.commandId))?.status).toBe("completed"), { interval: 1 }); + } + sendSecure(client!, { protocol: "paperclip.runner", version: 1, kind: "command_result", ...identity, payload: oldResult }); + await vi.waitFor(() => expect(core.store.state.duplicateCommandResults).toBe(1), { interval: 1 }); + const next = await core.queueCommand("session.snapshot", {}, "still-live", true); + expect((await receiveSecure(client!))?.payload).toMatchObject({ commandId: next.commandId, controllerSeq: 2 }); + } finally { + client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await store.close(); rmSync(root, { recursive: true, force: true }); + } +}); + + +it("rotates event epochs across lost replies and consumer lag without losing original receipts", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-event-epochs-")); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, eventEpochLimit: 4 }; + const storage = { path: resolve(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let authority = await SqliteAuthorityStore.open(storage); + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: authority }); + let client: AuthenticatedClient | null = null; + const resume = (sourceEpoch: string | undefined, sourceSeq: number, transition: import("./event-epochs.js").EventEpochTransition | null = null) => ({ sourceEpoch: sourceEpoch ?? null, nextSourceEventSeq: sourceSeq + 1, ackedSourceSeq: sourceSeq, transition }); + const connect = (token: string, input: ReturnType) => authenticate(core, token, identity, expectedRunnerDigest, undefined, false, 2, false, undefined, input); + let first: Record | undefined, epoch: string | undefined; + const transitions: import("./event-epochs.js").EventEpochTransition[] = []; + try { + await core.start(); client = await connect(await core.issueBootstrapTicket(), resume(undefined, 0)); + const lease = client!.leaseToken!; + for (let round = 0; round < 3; round++) { + for (let ordinal = 1; ordinal <= 4; ordinal++) { + const raw = semanticInputEvent(ordinal); raw.version = 2; + Object.assign(raw.payload as object, { sourceEventId: `event-${round}-${ordinal}`, ...(epoch ? { sourceEpoch: epoch } : {}), + eventType: ordinal === 1 ? "session.started" : "harness.diagnostic", payload: { round, ordinal } }); + if (!first) first = structuredClone(raw); + sendSecure(client!, raw); expect((await receiveSecure(client!))?.payload).toMatchObject({ ackedSourceSeq: ordinal, ...(epoch ? { sourceEpoch: epoch } : {}) }); + } + const transition = { schema: "paperclip.prp.event-epoch.v1" as const, runId: identity.runId, transitionId: randomUUID(), fromEpoch: epoch ?? null, nextEpoch: randomUUID(), finalOrdinal: 4 }; + transitions.push(transition); + const frame = { protocol: "paperclip.runner", version: 2, kind: "event_epoch_rotate", ...identity, payload: transition }; + sendSecure(client!, frame); + await vi.waitFor(() => expect(core.store.state.indexedState?.sourceEpoch).toBe(transition.nextEpoch)); + if (round === 0) { + // Commit reached storage, its network reply is lost, and the receiver + // restarts while normalization is still in the earliest namespace. + client!.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await authority.close(); + authority = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: authority }); await core.start(); + expect(await authenticate(core, lease, identity, expectedRunnerDigest, undefined, false, 2)).toBeNull(); + expect(await connect(lease, resume(undefined, 4))).toBeNull(); + client = await connect(lease, resume(undefined, 4, transition)); + expect(client!.welcome.payload).toMatchObject({ ackedSourceSeq: 4, sourceEpoch: null }); + sendSecure(client!, frame); + } + expect((await receiveSecure(client!))?.payload).toEqual(transition); + expect(core.store.state.ackedSourceSeq).toBe(0); + epoch = transition.nextEpoch; + } + expect((await authority.getRecord(identity.runId, "event", "event-0-1"))?.body.envelope).toEqual(first); + sendSecure(client!, first!); + expect((await receiveSecure(client!))?.payload).toEqual({ ackedSourceSeq: 4 }); // old ACK, never current + expect(core.store.state.ackedSourceSeq).toBe(0); + const port = core.normalizedDelivery()!; + let consumedEpoch: string | undefined; + for (const transition of transitions) { + const page = await core.readCommittedEvents(0, 128, consumedEpoch); + expect(page.map(event => event.sourceSeq)).toEqual([1, 2, 3, 4]); + expect(page.every(event => event.sourceEpoch === consumedEpoch)).toBe(true); + const previous = port.load(); + await port.commit({ expectedRevision: previous?.revision ?? 0, raw: { epoch: identity.runId, ...(consumedEpoch ? {sourceEpoch: consumedEpoch} : {}), sourceSeq: 4, ordinal: Number.MAX_SAFE_INTEGER }, driver: {}, events: [] }); + await expect(port.commit({ expectedRevision: port.load()!.revision, raw: { epoch: identity.runId, sourceEpoch: transition.nextEpoch, sourceSeq: 1, ordinal: 0 }, driver: {}, events: [] })).rejects.toThrow(); + await port.commit({ expectedRevision: port.load()!.revision, raw: { epoch: identity.runId, sourceEpoch: transition.nextEpoch, sourceSeq: 0, ordinal: 0 }, driver: {}, events: [] }); + consumedEpoch = transition.nextEpoch; + } + expect(port.load()!.raw.sourceEpoch).toBe(epoch); + expect(core.store.state.committedEvents.map(event => event.sourceEventId)).toEqual(["event-2-1"]); + client!.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: authority }); + expect(core.store.state.committedEvents.map(event => event.sourceEventId)).toEqual(["event-2-1"]); + expect(core.normalizedDelivery()!.load()!.raw.sourceEpoch).toBe(epoch); + // Reusing a transition identity with changed bytes cannot advance the head. + await core.start(); client = await connect(lease, resume(epoch, 0)); + sendSecure(client!, { protocol: "paperclip.runner", version: 2, kind: "event_epoch_rotate", ...identity, payload: { ...transitions[0], nextEpoch: randomUUID() } }); + expect(await receiveSecure(client!)).toBeNull(); + expect(core.store.state.indexedState!.sourceEpoch).toBe(epoch); + } finally { client?.socket.destroy(); await core.stop(); await core.drainPendingConnectionProcessing(); await authority.close(); rmSync(root, {recursive:true,force:true}); } +}, 30_000); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 747b56a62bf..11f33c4fb47 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -1,3 +1,5 @@ +import { EVENT_EPOCH_CAPABILITY, EVENT_EPOCH_LIMIT, isEventEpochTransition, isEventResume, eventEpochCloseId, compareCurrentEvents, type EventEpochTransition, type EventResume } from "./event-epochs.js"; +import { COMMAND_EPOCH_CAPABILITY, COMMAND_EPOCH_LIMIT, isCommandEpoch, isCommandEpochTransition, compareCurrentCommands, commandEpochCloseId, type CommandEpochTransition } from "./command-epochs.js"; import { spawn } from "node:child_process"; import { createCipheriv, @@ -35,6 +37,13 @@ import { type PrpEvent, } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; +import { authorityInteger, authorityJson, DurableAuthorityStoreError, INDEXED_DURABILITY_CAPABILITY, MAX_LEGACY_AUTHORITY_STATE_BYTES, type AuthorityRecord, type AuthoritySnapshot, type DurableAuthorityStore } from "./durable-authority-store.js"; +import { materializeCurrentAuthority, referenceCurrentAuthority, type CurrentAuthorityEvidence } from "./current-authority-evidence.js"; +import { processOwnerChanges, processOwnerEvidence } from "./process-owner-evidence.js"; +import { LegacyJsonReader, type LegacyJsonCursor } from "./legacy-json-reader.js"; +import { assertNoPendingLegacyMigration } from "./legacy-migration-gate.js"; +import { normalizedEpochCloseId } from "./event-epochs.js"; +import { acknowledgeNormalizedEvent, applyNormalizedBatch, validateNormalizedDelivery, type NormalizedDeliveryState, type NormalizedDeliveryPort } from "./normalized-delivery.js"; import { type DurableRecoveryCommittedEvent, type DurableRecoveryCoreCommand, @@ -51,6 +60,12 @@ const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; const transitionCoreStateSchema = "paperclip.runner.durable.control-plane-state.warm-transition.v1"; const maxFrameBytes = 1024 * 1024; +// Connection-local nonce counters are renewable. Re-authentication derives new +// keys and resumes durable cursors without changing any task/provider identity. +const maxSecureChannelFrames = 1_048_576; +// Diagnostic totals are lower bounds after saturation, never authority, +// sequence namespaces, or unique identity sources. +const incrementDiagnosticCount = (value: number) => Math.min(Number.MAX_SAFE_INTEGER, value + 1); const maxCommandBytes = maxFrameBytes - 4 * 1024; const maxCommands = 500; // A provider can emit several 100-event runner batches before the transport's @@ -59,7 +74,7 @@ const maxCommands = 500; const maxCommittedEventWindow = 4_096; // The controller admission, cleanup, and recovery readers inspect this same // journal. Keep their bound aligned with the durable store as history grows. -export const DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES = 192 * 1024 * 1024; +export const DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES = MAX_LEGACY_AUTHORITY_STATE_BYTES; const authChallengeTtlMs = 5_000; const stableIdPattern = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/; const runnerDigestPattern = /^sha256:[0-9a-f]{64}$/; @@ -128,7 +143,7 @@ interface ConnectionLeaseRecord { revokedAt: string | null; } -interface StoredCoreState { +export interface StoredCoreState { schema: typeof coreStateSchema | typeof transitionCoreStateSchema; identity: DurableRecoveryIdentity; warmTransition?: { @@ -163,6 +178,24 @@ interface StoredCoreState { malformedFrames: number; lastLeaseId: string | null; lastLeaseExpiresAt: string | null; + indexedState?: { + schema: "paperclip.runner.current-authority.v1" | "paperclip.runner.current-authority.v2"; + recoveryEvidence?: CurrentAuthorityEvidence; + nextControllerSeq: number; + sourceEpoch?: string; + lastEventEpochTransition?: EventEpochTransition; + commandEpochsNegotiated?: true; + controllerEpoch?: string; + commandEpochTransition?: CommandEpochTransition; + lastCommandEpochTransition?: CommandEpochTransition; + providerEverStarted: boolean; + externalEffectEverAdmitted: boolean; + pendingSemanticInputIds: string[]; + normalizedDelivery?: NormalizedDeliveryState; + driverReceiptSequence?: string; + processOwnerIndexVersion?: 1; + legacyActivation?: { fenceId: string; preparationDigest: string }; + }; } type PendingAuthorization = @@ -205,6 +238,13 @@ type LiveAuthorization = }; interface PendingChallenge { + durability?: string; + outputBodies?: string; + eventEpochs?: string; + eventResume?: EventResume; + eventEpochLimit?: number; + commandEpochs?: string; + commandResume?: { controllerEpoch: string | null; lastControllerCommandSeq: number }; authorization: PendingAuthorization; deadlineUnixMs: number; canonicalChallenge: string; @@ -230,6 +270,8 @@ export interface DurablePrpControlPlaneOptions { identity: DurableRecoveryIdentity; expectedRunnerVersion: string; expectedRunnerDigest: string; + /** Open via DurablePrpControlPlane.open when supplying asynchronous storage. */ + authorityStore?: DurableAuthorityStore; /** Complete caller-owned admission before consuming a credential or releasing commands. */ beforeAuthenticatedConnection?: (input: { readonly identity: DurableRecoveryIdentity; @@ -256,6 +298,12 @@ export interface DurablePrpControlPlaneOptions { error: NativeSessionProtocolIntegrityError, ) => void; connectionLeaseTtlMs?: number; + /** May lower the connection rekey interval; never raises the wire bound. */ + secureChannelFrameLimit?: number; + /** Qualification may lower, never raise, the renewable command namespace. */ + eventEpochLimit?: number; + normalizedEventEpochLimit?: number; + commandEpochLimit?: number; } export interface RunnerProcessResult { @@ -426,6 +474,7 @@ function warmTransitionReceipt( >, runnerVersion: string, runnerDigest: string, + oldSourceEpoch?: string, ): DurableWarmRunTransition { const boundary = command.payload.paperclipNextAuthority; if ( @@ -469,6 +518,7 @@ function warmTransitionReceipt( }), resultDigest: canonicalDigest(result), oldAckedSourceSeq: ackedSourceSeq, + ...(oldSourceEpoch ? { oldSourceEpoch } : {}), connection: structuredClone(boundary.connection), runnerVersion, runnerDigest, @@ -509,6 +559,7 @@ function validStoredWarmTransition(state: StoredCoreState): boolean { lease, transition.receipt.runnerVersion, transition.receipt.runnerDigest, + transition.receipt.oldSourceEpoch, ); return ( runnerDigestPattern.test(expected.runnerDigest) && @@ -520,7 +571,7 @@ function validStoredWarmTransition(state: StoredCoreState): boolean { : expected.oldIdentity, ) && (transition.phase === "activated" || - (state.ackedSourceSeq === expected.oldAckedSourceSeq && + (state.ackedSourceSeq === expected.oldAckedSourceSeq && state.indexedState?.sourceEpoch === expected.oldSourceEpoch && canonicalJson( state.commands.find( (command) => command.commandId === expected.commandId, @@ -538,8 +589,9 @@ function isRecord(value: unknown): value is Record { function unsettledSemanticInput( event: DurableRecoveryCommittedEvent, - state: Pick, + state: Pick, ): boolean { + if (state.indexedState) return state.indexedState.pendingSemanticInputIds.includes(event.sourceEventId); if ( event.eventType !== "semantic_tool.input" && event.eventType !== "mcp_app.tool_input" @@ -599,8 +651,34 @@ function unsettledSemanticInput( function isStoredCoreState( value: unknown, identity: DurableRecoveryIdentity, + allowIndexed = false, ): value is StoredCoreState { if (!isRecord(value)) return false; + if (value.indexedState !== undefined) { + const current = value.indexedState; + if (!allowIndexed || !isRecord(current) || current.schema !== "paperclip.runner.current-authority.v1" || + !Number.isSafeInteger(current.nextControllerSeq) || (current.nextControllerSeq as number) < 1 || + typeof current.providerEverStarted !== "boolean" || typeof current.externalEffectEverAdmitted !== "boolean" || + !Array.isArray(current.pendingSemanticInputIds) || current.pendingSemanticInputIds.length > maxCommands || + !current.pendingSemanticInputIds.every((id) => typeof id === "string" && stableIdPattern.test(id)) || + new Set(current.pendingSemanticInputIds).size !== current.pendingSemanticInputIds.length) return false; + if (current.commandEpochsNegotiated !== undefined && current.commandEpochsNegotiated !== true) return false; + if ((current.controllerEpoch !== undefined && !isCommandEpoch(current.controllerEpoch)) || + (current.commandEpochTransition !== undefined && (!isCommandEpochTransition(current.commandEpochTransition) || + current.commandEpochTransition.runId !== identity.runId || current.commandEpochTransition.fromEpoch !== (current.controllerEpoch ?? null) || + current.commandEpochTransition.finalOrdinal !== Number(current.nextControllerSeq) - 1))) return false; + if (current.controllerEpoch === undefined ? current.lastCommandEpochTransition !== undefined : + !isCommandEpochTransition(current.lastCommandEpochTransition) || current.lastCommandEpochTransition.nextEpoch !== current.controllerEpoch || current.lastCommandEpochTransition.runId !== identity.runId) return false; + if (current.sourceEpoch === undefined ? current.lastEventEpochTransition !== undefined : + !isCommandEpoch(current.sourceEpoch) || !isEventEpochTransition(current.lastEventEpochTransition) || current.lastEventEpochTransition.nextEpoch !== current.sourceEpoch || current.lastEventEpochTransition.runId !== identity.runId) return false; + if (current.normalizedDelivery !== undefined) { + try { validateNormalizedDelivery(current.normalizedDelivery as NormalizedDeliveryState); } + catch { return false; } + const delivery = current.normalizedDelivery as NormalizedDeliveryState; + if (delivery.raw.epoch !== identity.runId || (delivery.raw.sourceEpoch === current.sourceEpoch && delivery.raw.sourceSeq > Number(value.ackedSourceSeq)) + || delivery.pending.some((event) => event.runId !== identity.runId || event.sourceInstanceId !== identity.runnerInstanceId || event.normalizedSessionId !== identity.normalizedSessionId || event.sourceKind !== "runner")) return false; + } + } const commands = value.commands; const events = value.committedEvents; if ( @@ -610,7 +688,7 @@ function isStoredCoreState( !isRecord(value.tickets) || !isRecord(value.leases) || !Array.isArray(commands) || - commands.length > maxCommands || + commands.length > maxCommands + (allowIndexed ? 64 : 0) || !Array.isArray(events) || events.length > maxCommittedEventWindow || !Number.isSafeInteger(value.ackedSourceSeq) || @@ -653,6 +731,7 @@ function isStoredCoreState( }, completed.receipt.runnerVersion as string, completed.receipt.runnerDigest as string, + completed.receipt.oldSourceEpoch as string | undefined, ), ) !== canonicalJson(completed.receipt) ) @@ -677,7 +756,11 @@ function isStoredCoreState( typeof command.commandId === "string" && stableIdPattern.test(command.commandId) && command.commandId.length <= 160 && - command.controllerSeq === index + 1 && + (value.indexedState + ? Number.isSafeInteger(command.controllerSeq) && (command.controllerSeq as number) > 0 + : command.controllerSeq === index + 1) && + (command.controllerEpoch === undefined || (value.indexedState && isCommandEpoch(command.controllerEpoch))) && + (command.status !== "pending" || command.controllerEpoch === (value.indexedState as StoredCoreState["indexedState"])?.controllerEpoch) && typeof command.type === "string" && commandTypes.has(command.type) && typeof command.issuedAt === "string" && @@ -698,6 +781,7 @@ function isStoredCoreState( !events.every( (event) => isRecord(event) && + (event.sourceEpoch === undefined || (allowIndexed && isCommandEpoch(event.sourceEpoch))) && Number.isSafeInteger(event.sourceSeq) && (event.sourceSeq as number) > 0 && typeof event.sourceEventId === "string" && @@ -755,7 +839,7 @@ function warmTransitionRecoveryProof(input: WarmTransitionInspectionInput): { !exactIdentity(input.expectedNewIdentity) || !isRecord(state) || !exactIdentity(state.identity) || - !isStoredCoreState(state, state.identity) || + !isStoredCoreState(state, state.identity, true) || !isRecord(runner) || runner.schema !== "paperclip.runner.durable.state.warm-transition.v1" ) @@ -826,6 +910,7 @@ function warmTransitionRecoveryProof(input: WarmTransitionInspectionInput): { original, input.expectedRunnerVersion, input.expectedRunnerDigest, + state.indexedState?.sourceEpoch, ); if (canonicalJson(expected) !== canonicalJson(receipt)) return null; transition = { @@ -888,6 +973,7 @@ function warmTransitionRecoveryProof(input: WarmTransitionInspectionInput): { canonicalJson({ ...wire, deadlineAt: null, precondition: null }) || canonicalJson(pending.result) !== canonicalJson(transition.expectedResult ?? transition.command.result) || + runner.sourceEpoch !== (pending.phase === "prepared" ? transition.receipt.oldSourceEpoch : undefined) || runner.ackedSourceSeq !== (pending.phase === "prepared" ? transition.receipt.oldAckedSourceSeq @@ -1177,12 +1263,21 @@ function atomicPrivateWrite(path: string, contents: string): void { } } +function encodeLegacyState(state: StoredCoreState): string { + const bytes = `${JSON.stringify(state, null, 2)}\n`; + if (Buffer.byteLength(bytes) > DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES) { + throw new DurableAuthorityStoreError("storage_pressure", "legacy journal requires indexed migration before more history can be admitted"); + } + return bytes; +} + class DurableCoreStore { readonly path: string; #state: StoredCoreState; #writeIndeterminate = false; constructor(directory: string, identity: DurableRecoveryIdentity) { + assertNoPendingLegacyMigration(directory); try { const metadata = lstatSync(directory); if (metadata.isSymbolicLink() || !metadata.isDirectory()) { @@ -1208,7 +1303,7 @@ class DurableCoreStore { this.#state = parsed; } else { this.#state = initialCoreState(identity); - this.save(); + atomicPrivateWrite(this.path, encodeLegacyState(this.#state)); } } @@ -1216,12 +1311,13 @@ class DurableCoreStore { return this.#state; } - save(): void { + async save(): Promise { this.assertWritable(); - atomicPrivateWrite(this.path, `${JSON.stringify(this.#state, null, 2)}\n`); + await this.commit(this.#state); } assertWritable(): void { + assertNoPendingLegacyMigration(dirname(this.path)); if (this.#writeIndeterminate) throw new Error( "Durable authority commit is indeterminate; reload is required.", @@ -1229,10 +1325,10 @@ class DurableCoreStore { } /** Persist a complete candidate before publishing any new authority in memory. */ - commit(candidate: StoredCoreState): void { + async commit(candidate: StoredCoreState): Promise { this.assertWritable(); try { - atomicPrivateWrite(this.path, `${JSON.stringify(candidate, null, 2)}\n`); + atomicPrivateWrite(this.path, encodeLegacyState(candidate)); this.#state = candidate; } catch (error) { // Rename may already have succeeded before directory fsync failed. @@ -1243,6 +1339,344 @@ class DurableCoreStore { } } +function indexedCurrentState(): NonNullable { + return { schema: "paperclip.runner.current-authority.v1", nextControllerSeq: 1, processOwnerIndexVersion: 1, providerEverStarted: false, externalEffectEverAdmitted: false, pendingSemanticInputIds: [] }; +} + +/** Only current authority remains in the checkpoint. These references are + * independent of the paged history used by transcript/event consumers. */ +function compactIndexedState(state: StoredCoreState): StoredCoreState { + const current = state.indexedState!; + const commands = new Map(); + for (const command of state.commands) { + if (command.status === "pending" || command.status === "indeterminate") commands.set(command.commandId, command); + else if (command.type !== "semantic_tool.result") commands.set(`latest:${command.type}`, command); + else if (["paperclip_finish", "paperclip_block"].includes(String(command.payload.operationId))) commands.set(`terminal:${command.payload.operationId}`, command); + } + const events = new Map(); + for (const event of state.committedEvents) { + const body = isRecord(event.envelope.payload) && isRecord(event.envelope.payload.payload) ? event.envelope.payload.payload : {}; + const semantic = isRecord(body.semantic_tool) ? body.semantic_tool : {}; + if (current.pendingSemanticInputIds.includes(event.sourceEventId)) events.set(event.sourceEventId, event); + else if (["paperclip_finish", "paperclip_block"].includes(String(semantic.operationId))) events.set(`${event.eventType}:${semantic.operationId}`, event); + else if (["session.started", "session.resumed", "session.reconciled", "harness.ready", "turn.accepted", "turn.started", "turn.completed", "turn.failed", "run.terminal"].includes(event.eventType)) events.set(event.eventType, event); + + } + const retained = [...commands.values()].sort((a, b) => compareCurrentCommands(a, b, current.controllerEpoch)); + return { + ...state, commands: retained, + commandDeliveryCounts: Object.fromEntries(retained.map((command) => [command.commandId, state.commandDeliveryCounts[command.commandId] ?? 0])), + committedEvents: [...events.values()].sort((a, b) => compareCurrentEvents(a, b, current.sourceEpoch)), + }; +} + + +export interface LegacyAuthorityImportOptions { + sourcePath: string; + identity: DurableRecoveryIdentity; + authority: DurableAuthorityStore; + /** Stable identity of the exclusive migration, retained across retries. The + * embedding runtime must fence legacy writers and verify every process owner + * before invoking this API; the source file's presence is not that proof. */ + fenceId: string; + assertExclusiveFence(): Promise; + onCheckpoint?(): Promise; +} +interface LegacyAuthorityImportState { + schema: "paperclip.runner.legacy-authority-import.v1"; + identity: DurableRecoveryIdentity; + fenceId: string; + source: string | null; + cursor: LegacyJsonCursor | null; + phase: "copying" | "reconciling" | "prepared"; + commandCount: number; + eventCount: number; + lastSourceSeq: number; + reconcileAfter: string; + projection: StoredCoreState; +} + +/** Prepare a lossless indexed controller import. Nothing publishes a locator + * or starts a peer here: the cross-store activation handshake must separately + * verify runner/provider preparation under this same ownership fence. Staged + * rows cannot be opened by DurablePrpControlPlane as execution authority. */ +export async function stageLegacyControlPlaneAuthority(options: LegacyAuthorityImportOptions): Promise { + if (!stableIdPattern.test(options.fenceId)) throw new Error("Invalid legacy migration fence identity."); + await options.assertExclusiveFence(); + let snapshot = await options.authority.load(); + let generation = snapshot?.generation ?? "0"; + let migration = snapshot?.state as unknown as LegacyAuthorityImportState | undefined; + if (migration && (migration.schema !== "paperclip.runner.legacy-authority-import.v1" || migration.fenceId !== options.fenceId || authorityJson(migration.identity) !== authorityJson(options.identity))) { + throw new Error("Legacy migration staging belongs to a different authority or fence."); + } + migration ??= { + schema: "paperclip.runner.legacy-authority-import.v1", identity: options.identity, fenceId: options.fenceId, + source: null, cursor: null, phase: "copying", commandCount: 0, eventCount: 0, lastSourceSeq: 0, reconcileAfter: "0", + projection: { ...initialCoreState(options.identity), indexedState: indexedCurrentState() }, + }; + const reader = new LegacyJsonReader(options.sourcePath); + const persist = async (records: AuthorityRecord[]) => { + await options.assertExclusiveFence(); + const facts = records.filter(record => record.kind === "event").map(record => processOwnerEvidence(record.body as unknown as DurableRecoveryCommittedEvent, options.identity)).filter(fact => fact !== null); + const work = await processOwnerChanges(options.authority, facts, { runId: options.identity.runId }); + generation = await options.authority.commit({ expectedGeneration: generation, state: migration as unknown as Record, records, work }); + await options.onCheckpoint?.(); + }; + try { + while (migration.phase === "copying") { + await options.assertExclusiveFence(); + const page = await reader.page(migration.source, migration.cursor); + const records: AuthorityRecord[] = []; + for (const entry of page.entries) { + const current = migration.projection; + if (entry.field === "commands") { + const command = entry.value as DurableRecoveryCoreCommand; + const sample = { ...initialCoreState(options.identity), indexedState: indexedCurrentState(), commands: [command] }; + if (!isStoredCoreState(sample, options.identity, true) || command.controllerSeq !== migration.commandCount + 1) throw new Error("Invalid legacy command during migration."); + migration.commandCount++; + current.indexedState!.nextControllerSeq = command.controllerSeq + 1; + if (["session.open", "turn.start", "run.attach"].includes(command.type)) current.indexedState!.providerEverStarted = true; + if (command.type === "semantic_tool.result") current.indexedState!.externalEffectEverAdmitted = true; + current.commands.push(command); + if (command.status !== "pending") records.push({ epoch: options.identity.runId, kind: "command", id: command.commandId, sequence: String(command.controllerSeq), ...(command.controllerEpoch ? { sequenceEpoch: command.controllerEpoch } : {}), body: command as unknown as Record }); + } else if (entry.field === "committedEvents") { + const event = entry.value as DurableRecoveryCommittedEvent; + const sample = { ...initialCoreState(options.identity), indexedState: indexedCurrentState(), committedEvents: [event] }; + if (!isStoredCoreState(sample, options.identity, true) || event.sourceSeq <= migration.lastSourceSeq) throw new Error("Invalid legacy event during migration."); + migration.eventCount++; migration.lastSourceSeq = event.sourceSeq; + current.indexedState!.providerEverStarted = true; + if (["semantic_tool.input", "mcp_app.tool_input", "runtime.input.requested", "runtime_request.created"].includes(event.eventType)) current.indexedState!.externalEffectEverAdmitted = true; + current.committedEvents.push(event); + records.push({ epoch: options.identity.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: { ...event, deliveryCount: 1 } as unknown as Record }); + } else if (entry.field === "commandDeliveryCounts") { + if (entry.key === null || !stableIdPattern.test(entry.key) || !Number.isSafeInteger(entry.value) || Number(entry.value) < 0 || Object.hasOwn(current.commandDeliveryCounts, entry.key)) throw new Error("Invalid legacy command delivery count."); + Object.defineProperty(current.commandDeliveryCounts, entry.key, { value: entry.value, enumerable: true, configurable: true, writable: true }); + } else { + if (entry.key !== null || entry.field === "indexedState") throw new Error("Unexpected indexed metadata in legacy migration."); + Object.defineProperty(current, entry.field, { value: entry.value, enumerable: true, configurable: true, writable: true }); + } + // Each completed entry becomes an immutable row before the staging + // checkpoint advances. No imported history accumulates in this view. + const deliveryCounts = current.commandDeliveryCounts; + migration.projection = compactIndexedState(current); + // Counts are at most the legacy pending/command admission bound. They + // can precede commands in valid JSON, so filter only after the copy. + migration.projection.commandDeliveryCounts = deliveryCounts; + if (Object.keys(deliveryCounts).length > maxCommands) throw new Error("Legacy delivery counts exceed admission capacity."); + } + migration.source = page.source; migration.cursor = page.cursor; + if (page.done) { + // Initialization defaults are staging scaffolding, never evidence for + // a field absent from the source. Empty collections must be explicit. + const required = ["schema", "identity", "tickets", "leases", "commands", "committedEvents", "ackedSourceSeq", "connectionCount", "commandDeliveryCounts", "replayDeliveries", "duplicateCommandResults", "freshBootstraps", "malformedFrames"]; + if (required.some(field => !page.cursor.fields.includes(field))) throw new Error("Legacy authority is missing required source fields."); + if (migration.lastSourceSeq > migration.projection.ackedSourceSeq || migration.commandCount > maxCommands || migration.eventCount > maxCommittedEventWindow || !isStoredCoreState(migration.projection, options.identity, true)) throw new Error("Legacy authority is invalid after import."); + migration.projection = compactIndexedState(migration.projection); + migration.phase = "reconciling"; + } + await persist(records); + } + // JSON field order is irrelevant. Resolve semantic inputs only after all + // completed commands have been imported, using exact indexed lookups. + while (migration.phase === "reconciling") { + await options.assertExclusiveFence(); + const page = await options.authority.readEvents(options.identity.runId, migration.reconcileAfter, 128, 8 * 1024 * 1024); + for (const record of page.records) { + const event = record.body as unknown as DurableRecoveryCommittedEvent; + if (["semantic_tool.input", "mcp_app.tool_input"].includes(event.eventType)) { + const payload = isRecord(event.envelope.payload) && isRecord(event.envelope.payload.payload) ? event.envelope.payload.payload : {}; + const semantic = isRecord(payload.semantic_tool) ? payload.semantic_tool : {}; + const commandId = `command_tool_${createHash("sha256").update(`${options.identity.runId}\0${String(semantic.callId)}`).digest("hex").slice(0, 32)}`; + const command = await options.authority.getRecord(options.identity.runId, "command", commandId); + if (unsettledSemanticInput(event, { identity: options.identity, commands: command ? [command.body as unknown as DurableRecoveryCoreCommand] : [] })) { + migration.projection.indexedState!.pendingSemanticInputIds.push(event.sourceEventId); + migration.projection.committedEvents.push(event); + } + } + } + migration.projection = compactIndexedState(migration.projection); + if (!isStoredCoreState(migration.projection, options.identity, true)) throw new Error("Imported unresolved authority exceeds current admission capacity."); + migration.reconcileAfter = page.nextAfter ?? migration.reconcileAfter; + if (!page.records.length) migration.phase = "prepared"; + await persist([]); + } + if (migration.phase !== "prepared" || !migration.cursor || !migration.source) throw new Error("Invalid legacy migration phase."); + // Re-stat the original through the worker even after restarting a prepared + // import. The original remains untouched until cross-store activation. + await reader.page(migration.source, migration.cursor); + await options.assertExclusiveFence(); + snapshot = { generation, state: migration as unknown as Record }; + return snapshot; + } finally { await reader.close(); } +} + +/** Internal activation projection. It carries an exact preparation receipt so + * a commit-before-filesystem-publication restart can finish the same migration. */ +export function legacyControllerActivation(snapshot: AuthoritySnapshot, identity: DurableRecoveryIdentity, fenceId: string, preparationDigest: string): StoredCoreState { + const migration = snapshot.state as unknown as LegacyAuthorityImportState; + if (migration.schema !== "paperclip.runner.legacy-authority-import.v1" || migration.phase !== "prepared" + || migration.fenceId !== fenceId || authorityJson(migration.identity) !== authorityJson(identity) + || !/^[a-f0-9]{64}$/.test(preparationDigest) || !isStoredCoreState(migration.projection, identity, true)) { + throw new Error("Invalid prepared controller activation."); + } + const projection = structuredClone(migration.projection); + projection.indexedState!.legacyActivation = { fenceId, preparationDigest }; + return projection; +} + +function persistenceFailureCode(error: unknown): string { + if (error instanceof DurableAuthorityStoreError) return error.message; + const cause = isRecord(error) && isRecord(error.cause) ? error.cause : error; + const code = typeof cause === "object" && cause !== null && "code" in cause ? String(cause.code) : "unknown"; + return /^[a-zA-Z0-9_]{1,80}$/.test(code) ? code : "unknown"; +} + +class IndexedCoreStore { + readonly path: string; + readonly authority: DurableAuthorityStore; + #state: StoredCoreState; + #durableState: StoredCoreState; + #generation: string; + #writeIndeterminate = false; + #persistedCommandIds = new Set(); + #persistedEventSeq: number; + #persistedSourceEpoch?: string; + #epoch: string; + #locator: string | null = null; + #failure = "unknown"; + + constructor(directory: string, identity: DurableRecoveryIdentity, authority: DurableAuthorityStore, snapshot: AuthoritySnapshot | null) { + assertNoPendingLegacyMigration(directory); + if (!lstatSync(directory, { throwIfNoEntry: false })) mkdirSync(directory, { recursive: true, mode: 0o700 }); + verifyPrivateDirectory(directory); + this.path = resolve(directory, "control-plane-state.json"); + this.authority = authority; + const activated = this.#validateLocator(); + if (activated && snapshot === null) { + throw new DurableAuthorityStoreError("storage_unavailable", "activated current authority is missing; refusing to recreate it"); + } + const state = snapshot?.state ?? { ...initialCoreState(identity), indexedState: indexedCurrentState() }; + if (!isStoredCoreState(state, identity, true) || !state.indexedState) throw new Error("Indexed current authority is invalid or does not match its identity."); + this.#state = state; + this.#durableState = structuredClone(state); + this.#generation = snapshot?.generation ?? "0"; + this.#persistedEventSeq = state.ackedSourceSeq; + this.#persistedSourceEpoch = state.indexedState?.sourceEpoch; + this.#epoch = identity.runId; + this.#rememberCommands(); + } + + #validateLocator(): boolean { + const existing = readPrivateFile(this.path); + if (existing !== null) { + const previous = JSON.parse(existing) as { schema?: string; location?: { binding?: string } }; + if (previous.schema !== "paperclip.runner.authority-locator.v1" || previous.location?.binding !== this.authority.binding) { + throw new Error("Indexed authority activation requires a fenced legacy migration."); + } + return true; + } + return false; + } + + publishLocator(): void { + const value = `${JSON.stringify({ schema: "paperclip.runner.authority-locator.v1", location: this.authority.location })}\n`; + if (value === this.#locator) return; + this.#validateLocator(); + atomicPrivateWrite(this.path, value); + this.#locator = value; + } + + #rememberCommands(): void { + this.#persistedCommandIds = new Set(this.#state.commands.filter((command) => command.status !== "pending").map((command) => command.commandId)); + } + + get state(): StoredCoreState { return this.#state; } + rememberPersistedState(state: Record): void { this.#durableState = structuredClone(state) as unknown as StoredCoreState; } + assertWritable(): void { + assertNoPendingLegacyMigration(dirname(this.path)); + if (this.#writeIndeterminate) throw new Error(`Durable authority commit is indeterminate (${this.#failure}); reload is required.`); + } + async save(): Promise { await this.commit(this.#state); } + + async backfillProcessOwners(): Promise { + this.assertWritable(); + if (this.#state.indexedState?.processOwnerIndexVersion === 1) return; + const facts = this.#state.committedEvents.map(event => processOwnerEvidence(event, this.#state.identity)).filter(fact => fact !== null); + const work = await processOwnerChanges(this.authority, facts, { runId: this.#state.identity.runId, sourceEpoch: this.#state.indexedState?.sourceEpoch }); + try { + for (let offset = 0; offset < work.length; offset += 128) { + // Retain the original bounded prototype checkpoint until every page + // commits. A crash can repeat the backfill without losing launch facts. + this.#generation = await this.authority.commit({ expectedGeneration: this.#generation, + state: this.#state as unknown as Record, records: [], work: work.slice(offset, offset + 128) }); + } + await this.save(); + } catch (error) { + this.#failure = persistenceFailureCode(error); + this.#writeIndeterminate = true; + throw error; + } + } + + async commit(candidate: StoredCoreState, additionalRecords: AuthorityRecord[] = []): Promise { + this.assertWritable(); + candidate = structuredClone(candidate); + if (candidate.identity.runId !== this.#epoch) { + if (this.#state.indexedState?.normalizedDelivery?.pending.length) { + throw new DurableAuthorityStoreError("storage_pressure", "run-log delivery must settle before authority rotation"); + } + candidate.indexedState = indexedCurrentState(); + } + const records: AuthorityRecord[] = [...additionalRecords]; + const sameEpoch = candidate.identity.runId === this.#epoch; + for (const command of candidate.commands) { + if (command.status !== "pending" && !(sameEpoch && this.#persistedCommandIds.has(command.commandId))) { + records.push({ epoch: candidate.identity.runId, kind: "command", id: command.commandId, sequence: String(command.controllerSeq), ...(command.controllerEpoch ? { sequenceEpoch: command.controllerEpoch } : {}), body: command as unknown as Record }); + if (command.type === "semantic_tool.result" && command.status === "completed") { + candidate.indexedState!.pendingSemanticInputIds = candidate.indexedState!.pendingSemanticInputIds.filter((id) => id !== command.payload.sourceEventId); + } + } + } + for (const event of candidate.committedEvents) { + if (event.sourceEpoch === candidate.indexedState?.sourceEpoch && event.sourceSeq > (sameEpoch && event.sourceEpoch === this.#persistedSourceEpoch ? this.#persistedEventSeq : 0)) records.push({ epoch: candidate.identity.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), ...(event.sourceEpoch ? { sequenceEpoch: event.sourceEpoch } : {}), body: { ...event, deliveryCount: 1 } as unknown as Record }); + } + const facts = candidate.committedEvents.map(event => processOwnerEvidence(event, candidate.identity)).filter(fact => fact !== null); + candidate.indexedState!.processOwnerIndexVersion = 1; + const work = await processOwnerChanges(this.authority, facts, { runId: candidate.identity.runId, sourceEpoch: candidate.indexedState?.sourceEpoch }); + const compact = compactIndexedState(candidate); + const persisted = referenceCurrentAuthority(compact); + try { + this.#generation = await this.authority.commit({ expectedGeneration: this.#generation, state: persisted as unknown as Record, records, work }); + this.#state = compact; + this.#durableState = persisted; + this.#epoch = compact.identity.runId; + this.#persistedEventSeq = compact.ackedSourceSeq; + this.#persistedSourceEpoch = compact.indexedState?.sourceEpoch; + this.#rememberCommands(); + this.publishLocator(); + } catch (error) { + // A resource rejection can be retried through the same live owner only + // after a fresh read proves that its transaction did not advance. Never + // treat an indeterminate commit or a different writer as a safe rollback. + if (error instanceof DurableAuthorityStoreError && error.code === "storage_pressure") { + try { + const committed = await this.authority.load(); + if (committed?.generation === this.#generation && authorityJson(committed.state) === authorityJson(this.#durableState)) { + this.#state = await materializeCurrentAuthority(committed.state, this.authority.getRecord.bind(this.authority)) as unknown as StoredCoreState; + throw error; + } + } catch (readError) { + if (readError === error) throw error; + // Failure to verify keeps the normal indeterminate-write fence. + } + } + this.#failure = persistenceFailureCode(error); + this.#writeIndeterminate = true; + throw error; + } + } +} + /** Reason supplied when a transport-neutral PRP peer closes. */ export interface TransportCloseReason { readonly code?: number; @@ -1256,6 +1690,10 @@ export interface PrpWireConnection { close(code?: number): void; onJson(listener: (value: unknown) => void): void; onClose(listener: (reason: TransportCloseReason) => void): void; + /** Stop delivering new frames while durable admission drains. Peers without + * flow control are disconnected at the bounded ingress budget and replay. */ + pauseRead?(): void; + resumeRead?(): void; } /** Read-only authentication state for an attached PRP peer. */ @@ -1273,6 +1711,7 @@ class RawWebSocketWireConnection implements PrpWireConnection { readonly socket: Duplex; #buffer = Buffer.alloc(0); #closed = false; + #readPaused = false; #onJson: (value: unknown) => void = () => undefined; #onClose: (reason: TransportCloseReason) => void = () => undefined; @@ -1307,6 +1746,14 @@ class RawWebSocketWireConnection implements PrpWireConnection { if (data.length > 0) this.#consume(data); } + pauseRead(): void { this.#readPaused = true; this.socket.pause(); } + resumeRead(): void { + if (this.#closed) return; + this.#readPaused = false; + this.#consume(Buffer.alloc(0)); + if (!this.#readPaused && !this.#closed) this.socket.resume(); + } + sendJson(value: unknown): void { this.sendText(JSON.stringify(value)); } @@ -1316,6 +1763,7 @@ class RawWebSocketWireConnection implements PrpWireConnection { return; } const payload = Buffer.from(text); + if (this.socket.writableLength + payload.length > 4 * 1024 * 1024) { this.close(1013); return; } const header: number[] = [0x81]; if (payload.length <= 125) { header.push(payload.length); @@ -1336,13 +1784,15 @@ class RawWebSocketWireConnection implements PrpWireConnection { return; } this.#closed = true; + this.#buffer = Buffer.alloc(0); this.socket.destroy(); this.#onClose({ message: "local_close" }); } #consume(chunk: Buffer): void { + if (this.#closed) return; this.#buffer = Buffer.concat([this.#buffer, chunk]); - while (this.#buffer.length >= 2) { + while (!this.#closed && !this.#readPaused && this.#buffer.length >= 2) { const first = this.#buffer[0]!; const second = this.#buffer[1]!; const opcode = first & 0x0f; @@ -1407,12 +1857,17 @@ class RawWebSocketWireConnection implements PrpWireConnection { } class AuthorityConnection { + readonly secureFrameLimit: bigint; pendingChallenge: PendingChallenge | null = null; secureChannel: SecureChannel | null = null; lease: ConnectionLeaseRecord | null = null; connectionId: string | null = null; terminalLifecycleCommandId: string | null = null; warmTransitionVersion: 1 | null = null; + commandEpochs = false; + eventEpochs = false; + eventResume: EventResume | null = null; + eventEpochLimit = EVENT_EPOCH_LIMIT; identity: DurableRecoveryIdentity | null = null; replayOnly = false; activationReceipt: DurableWarmRunTransition | null = null; @@ -1424,8 +1879,10 @@ class AuthorityConnection { wire: PrpWireConnection; onJson: (value: unknown) => void; onClose: () => void; + secureFrameLimit: number; }) { this.wire = input.wire; + this.secureFrameLimit = BigInt(input.secureFrameLimit); this.#onClose = input.onClose; this.wire.onJson(input.onJson); this.wire.onClose(() => this.#markClosed()); @@ -1433,6 +1890,12 @@ class AuthorityConnection { sendJson(value: unknown): void { if (this.#closed) return; + if (this.secureChannel && this.secureChannel.sendCounter >= this.secureFrameLimit) { + // The underlying authority already owns commands/events/ACK cursors. + // A lost reply is reconciled on the next authenticated connection. + this.close(1012); + return; + } this.wire.sendJson( this.secureChannel === null ? value @@ -1456,8 +1919,20 @@ class AuthorityConnection { /** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */ export class DurablePrpControlPlane { + #mutationTail: Promise = Promise.resolve(); + #queuedMutations = 0; + #queuedWireFrames = 0; + #queuedWireBytes = 0; + + #serialize(operation: () => Promise): Promise { + if (this.#queuedMutations >= 64) return Promise.reject(new Error("storage_pressure: authority mutation queue is full")); + this.#queuedMutations++; + const result = this.#mutationTail.then(operation); + this.#mutationTail = result.catch(() => undefined).finally(() => { this.#queuedMutations--; }); + return result; + } #identity: DurableRecoveryIdentity; - readonly #store: DurableCoreStore; + readonly #store: DurableCoreStore | IndexedCoreStore; #expectedRunnerVersion: string; #expectedRunnerDigest: string; #server: Server | null = null; @@ -1472,8 +1947,13 @@ export class DurablePrpControlPlane { #onProtocolIntegrityError?: DurablePrpControlPlaneOptions["onProtocolIntegrityError"]; #protocolIntegrityError: NativeSessionProtocolIntegrityError | null = null; #connectionLeaseTtlMs: number; + #secureChannelFrameLimit: number; + #eventEpochLimit: number; + #normalizedEventEpochLimit: number; + #commandEpochLimit: number; + #commandEpochWaiters = new Set<(error?: Error) => void>(); - constructor(options: DurablePrpControlPlaneOptions) { + constructor(options: DurablePrpControlPlaneOptions, openedSnapshot?: AuthoritySnapshot | null) { if ( !Object.values(options.identity).every( (value) => typeof value === "string" && stableIdPattern.test(value), @@ -1483,15 +1963,18 @@ export class DurablePrpControlPlane { (options.connectionLeaseTtlMs !== undefined && (!Number.isInteger(options.connectionLeaseTtlMs) || options.connectionLeaseTtlMs < 60_000 || - options.connectionLeaseTtlMs > 24 * 60 * 60 * 1_000)) + options.connectionLeaseTtlMs > 24 * 60 * 60 * 1_000)) || + (options.secureChannelFrameLimit !== undefined && + (!Number.isInteger(options.secureChannelFrameLimit) || options.secureChannelFrameLimit < 4 || options.secureChannelFrameLimit > maxSecureChannelFrames)) || + (options.commandEpochLimit !== undefined && (!Number.isSafeInteger(options.commandEpochLimit) || options.commandEpochLimit < 4 || options.commandEpochLimit > COMMAND_EPOCH_LIMIT)) ) { throw new Error("Durable PRP control plane options are invalid."); } this.#identity = structuredClone(options.identity); - this.#store = new DurableCoreStore( - options.stateDirectory, - options.identity, - ); + if (options.authorityStore && openedSnapshot === undefined) throw new Error("Indexed authority must be opened asynchronously."); + this.#store = options.authorityStore + ? new IndexedCoreStore(options.stateDirectory, options.identity, options.authorityStore, openedSnapshot!) + : new DurableCoreStore(options.stateDirectory, options.identity); this.#expectedRunnerVersion = options.expectedRunnerVersion; this.#expectedRunnerDigest = options.expectedRunnerDigest; const transition = this.#store.state.warmTransition; @@ -1509,13 +1992,145 @@ export class DurablePrpControlPlane { this.#beforeAuthenticatedConnection = options.beforeAuthenticatedConnection; this.#onProtocolIntegrityError = options.onProtocolIntegrityError; this.#connectionLeaseTtlMs = options.connectionLeaseTtlMs ?? 60_000; + this.#secureChannelFrameLimit = options.secureChannelFrameLimit ?? maxSecureChannelFrames; + if (options.eventEpochLimit !== undefined && (!Number.isSafeInteger(options.eventEpochLimit) || options.eventEpochLimit < 4 || options.eventEpochLimit > EVENT_EPOCH_LIMIT)) throw new Error("Invalid event epoch limit"); + this.#eventEpochLimit = options.eventEpochLimit ?? EVENT_EPOCH_LIMIT; + if (options.normalizedEventEpochLimit !== undefined && (!Number.isSafeInteger(options.normalizedEventEpochLimit) || options.normalizedEventEpochLimit < 4 || options.normalizedEventEpochLimit > EVENT_EPOCH_LIMIT)) throw new Error("Invalid normalized event epoch limit"); + this.#normalizedEventEpochLimit = options.normalizedEventEpochLimit ?? EVENT_EPOCH_LIMIT; + this.#commandEpochLimit = options.commandEpochLimit ?? COMMAND_EPOCH_LIMIT; } get store(): DurablePrpControlPlaneStore { return this.#store; } - getCommand(commandId: string): DurableRecoveryCoreCommand | undefined { + static async open(options: DurablePrpControlPlaneOptions): Promise { + const snapshot = options.authorityStore ? await options.authorityStore.load() : undefined; + let opened = snapshot; + if (snapshot && options.authorityStore) { + opened = { ...snapshot, state: await materializeCurrentAuthority(snapshot.state, options.authorityStore.getRecord.bind(options.authorityStore)) }; + const current = await options.authorityStore.load(); + if (current?.generation !== snapshot.generation || authorityJson(current.state) !== authorityJson(snapshot.state)) throw new DurableAuthorityStoreError("stale_authority", "authority changed while resolving current evidence"); + } + const core = new DurablePrpControlPlane(options, opened); + if (snapshot && core.#store instanceof IndexedCoreStore) core.#store.rememberPersistedState(snapshot.state); + if (options.authorityStore && snapshot === null) await core.#store.save(); + else if (core.#store instanceof IndexedCoreStore) { + await core.#store.backfillProcessOwners(); + core.#store.publishLocator(); + } + return core; + } + + get indexedPersistence(): boolean { return this.#store instanceof IndexedCoreStore; } + + normalizedDelivery(): NormalizedDeliveryPort | null { + if (!(this.#store instanceof IndexedCoreStore)) return null; + const storage = this.#store; + const epoch = this.#identity.runId; + const check = () => { + this.#store.assertWritable(); + if (this.#identity.runId !== epoch) throw new DurableAuthorityStoreError("stale_authority", "normalized consumer belongs to a retired run"); + }; + const receiptId = (collection: string, key: string) => `driver_${createHash("sha256").update(`${collection}\0${key}`).digest("hex")}`; + return { + epoch, + ...(storage.authority.unorderedEffectReceipts === true ? { eventEpochs: { limit: this.#normalizedEventEpochLimit } } : {}), + history: { get: async (collection, key) => { + check(); + const receipt = await storage.authority.getSessionEffect(receiptId(collection, key)); + check(); + if (!receipt) return null; + if (receipt.body.schema !== "paperclip.driver-receipt.v1" || receipt.body.collection !== collection || receipt.body.key !== key || receipt.body.value == null) throw new DurableAuthorityStoreError("invalid_authority", "driver receipt binding mismatch"); + return structuredClone(receipt.body.value); + } }, + load: () => { check(); return structuredClone(this.#store.state.indexedState?.normalizedDelivery ?? null); }, + commit: (batch) => this.#serialize(async () => { + check(); + const currentSource = this.#store.state.indexedState?.sourceEpoch; + const closed = batch.raw.sourceEpoch === currentSource ? null : await this.readEventEpochTransition(batch.raw.sourceEpoch); + if (batch.raw.epoch !== epoch || batch.raw.sourceSeq > (closed?.finalOrdinal ?? this.#store.state.ackedSourceSeq) || (batch.raw.sourceEpoch !== currentSource && !closed)) throw new DurableAuthorityStoreError("invalid_authority", "normalized consumer passed the committed raw inbox or its epoch"); + for (const event of batch.events) { + if (event.runId !== epoch || event.normalizedSessionId !== this.#identity.normalizedSessionId || event.sourceInstanceId !== this.#identity.runnerInstanceId || event.sourceKind !== "runner") { + throw new DurableAuthorityStoreError("invalid_authority", "normalized event binding mismatch"); + } + } + const candidate = structuredClone(this.#store.state); + const previousDelivery = candidate.indexedState!.normalizedDelivery ?? null; + const rawTransition = previousDelivery && previousDelivery.raw.sourceEpoch !== batch.raw.sourceEpoch ? await this.readEventEpochTransition(previousDelivery.raw.sourceEpoch) : null; + const next = applyNormalizedBatch(previousDelivery, batch, rawTransition ?? undefined); + candidate.indexedState!.normalizedDelivery = next; + const receipts: AuthorityRecord[] = []; + if ((batch.receipts?.length ?? 0) > 128) throw new DurableAuthorityStoreError("storage_pressure", "driver receipt batch exceeds admission capacity"); + const unordered = storage.authority.unorderedEffectReceipts === true; + const closedEpochs = new Set(); + for (const event of batch.events) { + const t = event.sourceEpochTransition; + if (!t) continue; + if (!unordered || closedEpochs.has(t.fromEpoch) || closedEpochs.has(t.nextEpoch)) throw new DurableAuthorityStoreError("invalid_authority", "normalized epoch reused or unsupported"); + const closeId = normalizedEpochCloseId(epoch, event.sourceInstanceId, t.fromEpoch); + if (await storage.authority.getRecord(epoch, "effect", closeId) + || await storage.authority.getRecord(epoch, "effect", normalizedEpochCloseId(epoch, event.sourceInstanceId, t.nextEpoch)) + || await storage.authority.getRecord(epoch, "effect", `normalized-epoch-${t.transitionId}`)) throw new DurableAuthorityStoreError("receipt_conflict", "normalized epoch identity reused"); + closedEpochs.add(t.fromEpoch); + for (const id of [closeId, `normalized-epoch-${t.transitionId}`]) receipts.push({ epoch, kind: "effect", id, sequence: "0", body: { ...t } }); + } + let sequence = unordered ? 0n : authorityInteger(candidate.indexedState!.driverReceiptSequence ?? "0"); + for (const receipt of batch.receipts ?? []) { + if (!["terminal", "file", "steering", "lineage"].includes(receipt.collection) || typeof receipt.key !== "string" || !receipt.key || Buffer.byteLength(receipt.key) > 4096 || receipt.value == null) throw new DurableAuthorityStoreError("invalid_authority", "invalid driver receipt"); + const id = receiptId(receipt.collection, receipt.key); + const body = { schema: "paperclip.driver-receipt.v1", ...receipt }; + const existing = await storage.authority.getSessionEffect(id); + if (existing) { + if (authorityJson(existing.body) !== authorityJson(body)) throw new DurableAuthorityStoreError("receipt_conflict", "driver identity changed its accepted outcome"); + } else receipts.push({ epoch, kind: "effect", id, sequence: unordered ? "0" : String(++sequence), body }); + } + if (unordered) delete candidate.indexedState!.driverReceiptSequence; + else candidate.indexedState!.driverReceiptSequence = String(sequence); + await storage.commit(candidate, receipts); + return structuredClone(next); + }), + acknowledge: (event) => this.#serialize(async () => { + check(); + const previous = this.#store.state.indexedState?.normalizedDelivery; + if (!previous) throw new DurableAuthorityStoreError("invalid_authority", "normalized consumer has no pending delivery"); + const candidate = structuredClone(this.#store.state); + candidate.indexedState!.normalizedDelivery = acknowledgeNormalizedEvent(previous, event); + await this.#store.commit(candidate); + }), + }; + } + + /** Cleanup pages current unresolved owners under one authority generation. + * Callers must revalidate ownership before taking any process action. */ + async readProcessOwners(after = "", limit = 128, expectedGeneration?: string) { + if (!(this.#store instanceof IndexedCoreStore)) throw new DurableAuthorityStoreError("invalid_authority", "legacy authority has no process-owner index"); + this.#store.assertWritable(); + const snapshot = await this.#store.authority.load(); + if (!snapshot) throw new DurableAuthorityStoreError("storage_unavailable", "process-owner authority is missing"); + const generation = expectedGeneration ?? snapshot.generation; + const page = await this.#store.authority.readWorkPage("process-owner", after, limit, generation); + return { ...page, generation }; + } + + async readCommittedEvents(after: number, limit = 128, sourceEpoch?: string): Promise { + this.#store.assertWritable(); + if (!(this.#store instanceof IndexedCoreStore)) return this.#store.state.committedEvents.filter((event) => event.sourceSeq > after).slice(0, limit); + const page = await this.#store.authority.readEvents(this.#identity.runId, String(after), limit, 4 * 1024 * 1024, sourceEpoch); + return page.records.map((record) => record.body as unknown as DurableRecoveryCommittedEvent); + } + + async readEventEpochTransition(sourceEpoch?: string): Promise { + if (!(this.#store instanceof IndexedCoreStore)) return null; + this.#store.assertWritable(); + const record = await this.#store.authority.getRecord(this.#identity.runId, "effect", eventEpochCloseId({runId: this.#identity.runId, fromEpoch: sourceEpoch ?? null})); + if (!record) return null; + if (!isEventEpochTransition(record.body) || record.body.runId !== this.#identity.runId || record.body.fromEpoch !== (sourceEpoch ?? null)) throw new DurableAuthorityStoreError("invalid_authority", "event epoch close receipt binding mismatch"); + return record.body; + } + + async getCommand(commandId: string): Promise { + this.#store.assertWritable(); return ( this.#store.state.commands.find( (command) => command.commandId === commandId, @@ -1526,6 +2141,9 @@ export class DurablePrpControlPlane { (this.#store.state.completedWarmTransition?.command.commandId === commandId ? this.#store.state.completedWarmTransition.command + : undefined) ?? + (this.#store instanceof IndexedCoreStore + ? (await this.#store.authority.getRecord(this.#identity.runId, "command", commandId))?.body as unknown as DurableRecoveryCoreCommand | undefined : undefined) ); } @@ -1563,6 +2181,7 @@ export class DurablePrpControlPlane { } async stop(): Promise { + for (const done of this.#commandEpochWaiters) done(new DurableAuthorityStoreError("storage_unavailable", "controller stopped with command epoch transition pending")); for (const connection of this.#connections) { connection.close(); } @@ -1628,10 +2247,11 @@ export class DurablePrpControlPlane { * retaining its existing connection lease secret. The runner performs the * matching state transition only after acknowledging `run.attach`. */ - rotateRunIdentity( + async rotateRunIdentity( identity: DurableRecoveryIdentity, runAttachTemplate?: Record, - ): void { + ): Promise { + return this.#serialize(async () => { if (this.#protocolIntegrityError !== null) throw this.#protocolIntegrityError; const completed = this.#store.state.completedWarmTransition; @@ -1678,7 +2298,7 @@ export class DurablePrpControlPlane { } const candidate = structuredClone(this.#store.state); candidate.runAttachTemplate = structuredClone(runAttachTemplate); - this.#store.commit(candidate); + await this.#store.commit(candidate); } return; } @@ -1710,15 +2330,16 @@ export class DurablePrpControlPlane { { ...lease, identity: structuredClone(identity) }, ]), ); - Object.assign(this.#store.state, initialCoreState(identity), { + const candidate = Object.assign(initialCoreState(identity), { leases, runAttachTemplate: runAttachTemplate === undefined ? null : structuredClone(runAttachTemplate), }); + await this.#store.commit(candidate); this.#identity = structuredClone(identity); - this.#store.save(); + }); } /** @@ -1728,7 +2349,8 @@ export class DurablePrpControlPlane { * source for a later run.attach. Repeating the same write is idempotent; * changing an established seed fails closed. */ - persistRunAttachTemplate(runAttachTemplate: Record): void { + async persistRunAttachTemplate(runAttachTemplate: Record): Promise { + return this.#serialize(async () => { if (!isRecord(runAttachTemplate.provider)) { throw new Error("Durable PRP run attachment template is invalid."); } @@ -1741,11 +2363,14 @@ export class DurablePrpControlPlane { throw new Error("Durable PRP run attachment template conflicts."); } if (existing !== undefined && existing !== null) return; - this.#store.state.runAttachTemplate = structuredClone(runAttachTemplate); - this.#store.save(); + const candidate = structuredClone(this.#store.state); + candidate.runAttachTemplate = structuredClone(runAttachTemplate); + await this.#store.commit(candidate); + }); } - issueBootstrapTicket(ttlMs = 5_000): string { + async issueBootstrapTicket(ttlMs = 5_000): Promise { + return this.#serialize(async () => { this.#store.assertWritable(); if (this.#store.state.warmTransition) { throw new Error( @@ -1755,11 +2380,12 @@ export class DurablePrpControlPlane { if (!Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 60_000) { throw new Error("Durable PRP bootstrap TTL is invalid."); } - this.#pruneCredentials(); + const candidate = structuredClone(this.#store.state); + this.#pruneCredentials(candidate); const ticket = `bootstrap_${randomUUID()}`; const material = credentialMaterial(ticket); const expiresAtUnixMs = Date.now() + ttlMs; - this.#store.state.tickets[material.credentialId] = { + candidate.tickets[material.credentialId] = { recordId: `bootstrap_ticket_${randomUUID()}`, credentialId: material.credentialId, authKeyDigest: `sha256:${material.authKey.toString("hex")}`, @@ -1770,19 +2396,21 @@ export class DurablePrpControlPlane { expiresAtUnixMs, usedAt: null, }; - this.#store.state.freshBootstraps += 1; - this.#store.save(); + candidate.freshBootstraps = incrementDiagnosticCount(candidate.freshBootstraps); + await this.#store.commit(candidate); return ticket; + }); } /** Caller-owned recovery admission is required; a receipt is not a credential. */ - issueWarmTransitionBootstrapTicket( + async issueWarmTransitionBootstrapTicket( input: { transitionId: string; runnerState: Record; }, ttlMs = 5_000, - ): string { + ): Promise { + return this.#serialize(async () => { this.#store.assertWritable(); const pending = input.runnerState.warmTransition; const proof = warmTransitionRecoveryProof({ @@ -1827,17 +2455,23 @@ export class DurablePrpControlPlane { usedAt: null, warmTransitionId: transition.receipt.transitionId, }; - candidate.freshBootstraps += 1; - this.#store.commit(candidate); + candidate.freshBootstraps = incrementDiagnosticCount(candidate.freshBootstraps); + await this.#store.commit(candidate); return ticket; + }); } - queueCommand( + async queueCommand( type: string, payload: Record = {}, commandId?: string, deliverImmediately = false, - ): DurableRecoveryCoreCommand { + ): Promise { + // Internal adapters can use optional object properties. Persist the exact + // JSON sent on the wire, just as the legacy JSON writer did. + payload = JSON.parse(JSON.stringify(payload)) as Record; + for (;;) { + const queued = await this.#serialize(async () => { this.#store.assertWritable(); const transition = this.#store.state.warmTransition; if (transition && transition.phase !== "activated") { @@ -1873,9 +2507,7 @@ export class DurablePrpControlPlane { throw new Error("Durable PRP command is invalid."); } if (commandId !== undefined) { - const existing = this.#store.state.commands.find( - (candidate) => candidate.commandId === commandId, - ); + const existing = await this.getCommand(commandId); if (existing !== undefined) { if ( existing.type !== type || @@ -1888,20 +2520,47 @@ export class DurablePrpControlPlane { if (deliverImmediately && existing.status === "pending") { for (const connection of this.#connections) { if (connection.secureChannel !== null) - this.#sendNextCommand(connection); + await this.#sendNextCommand(connection); } } return existing; } } - const controllerSeq = this.#store.state.commands.length + 1; + const current = this.#store.state.indexedState; + if (current && (current.commandEpochTransition || current.nextControllerSeq > this.#commandEpochLimit)) { + if (!(this.#store instanceof IndexedCoreStore) || !this.#store.authority.commandEpochs || !this.#store.authority.unorderedEffectReceipts) + throw new DurableAuthorityStoreError("storage_pressure", "command epoch rotation requires a capable indexed authority store"); + if (!current.commandEpochTransition) { + if (!current.commandEpochsNegotiated && ![...this.#connections].some(c => c.secureChannel && c.commandEpochs && !c.replayOnly)) + throw new DurableAuthorityStoreError("storage_pressure", "command epoch rotation requires an authenticated capable runner"); + let nextEpoch: string | undefined; + for (let attempt = 0; attempt < 32; attempt++) { + const proposed = randomUUID(); + if (proposed !== current.controllerEpoch && !await this.#store.authority.getRecord(this.#identity.runId, "effect", + commandEpochCloseId({ runId: this.#identity.runId, fromEpoch: proposed }))) { nextEpoch = proposed; break; } + } + if (!nextEpoch) throw new DurableAuthorityStoreError("storage_unavailable", "could not allocate a fresh command namespace"); + const candidate = structuredClone(this.#store.state); + candidate.indexedState!.commandEpochTransition = { + schema: "paperclip.prp.command-epoch.v1", runId: this.#identity.runId, + transitionId: randomUUID(), fromEpoch: current.controllerEpoch ?? null, + nextEpoch, finalOrdinal: current.nextControllerSeq - 1, + }; + await this.#store.commit(candidate); + } + for (const connection of this.#connections) if (connection.secureChannel) await this.#sendNextCommand(connection); + return null; + } + const controllerSeq = current?.nextControllerSeq ?? this.#store.state.commands.length + 1; + if (!Number.isSafeInteger(controllerSeq + 1)) throw new Error("PRP sequence requires an exact-integer protocol epoch transition."); const command: DurableRecoveryCoreCommand = { schema: type.startsWith("session.goal.") ? "paperclip.prp.command.v2" : "paperclip.prp.command.v1", commandId: - commandId ?? `command_prp_${controllerSeq.toString().padStart(8, "0")}`, + commandId ?? (current?.controllerEpoch ? `command_prp_${randomUUID()}` : `command_prp_${controllerSeq.toString().padStart(8, "0")}`), controllerSeq, + ...(current?.controllerEpoch ? { controllerEpoch: current.controllerEpoch } : {}), type, issuedAt: new Date().toISOString(), payload, @@ -1909,30 +2568,77 @@ export class DurablePrpControlPlane { result: null, }; if ( - this.#store.state.commands.length >= maxCommands || + (this.indexedPersistence + ? this.#store.state.commands.filter((entry) => entry.status === "pending" || entry.status === "indeterminate").length + : this.#store.state.commands.length) >= maxCommands || Buffer.byteLength(JSON.stringify(command)) > maxCommandBytes ) { throw new Error("Durable PRP command journal bound exceeded."); } - this.#store.state.commands.push(command); - this.#store.save(); + const candidate = structuredClone(this.#store.state); + candidate.commands.push(command); + if (candidate.indexedState) { + candidate.indexedState.nextControllerSeq = controllerSeq + 1; + if (["session.open", "turn.start", "run.attach"].includes(type)) candidate.indexedState.providerEverStarted = true; + } + await this.#store.commit(candidate); if (deliverImmediately) { for (const connection of this.#connections) { if (connection.secureChannel !== null) { - this.#sendNextCommand(connection); + await this.#sendNextCommand(connection); } } } return command; + }); + if (queued) return queued; + await this.#waitForCommandEpoch(); + } } - commandOutcome(commandId: string): { + #waitForCommandEpoch(): Promise { + if (!this.#store.state.indexedState?.commandEpochTransition) return Promise.resolve(); + if (this.#commandEpochWaiters.size >= 64) return Promise.reject(new DurableAuthorityStoreError("storage_pressure", "command epoch admission queue is full")); + return new Promise((resolve, reject) => { + const done = (error?: Error) => { clearTimeout(timer); this.#commandEpochWaiters.delete(done); if (error) reject(error); else resolve(); }; + const timer = setTimeout(() => { + this.#commandEpochWaiters.delete(done); + reject(new DurableAuthorityStoreError("storage_pressure", "command epoch transition remains pending; retry admission")); + }, 30_000); + timer.unref(); + this.#commandEpochWaiters.add(done); + }); + } + + async #commandEpochCommitted(connection: AuthorityConnection, envelope: Record): Promise { + if (!connection.commandEpochs || !this.#liveConnectionIsCurrent(connection) || !(this.#store instanceof IndexedCoreStore)) { connection.close(); return; } + const pending = this.#store.state.indexedState?.commandEpochTransition; + const receipt = envelope.payload; + if (!isCommandEpochTransition(receipt) || receipt.runId !== this.#identity.runId) { connection.close(); return; } + if (!pending) { + const committed = await this.#store.authority.getRecord(this.#identity.runId, "effect", `command-epoch-${receipt.transitionId}`); + if (!committed || authorityJson(committed.body) !== authorityJson(receipt)) connection.close(); + return; + } + if (authorityJson(pending) !== authorityJson(receipt) || this.#store.state.commands.some(c => c.status === "pending")) { connection.close(); return; } + const candidate = structuredClone(this.#store.state); + candidate.indexedState!.controllerEpoch = pending.nextEpoch; + candidate.indexedState!.lastCommandEpochTransition = structuredClone(pending); + candidate.indexedState!.nextControllerSeq = 1; + delete candidate.indexedState!.commandEpochTransition; + const body = receipt as unknown as Record; + await this.#store.commit(candidate, [ + { epoch: this.#identity.runId, kind: "effect", id: `command-epoch-${pending.transitionId}`, sequence: "0", body }, + { epoch: this.#identity.runId, kind: "effect", id: commandEpochCloseId(pending), sequence: "0", body }, + ]); + for (const done of this.#commandEpochWaiters) done(); + } + + async commandOutcome(commandId: string): Promise<{ status: DurableRecoveryCoreCommand["status"]; result: Record | null; - } | null { - const command = this.#store.state.commands.find( - (candidate) => candidate.commandId === commandId, - ); + } | null> { + const command = await this.getCommand(commandId); if (!command) return null; return { status: command.status, @@ -1995,12 +2701,35 @@ export class DurablePrpControlPlane { attachWireConnection(wire: PrpWireConnection): PrpWireAttachment { let connection!: AuthorityConnection; let processing = Promise.resolve(); + let closed = false, paused = false, frames = 0, bytes = 0; connection = new AuthorityConnection({ wire, + secureFrameLimit: this.#secureChannelFrameLimit, onJson: (value) => { + if (closed) return; + let size: number; + try { size = Buffer.byteLength(JSON.stringify(value)); } + catch { wire.close(1007); return; } + // Bound admission before adding a promise closure. The mutation queue + // alone cannot bound frames waiting to enter that queue. These limits + // concern currently buffered work; settled history returns all credit. + if (size > 4 * 1024 * 1024 || frames >= 64 || bytes + size > 16 * 1024 * 1024 || + this.#queuedWireFrames >= 256 || this.#queuedWireBytes + size > 64 * 1024 * 1024) { + closed = true; wire.close(1013); return; + } + frames++; bytes += size; this.#queuedWireFrames++; this.#queuedWireBytes += size; + if (!paused && wire.pauseRead && wire.resumeRead && (frames >= 16 || bytes >= 4 * 1024 * 1024)) { + paused = true; wire.pauseRead(); + } processing = processing .then(() => this.#handleJson(connection, value)) - .catch(() => connection.close()); + .catch(() => connection.close()) + .finally(() => { + frames--; bytes -= size; this.#queuedWireFrames--; this.#queuedWireBytes -= size; + if (!closed && paused && frames <= 8 && bytes <= 2 * 1024 * 1024) { + paused = false; wire.resumeRead!(); + } + }); const tail = processing; this.#connectionProcessing.set(connection, tail); const release = () => { @@ -2009,7 +2738,7 @@ export class DurablePrpControlPlane { }; void tail.then(release, release); }, - onClose: () => this.#connections.delete(connection), + onClose: () => { closed = true; this.#connections.delete(connection); }, }); this.#connections.add(connection); return { @@ -2022,6 +2751,10 @@ export class DurablePrpControlPlane { wire: unknown, ): Promise { this.#store.assertWritable(); + if (connection.secureChannel && connection.secureChannel.receiveCounter >= connection.secureFrameLimit) { + connection.close(1012); + return; + } let envelope: Record; try { envelope = @@ -2029,8 +2762,11 @@ export class DurablePrpControlPlane { ? (wire as Record) : decryptSecureJson(connection.secureChannel, wire); } catch { - this.#store.state.malformedFrames += 1; - this.#store.save(); + await this.#serialize(async () => { + const candidate = structuredClone(this.#store.state); + candidate.malformedFrames = incrementDiagnosticCount(candidate.malformedFrames); + await this.#store.commit(candidate); + }); connection.close(); return; } @@ -2074,7 +2810,7 @@ export class DurablePrpControlPlane { return; } if (kind === "lease_renew") { - this.#renewLease(connection, envelope); + await this.#serialize(() => this.#renewLease(connection, envelope)); return; } if (kind === "event") { @@ -2087,16 +2823,26 @@ export class DurablePrpControlPlane { await this.#event(connection, envelope); return; } + if (kind === "event_epoch_rotate") { + await this.#serialize(() => this.#eventEpochRotate(connection, envelope)); + return; + } + if (kind === "command_epoch_committed") { + await this.#serialize(() => this.#commandEpochCommitted(connection, envelope)); + return; + } if (kind === "command_result") { if ( this.#store.state.warmTransition && this.#store.state.warmTransition.phase !== "activated" ) connection.replayOnly = true; - this.#commandResult(connection, envelope); + await this.#serialize(() => this.#commandResult(connection, envelope)); return; } if (kind === "warm_transition_activated") { + return this.#serialize(async () => { + if (!connection.lease || !this.#liveConnectionIsCurrent(connection)) return; const transition = this.#store.state.warmTransition; const receipt = connection.activationReceipt; const completed = this.#store.state.completedWarmTransition; @@ -2131,7 +2877,7 @@ export class DurablePrpControlPlane { command: structuredClone(transition.command), }; delete candidate.warmTransition; - this.#store.commit(candidate); + await this.#store.commit(candidate); connection.lease = this.#store.state.leases[transition.credentialId]!; } connection.sendJson( @@ -2144,18 +2890,28 @@ export class DurablePrpControlPlane { ); connection.activationReceipt = null; connection.replayOnly = false; - this.#sendNextCommand(connection); - return; + await this.#sendNextCommand(connection); + }); } if (kind !== "pong") { connection.close(); } } - #renewLease( + #liveConnectionIsCurrent(connection: AuthorityConnection): boolean { + if (!connection.secureChannel || !connection.lease || connection.lease.revokedAt !== null || + connection.lease.expiresAtUnixMs <= Date.now() || canonicalJson(this.#store.state.leases[connection.lease.credentialId]) !== canonicalJson(connection.lease)) { + connection.close(); + return false; + } + return true; + } + + async #renewLease( connection: AuthorityConnection, envelope: Record, - ): void { + ): Promise { + if (!this.#liveConnectionIsCurrent(connection)) return; const lease = connection.lease!; const payload = envelope.payload as Record | undefined; const expectedExpiry = payload?.connectionLeaseExpiresAtUnixMs; @@ -2191,7 +2947,7 @@ export class DurablePrpControlPlane { ); renewed.expiresAt = new Date(renewed.expiresAtUnixMs).toISOString(); candidate.lastLeaseExpiresAt = renewed.expiresAt; - this.#store.commit(candidate); + await this.#store.commit(candidate); connection.lease = this.#store.state.leases[lease.credentialId]!; } connection.sendJson( @@ -2208,10 +2964,56 @@ export class DurablePrpControlPlane { ); } + async #eventEpochRotate(connection: AuthorityConnection, envelope: Record): Promise { + if (!connection.eventEpochs || !this.#liveConnectionIsCurrent(connection) || !(this.#store instanceof IndexedCoreStore)) { connection.close(); return; } + const transition = envelope.payload; + if (!isEventEpochTransition(transition) || transition.runId !== this.#identity.runId) { connection.close(); return; } + const existing = await this.#store.authority.getRecord(this.#identity.runId, "effect", `event-epoch-${transition.transitionId}`); + if (existing) { + if (authorityJson(existing.body) !== authorityJson(transition)) { connection.close(); return; } + } else { + const current = this.#store.state; + if (transition.fromEpoch !== (current.indexedState?.sourceEpoch ?? null) || transition.finalOrdinal !== current.ackedSourceSeq || current.warmTransition || + await this.readEventEpochTransition(transition.nextEpoch)) { connection.close(); return; } + const candidate = structuredClone(current); + candidate.indexedState!.sourceEpoch = transition.nextEpoch; + candidate.indexedState!.lastEventEpochTransition = transition; + candidate.ackedSourceSeq = 0; + await this.#store.commit(candidate, [ + {epoch: this.#identity.runId, kind: "effect", id: `event-epoch-${transition.transitionId}`, sequence: "0", body: transition as unknown as Record}, + {epoch: this.#identity.runId, kind: "effect", id: eventEpochCloseId(transition), sequence: "0", body: transition as unknown as Record}, + ]); + } + connection.sendJson(this.#controlEnvelope(connection, `event_epoch_${transition.transitionId}`, "event_epoch_committed", transition as unknown as Record)); + } + #authorizeHello( payload: Record, ): PendingAuthorization | null { - this.#pruneCredentials(); + if (this.indexedPersistence && (payload.durability !== INDEXED_DURABILITY_CAPABILITY || payload.outputBodies !== "history.output_bodies.v1")) return null; + const current = this.#store.state.indexedState; + if (payload.runId === this.#store.state.identity.runId && current) { + if (current.sourceEpoch && payload.eventEpochs !== EVENT_EPOCH_CAPABILITY) return null; + if (payload.eventEpochs === EVENT_EPOCH_CAPABILITY && this.#store instanceof IndexedCoreStore && this.#store.authority.eventEpochs) { + const resume = payload.eventResume; + if (!isEventResume(resume)) return null; + if (resume.sourceEpoch === (current.sourceEpoch ?? null)) { + if (resume.ackedSourceSeq > this.#store.state.ackedSourceSeq || resume.nextSourceEventSeq <= this.#store.state.ackedSourceSeq) return null; + } else if (!resume.transition || authorityJson(resume.transition) !== authorityJson(current.lastEventEpochTransition ?? null) || this.#store.state.ackedSourceSeq !== 0) return null; + } + + if ((current.commandEpochsNegotiated || current.controllerEpoch || current.commandEpochTransition) && payload.commandEpochs !== COMMAND_EPOCH_CAPABILITY) return null; + if (payload.commandEpochs === COMMAND_EPOCH_CAPABILITY && this.#store instanceof IndexedCoreStore && this.#store.authority.commandEpochs) { + if (!isRecord(payload.resume)) return null; + const epoch = payload.resume.controllerEpoch ?? null, ordinal = payload.resume.lastControllerCommandSeq; + if (!Number.isSafeInteger(ordinal) || Number(ordinal) < 0) return null; + const transition = current.commandEpochTransition; + if (transition && epoch === transition.nextEpoch) { + if (ordinal !== 0) return null; // successor commands are not admitted before its receipt + } else if (epoch !== (current.controllerEpoch ?? null) || Number(ordinal) > current.nextControllerSeq - 1 || + Number(ordinal) < current.nextControllerSeq - 1 - this.#store.state.commands.filter(c => c.status === "pending").length) return null; + } + } const credentialId = payload.credentialId; if (typeof credentialId !== "string") return null; const ticket = this.#store.state.tickets[credentialId]; @@ -2344,23 +3146,23 @@ export class DurablePrpControlPlane { return authorization; } - #pruneCredentials(): void { + #pruneCredentials(candidate: StoredCoreState): void { const now = Date.now(); for (const [credentialId, ticket] of Object.entries( - this.#store.state.tickets, + candidate.tickets, )) { if (ticket.usedAt !== null || ticket.expiresAtUnixMs <= now) { - delete this.#store.state.tickets[credentialId]; + delete candidate.tickets[credentialId]; } } for (const [credentialId, lease] of Object.entries( - this.#store.state.leases, + candidate.leases, )) { if ( - credentialId !== this.#store.state.warmTransition?.credentialId && + credentialId !== candidate.warmTransition?.credentialId && (lease.revokedAt !== null || lease.expiresAtUnixMs <= now) ) { - delete this.#store.state.leases[credentialId]; + delete candidate.leases[credentialId]; } } } @@ -2432,6 +3234,15 @@ export class DurablePrpControlPlane { ? authorization.protocolVersion : Math.min(protocolVersion, payload.protocolMax as number); const challengePayload: Record = { + ...(this.indexedPersistence ? { durability: INDEXED_DURABILITY_CAPABILITY, outputBodies: "history.output_bodies.v1" } : {}), + ...(this.#store instanceof IndexedCoreStore && this.#store.authority.commandEpochs && payload.commandEpochs === COMMAND_EPOCH_CAPABILITY ? { + commandEpochs: COMMAND_EPOCH_CAPABILITY, + commandResume: { controllerEpoch: (payload.resume as { controllerEpoch?: string })?.controllerEpoch ?? null, + lastControllerCommandSeq: Number((payload.resume as { lastControllerCommandSeq?: number })?.lastControllerCommandSeq) }, + } : {}), + ...(selectedVersion >= 2 && this.#store instanceof IndexedCoreStore && this.#store.authority.eventEpochs && payload.eventEpochs === EVENT_EPOCH_CAPABILITY && isEventResume(payload.eventResume) ? { + eventEpochs: EVENT_EPOCH_CAPABILITY, eventResume: structuredClone(payload.eventResume), eventEpochLimit: this.#eventEpochLimit, + } : {}), credentialId: authorization.credentialId, credentialKind: authorization.kind, clientNonce: payload.clientNonce, @@ -2465,6 +3276,15 @@ export class DurablePrpControlPlane { [Buffer.from(canonicalChallenge)], ).toString("hex"); connection.pendingChallenge = { + ...(this.indexedPersistence ? { durability: INDEXED_DURABILITY_CAPABILITY, outputBodies: "history.output_bodies.v1" } : {}), + ...(this.#store instanceof IndexedCoreStore && this.#store.authority.commandEpochs && payload.commandEpochs === COMMAND_EPOCH_CAPABILITY ? { + commandEpochs: COMMAND_EPOCH_CAPABILITY, + commandResume: { controllerEpoch: (payload.resume as { controllerEpoch?: string })?.controllerEpoch ?? null, + lastControllerCommandSeq: Number((payload.resume as { lastControllerCommandSeq?: number })?.lastControllerCommandSeq) }, + } : {}), + ...(selectedVersion >= 2 && this.#store instanceof IndexedCoreStore && this.#store.authority.eventEpochs && payload.eventEpochs === EVENT_EPOCH_CAPABILITY && isEventResume(payload.eventResume) ? { + eventEpochs: EVENT_EPOCH_CAPABILITY, eventResume: structuredClone(payload.eventResume), eventEpochLimit: this.#eventEpochLimit, + } : {}), authorization, deadlineUnixMs: Math.min( authorization.expiresAtUnixMs, @@ -2549,6 +3369,12 @@ export class DurablePrpControlPlane { return; } } + return this.#serialize(async () => { + authorization = this.#reauthorizePendingChallenge(pending, Date.now()); + if (!authorization || !this.#connections.has(connection) || connection.pendingChallenge !== pending) { + connection.close(); + return; + } const clientProof = expectedClientProof.toString("hex"); // A held proof may span preparation or activation on another connection. // Reapply today's transition lane policy, not merely the old credential @@ -2556,6 +3382,10 @@ export class DurablePrpControlPlane { if ( this.#authorizeHello({ credentialId: pending.authorization.credentialId, + ...(pending.durability ? { durability: pending.durability } : {}), + ...(pending.outputBodies ? { outputBodies: pending.outputBodies } : {}), + ...(pending.commandEpochs ? { commandEpochs: pending.commandEpochs, resume: pending.commandResume } : {}), + ...(pending.eventEpochs ? { eventEpochs: pending.eventEpochs, eventResume: pending.eventResume } : {}), ...pending.requestedIdentity, runnerVersion: this.#expectedRunnerVersion, runnerDigest: this.#expectedRunnerDigest, @@ -2615,7 +3445,7 @@ export class DurablePrpControlPlane { new Date().toISOString(); candidate.warmTransition!.credentialId = material.credentialId; } - this.#store.commit(candidate); + await this.#store.commit(candidate); } else { lease = authorization.lease; } @@ -2643,7 +3473,7 @@ export class DurablePrpControlPlane { }; candidate.leases = { [lease.credentialId]: structuredClone(lease) }; candidate.runAttachTemplate = this.#store.state.runAttachTemplate; - this.#store.commit(candidate); + await this.#store.commit(candidate); this.#identity = structuredClone(candidate.identity); lease = this.#store.state.leases[lease.credentialId]!; } @@ -2652,6 +3482,10 @@ export class DurablePrpControlPlane { connection.lease = lease; connection.identity = structuredClone(requestedIdentity); connection.warmTransitionVersion = pending.warmTransitionVersion ?? null; + connection.commandEpochs = pending.commandEpochs === COMMAND_EPOCH_CAPABILITY; + connection.eventEpochs = pending.eventEpochs === EVENT_EPOCH_CAPABILITY; + connection.eventResume = pending.eventResume ?? null; + connection.eventEpochLimit = pending.eventEpochLimit ?? EVENT_EPOCH_LIMIT; connection.activationReceipt = this.#store.state.warmTransition?.phase === "activated" ? structuredClone(this.#store.state.warmTransition.receipt) @@ -2664,7 +3498,7 @@ export class DurablePrpControlPlane { this.#store.state.warmTransition !== undefined && this.#store.state.warmTransition.phase !== "activated"; if (connection.activationReceipt) connection.replayOnly = true; - connection.connectionId = `connection_${this.#store.state.connectionCount + 1}`; + connection.connectionId = `connection_${randomUUID()}`; connection.secureChannel = createSecureChannel( authorization.authKey, pending.canonicalChallenge, @@ -2675,19 +3509,22 @@ export class DurablePrpControlPlane { if (active !== connection && active.secureChannel !== null) active.close(); } - this.#welcome(connection, leaseToken); + await this.#welcome(connection, leaseToken); + }); } - #welcome(connection: AuthorityConnection, leaseToken: string | null): void { + async #welcome(connection: AuthorityConnection, leaseToken: string | null): Promise { const lease = connection.lease; if (lease === null || connection.connectionId === null) { connection.close(); return; } - this.#store.state.connectionCount += 1; - this.#store.state.lastLeaseId = lease.leaseId; - this.#store.state.lastLeaseExpiresAt = lease.expiresAt; + const candidate = structuredClone(this.#store.state); + if (connection.commandEpochs && candidate.indexedState) candidate.indexedState.commandEpochsNegotiated = true; + candidate.connectionCount = incrementDiagnosticCount(candidate.connectionCount); + candidate.lastLeaseId = lease.leaseId; + candidate.lastLeaseExpiresAt = lease.expiresAt; const pending = connection.replayOnly ? [] : this.#nextPendingCommand(); const [pendingCommand] = pending; @@ -2696,14 +3533,14 @@ export class DurablePrpControlPlane { ? pendingCommand.commandId : null; for (const command of pending) { - this.#store.state.commandDeliveryCounts[command.commandId] = - (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; + candidate.commandDeliveryCounts[command.commandId] = + incrementDiagnosticCount(candidate.commandDeliveryCounts[command.commandId] ?? 0); } - this.#store.save(); + await this.#store.commit(candidate); connection.sendJson({ protocol, version: lease.protocolVersion, - envelopeId: `welcome_${this.#store.state.connectionCount}`, + envelopeId: `welcome_${randomUUID()}`, kind: "welcome", runnerInstanceId: this.#identity.runnerInstanceId, environmentLeaseId: this.#identity.environmentLeaseId, @@ -2718,6 +3555,8 @@ export class DurablePrpControlPlane { selectedVersion: lease.protocolVersion, heartbeatIntervalMs: 250, connectionLeaseRenewalVersion: 1, + ...(connection.commandEpochs ? { commandEpochs: COMMAND_EPOCH_CAPABILITY } : {}), + ...(connection.eventEpochs ? { eventEpochs: EVENT_EPOCH_CAPABILITY, eventEpochLimit: connection.eventEpochLimit, sourceEpoch: connection.eventResume!.sourceEpoch } : {}), connectionLeaseId: lease.leaseId, ...(leaseToken === null ? {} : { connectionLeaseToken: leaseToken }), connectionLeaseExpiresAt: lease.expiresAt, @@ -2732,7 +3571,7 @@ export class DurablePrpControlPlane { }, maxFrameBytes, maxBatchEvents: 100, - ackedSourceSeq: this.#store.state.ackedSourceSeq, + ackedSourceSeq: connection.eventResume && connection.eventResume.sourceEpoch !== (this.#store.state.indexedState?.sourceEpoch ?? null) ? this.#store.state.indexedState!.lastEventEpochTransition!.finalOrdinal : this.#store.state.ackedSourceSeq, pendingCommands: pending.map(this.#wireCommand), ...(connection.warmTransitionVersion === 1 ? { warmTransitionVersion: 1 } @@ -2750,6 +3589,7 @@ export class DurablePrpControlPlane { : {}), }, }); + if (this.#store.state.indexedState?.commandEpochTransition && pending.length === 0) await this.#sendNextCommand(connection); } #wireCommand( @@ -2796,7 +3636,7 @@ export class DurablePrpControlPlane { }; } - #sendNextCommand(connection: AuthorityConnection): void { + async #sendNextCommand(connection: AuthorityConnection): Promise { if ( connection.terminalLifecycleCommandId !== null || connection.replayOnly || @@ -2804,27 +3644,34 @@ export class DurablePrpControlPlane { ) return; const [command] = this.#nextPendingCommand(); - if (command === undefined) return; + if (command === undefined) { + const transition = this.#store.state.indexedState?.commandEpochTransition; + if (transition && connection.commandEpochs) connection.sendJson(this.#controlEnvelope(connection, + `command_epoch_${transition.transitionId}`, "command_epoch_rotate", transition as unknown as Record)); + return; + } if (this.#isTerminalLifecycleCommand(command)) { connection.terminalLifecycleCommandId = command.commandId; } - this.#store.state.commandDeliveryCounts[command.commandId] = - (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; - this.#store.save(); + const candidate = structuredClone(this.#store.state); + candidate.commandDeliveryCounts[command.commandId] = + incrementDiagnosticCount(candidate.commandDeliveryCounts[command.commandId] ?? 0); + await this.#store.commit(candidate); connection.sendJson( this.#controlEnvelope( connection, - `command_${command.commandId}_${this.#store.state.commandDeliveryCounts[command.commandId]}`, + `command_${randomUUID()}`, "command", this.#wireCommand(command), ), ); } - #commandResult( + async #commandResult( connection: AuthorityConnection, envelope: Record, - ): void { + ): Promise { + if (!this.#liveConnectionIsCurrent(connection)) return; const result = envelope.payload as Record | undefined; const commandId = result?.commandId; if (result === undefined || typeof commandId !== "string") { @@ -2857,14 +3704,12 @@ export class DurablePrpControlPlane { credentialId: transition.credentialId, command: structuredClone(completed), }; - this.#store.commit(candidate); + await this.#store.commit(candidate); } this.#ackWarmTransition(connection, transition.receipt); return; } - const command = this.#store.state.commands.find( - (candidate) => candidate.commandId === commandId, - ); + const command = await this.getCommand(commandId); if (command === undefined) { connection.close(); return; @@ -2892,11 +3737,12 @@ export class DurablePrpControlPlane { connection.close(); return; } - this.#store.state.duplicateCommandResults += 1; - this.#store.save(); + const candidate = structuredClone(this.#store.state); + candidate.duplicateCommandResults = incrementDiagnosticCount(candidate.duplicateCommandResults); + await this.#store.commit(candidate); this.#ackTerminalCommandResult(connection, command); if (!this.#isTerminalLifecycleCommand(command)) { - this.#sendNextCommand(connection); + await this.#sendNextCommand(connection); } return; } @@ -2917,6 +3763,7 @@ export class DurablePrpControlPlane { connection.lease, this.#expectedRunnerVersion, this.#expectedRunnerDigest, + this.#store.state.indexedState?.sourceEpoch, ); const candidate = structuredClone(this.#store.state); const completed = candidate.commands.find( @@ -2931,17 +3778,19 @@ export class DurablePrpControlPlane { credentialId: connection.lease.credentialId, command: structuredClone(completed), }; - this.#store.commit(candidate); + await this.#store.commit(candidate); connection.replayOnly = true; this.#ackWarmTransition(connection, receipt); return; } - command.status = status; - command.result = structuredClone(result); - this.#store.save(); - this.#ackTerminalCommandResult(connection, command); + const candidate = structuredClone(this.#store.state); + const settled = candidate.commands.find((entry) => entry.commandId === commandId)!; + settled.status = status; + settled.result = structuredClone(result); + await this.#store.commit(candidate); + this.#ackTerminalCommandResult(connection, settled); if (!this.#isTerminalLifecycleCommand(command)) { - this.#sendNextCommand(connection); + await this.#sendNextCommand(connection); } } @@ -2967,7 +3816,8 @@ export class DurablePrpControlPlane { #isTerminalLifecycleCommand(command: DurableRecoveryCoreCommand): boolean { return ( - command.type === "runner.suspend" || command.type === "runner.shutdown" + (command.type === "runner.suspend" || command.type === "runner.shutdown") && + command.controllerEpoch === this.#store.state.indexedState?.controllerEpoch ); } @@ -3036,10 +3886,12 @@ export class DurablePrpControlPlane { } const event = validated.event; const sourceSeq = event?.sourceSeq; + const sourceEpoch = event?.sourceEpoch; const sourceEventId = event?.sourceEventId; const eventType = event?.eventType; const priority = event?.priority; if ( + (sourceEpoch !== undefined && !connection.eventEpochs) || typeof sourceSeq !== "number" || typeof sourceEventId !== "string" || typeof eventType !== "string" || @@ -3077,13 +3929,15 @@ export class DurablePrpControlPlane { connection.close(); return; } - const existing = this.#store.state.committedEvents.find( + let existing = this.#store.state.committedEvents.find( (candidate) => candidate.sourceEventId === sourceEventId, - ); + ) ?? (this.#store instanceof IndexedCoreStore + ? (await this.#store.authority.getRecord(this.#identity.runId, "event", sourceEventId))?.body as unknown as DurableRecoveryCommittedEvent | undefined + : undefined); if ( existing === undefined - ? sourceSeq !== this.#store.state.ackedSourceSeq + 1 - : sourceSeq !== existing.sourceSeq + ? sourceEpoch !== this.#store.state.indexedState?.sourceEpoch || sourceSeq !== this.#store.state.ackedSourceSeq + 1 + : sourceEpoch !== existing.sourceEpoch || sourceSeq !== existing.sourceSeq ) { connection.close(); return; @@ -3123,6 +3977,7 @@ export class DurablePrpControlPlane { // a new external effect whose local evidence would then be discarded. const eventToEvict = existing === undefined && + !this.indexedPersistence && this.#store.state.committedEvents.length >= maxCommittedEventWindow ? this.#store.state.committedEvents.findIndex( (candidate) => @@ -3139,7 +3994,7 @@ export class DurablePrpControlPlane { // then do we advance the cumulative cursor. Reversing this order can make // an uncommitted event disappear from the runner outbox permanently. try { - await this.#onCommittedEvent?.(event); + if (!this.indexedPersistence) await this.#onCommittedEvent?.(event); } catch (error) { if (error instanceof NativeSessionProtocolIntegrityError) { this.#failProtocolIntegrity(connection, error); @@ -3148,32 +4003,41 @@ export class DurablePrpControlPlane { } return; } + const committed = await this.#serialize(async () => { // Another authenticated connection can replace this one while its commit // is in flight. Once that exact owner has faulted, even a prior successful // commit cannot reopen delivery or invoke a new business operation. if (this.#protocolIntegrityError !== null) { connection.close(); - return; + return false; } + const candidate = structuredClone(this.#store.state); + existing = candidate.committedEvents.find((candidate) => candidate.sourceEventId === sourceEventId) ?? + (this.#store instanceof IndexedCoreStore ? (await this.#store.authority.getRecord(this.#identity.runId, "event", sourceEventId))?.body as unknown as DurableRecoveryCommittedEvent | undefined : undefined); + if ((existing ? existing.sourceEpoch !== sourceEpoch || existing.sourceSeq !== sourceSeq || canonicalJson(existing.envelope) !== canonicalJson(envelope) : sourceEpoch !== candidate.indexedState?.sourceEpoch || sourceSeq !== candidate.ackedSourceSeq + 1) || envelope.runId !== this.#identity.runId) { + connection.close(); + return false; + } if (existing !== undefined) { - existing.deliveryCount += 1; - this.#store.state.replayDeliveries += 1; + existing.deliveryCount = incrementDiagnosticCount(existing.deliveryCount); + candidate.replayDeliveries = incrementDiagnosticCount(candidate.replayDeliveries); } else { - if (this.#store.state.committedEvents.length >= maxCommittedEventWindow) { + if (!this.indexedPersistence && candidate.committedEvents.length >= maxCommittedEventWindow) { // The awaited business commit may allow another authenticated owner // or a tool completion to advance the window. Re-evaluate, never use // an index sampled before that await to delete a different input. - const currentEviction = this.#store.state.committedEvents.findIndex( - (candidate) => !unsettledSemanticInput(candidate, this.#store.state), + const currentEviction = candidate.committedEvents.findIndex( + (event) => !unsettledSemanticInput(event, candidate), ); if (currentEviction < 0) { connection.close(); - return; + return false; } - this.#store.state.committedEvents.splice(currentEviction, 1); + candidate.committedEvents.splice(currentEviction, 1); } - this.#store.state.committedEvents.push({ + candidate.committedEvents.push({ + ...(sourceEpoch ? { sourceEpoch } : {}), sourceSeq, sourceEventId, eventType, @@ -3182,9 +4046,23 @@ export class DurablePrpControlPlane { deliveryCount: 1, logicalEffectCount: 1, }); - this.#store.state.ackedSourceSeq = sourceSeq; + candidate.ackedSourceSeq = sourceSeq; + if (candidate.indexedState) { + candidate.indexedState.providerEverStarted = true; + if (isSemanticInput || ["mcp_app.tool_input", "runtime.input.requested", "runtime_request.created"].includes(eventType)) { + candidate.indexedState.externalEffectEverAdmitted = true; + } + if (isSemanticInput) candidate.indexedState.pendingSemanticInputIds.push(sourceEventId); + } } - this.#store.save(); + await this.#store.commit(candidate); + + + return true; + }); + if (!committed) return; + if (this.indexedPersistence) await this.#onCommittedEvent?.(event); + if (this.#protocolIntegrityError !== null || envelope.runId !== this.#identity.runId) { connection.close(); return; } if ( isSemanticInput && @@ -3213,14 +4091,12 @@ export class DurablePrpControlPlane { .update(`${this.#identity.runId}\0${call.callId}`) .digest("hex") .slice(0, 32)}`; - const alreadyQueued = this.#store.state.commands.some( - (command) => command.commandId === commandId, - ); + const alreadyQueued = (await this.getCommand(commandId)) !== undefined; if (!alreadyQueued && !this.#pendingSemanticCalls.has(commandId)) { this.#pendingSemanticCalls.add(commandId); - const queueResult = (result: unknown, isError: boolean): void => { + const queueResult = async (result: unknown, isError: boolean): Promise => { try { - this.queueCommand( + await this.queueCommand( "semantic_tool.result", { ...call, result, isError }, commandId, @@ -3245,16 +4121,11 @@ export class DurablePrpControlPlane { } } - connection.sendJson( - this.#controlEnvelope( - connection, - `ack_${this.#store.state.ackedSourceSeq}`, - "ack", - { - ackedSourceSeq: this.#store.state.ackedSourceSeq, - }, - ), - ); + const ackedSourceSeq = sourceEpoch === this.#store.state.indexedState?.sourceEpoch ? this.#store.state.ackedSourceSeq : (await this.readEventEpochTransition(sourceEpoch))?.finalOrdinal; + if (ackedSourceSeq === undefined) { connection.close(); return; } + connection.sendJson(this.#controlEnvelope(connection, `ack_${randomUUID()}`, "ack", { + ackedSourceSeq, ...(sourceEpoch ? { sourceEpoch } : {}), + })); } } @@ -3336,6 +4207,7 @@ function runnerEnvironment( } export function spawnRunner(options: { + indexedDurability?: boolean; connectUrl?: string; connection?: RunnerProcessConnection; stateDirectory: string; @@ -3396,6 +4268,7 @@ export function spawnRunner(options: { connection.listenPath, ]; const args = [ + ...(options.indexedDurability ? ["--indexed-state"] : []), ...connectionArgs, "--state-dir", options.stateDirectory, diff --git a/packages/paperclip-runner/src/control-plane/event-epochs.ts b/packages/paperclip-runner/src/control-plane/event-epochs.ts new file mode 100644 index 00000000000..5f7ccfac887 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/event-epochs.ts @@ -0,0 +1,83 @@ +import { createHash } from "node:crypto"; +import { isCommandEpoch } from "./command-epochs.js"; +export const EVENT_EPOCH_CAPABILITY = "transport.event_epochs.v1"; +export const EVENT_EPOCH_LIMIT = 1_048_576; +/** The sender drains its durable outbox before installing this successor. */ +export interface EventEpochTransition { + schema: "paperclip.prp.event-epoch.v1"; + runId: string; + transitionId: string; + fromEpoch: string | null; + nextEpoch: string; + finalOrdinal: number; +} +export interface EventResume { + sourceEpoch: string | null; + nextSourceEventSeq: number; + ackedSourceSeq: number; + transition: EventEpochTransition | null; +} +export function isEventEpochTransition(value: unknown): value is EventEpochTransition { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const v = value as EventEpochTransition; + return Object.keys(v).length === 6 && v.schema === "paperclip.prp.event-epoch.v1" + && typeof v.runId === "string" && v.runId.length > 0 && v.runId.length <= 240 + && isCommandEpoch(v.transitionId) && (v.fromEpoch === null || isCommandEpoch(v.fromEpoch)) + && isCommandEpoch(v.nextEpoch) && v.nextEpoch !== v.fromEpoch + && Number.isSafeInteger(v.finalOrdinal) && v.finalOrdinal > 0; +} +export function isEventResume(value: unknown): value is EventResume { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const v = value as EventResume; + return Object.keys(v).length === 4 && (v.sourceEpoch === null || isCommandEpoch(v.sourceEpoch)) + && Number.isSafeInteger(v.nextSourceEventSeq) && v.nextSourceEventSeq > 0 + && Number.isSafeInteger(v.ackedSourceSeq) && v.ackedSourceSeq >= 0 && v.ackedSourceSeq < v.nextSourceEventSeq + && (v.transition === null || (isEventEpochTransition(v.transition) && v.transition.fromEpoch === v.sourceEpoch && v.transition.finalOrdinal === v.nextSourceEventSeq - 1)); +} +export function eventEpochCloseId(transition: Pick): string { + return `event-epoch-from-${createHash("sha256").update(JSON.stringify([transition.runId, transition.fromEpoch])).digest("hex")}`; +} +export function compareCurrentEvents(a: { sourceEpoch?: string; sourceSeq: number }, b: { sourceEpoch?: string; sourceSeq: number }, epoch: string | undefined): number { + if (a.sourceEpoch === b.sourceEpoch) return a.sourceSeq - b.sourceSeq; + return Number(a.sourceEpoch === epoch) - Number(b.sourceEpoch === epoch); +} + +/** The normalized outbox commits this boundary with its first successor event. + * These identities never depend on a lifetime ordinal. */ +export function normalizedEpochCloseId(runId: string, sourceInstanceId: string, fromEpoch: string | null): string { + return `normalized-epoch-from-${createHash("sha256").update(JSON.stringify([runId, sourceInstanceId, fromEpoch])).digest("hex")}`; +} +export function normalizedEventId(sourceInstanceId: string, runId: string, sourceSeq: number, sourceEpoch?: string): string { + return sourceEpoch === undefined ? `${sourceInstanceId}:${runId}:${sourceSeq}` + : `normalized-${createHash("sha256").update(JSON.stringify([sourceInstanceId, runId, sourceEpoch, sourceSeq])).digest("hex")}`; +} +export interface SourceCursor { sourceEpoch?: string; sourceSeq: number } +/** Only a verified predecessor and an explicit first-event receipt can cross + * namespaces. Numeric ordering is meaningful inside one namespace only. */ +export function advanceSourceCursor(previous: SourceCursor, event: SourceCursor & { runId: string; sourceEpochTransition?: EventEpochTransition }): SourceCursor { + if (!Number.isSafeInteger(previous.sourceSeq) || previous.sourceSeq < 0 || !Number.isSafeInteger(event.sourceSeq) || event.sourceSeq < 1 + || (previous.sourceEpoch !== undefined && !isCommandEpoch(previous.sourceEpoch)) + || (event.sourceEpoch !== undefined && !isCommandEpoch(event.sourceEpoch))) throw new Error("invalid source cursor"); + if (previous.sourceEpoch === event.sourceEpoch) { + if (event.sourceEpochTransition !== undefined || previous.sourceSeq === Number.MAX_SAFE_INTEGER || event.sourceSeq !== previous.sourceSeq + 1) throw new Error("source cursor is not contiguous"); + } else { + const t = event.sourceEpochTransition; + if (!isEventEpochTransition(t) || t.runId !== event.runId || t.fromEpoch !== (previous.sourceEpoch ?? null) + || t.nextEpoch !== event.sourceEpoch || t.finalOrdinal !== previous.sourceSeq || event.sourceSeq !== 1) throw new Error("source epoch has no exact predecessor"); + } + return { ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), sourceSeq: event.sourceSeq }; +} + +export function encodeSourceCursor(cursor: SourceCursor): string { + if (!Number.isSafeInteger(cursor.sourceSeq) || cursor.sourceSeq < 0 || (cursor.sourceEpoch !== undefined && !isCommandEpoch(cursor.sourceEpoch))) throw new Error("invalid source cursor"); + return cursor.sourceEpoch ? `e:${cursor.sourceEpoch}:${cursor.sourceSeq}` : String(cursor.sourceSeq); +} +export function decodeSourceCursor(value: string | null | undefined): SourceCursor { + if (value?.startsWith("e:")) { + const match = /^e:([^:]+):(0|[1-9][0-9]*)$/.exec(value); + if (!match || !isCommandEpoch(match[1]) || !Number.isSafeInteger(Number(match[2]))) throw new Error("invalid source cursor"); + return { sourceEpoch: match[1], sourceSeq: Number(match[2]) }; + } + const sourceSeq = Number(value ?? 0); + return { sourceSeq: Number.isSafeInteger(sourceSeq) && sourceSeq >= 0 ? sourceSeq : 0 }; +} diff --git a/packages/paperclip-runner/src/control-plane/indexed-inspection-codec.ts b/packages/paperclip-runner/src/control-plane/indexed-inspection-codec.ts new file mode 100644 index 00000000000..acb0c38502d --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/indexed-inspection-codec.ts @@ -0,0 +1,75 @@ +import { createHash } from "node:crypto"; +import type { IndexedLocalSnapshot } from "./indexed-local-state-reader.js"; + +const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; +const { authorityGeneration } = await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js"); + +const CHUNK_BYTES = 192 * 1024; +const MAX_SNAPSHOT_BYTES = 40 * 1024 * 1024; +const MAX_FRAME_BYTES = CHUNK_BYTES * 4 / 3 + 128; +const object = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +const invalid = () => new Error("native_indexed_inspection_invalid_or_incomplete"); +const digest = (value: unknown): value is string => typeof value === "string" && value.length === 64 && /^[a-f0-9]+$/.test(value); + +/** Decoder shared by local worker reads and owned remote inspection channels. + * Limits apply to one bounded current snapshot, never accumulated history. */ +export class IndexedInspectionDecoder { + private pending: Buffer = Buffer.alloc(0); + private header?: { byteLength: number; sha256: string }; + private chunks: Buffer[] = []; + private hash = createHash("sha256"); + private length = 0; + private complete = false; + private finished = false; + + push(chunk: Uint8Array): void { + if (this.finished) throw invalid(); + try { + const bytes = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + let start = 0; + while (start < bytes.length) { + const end = bytes.indexOf(10, start); + const part = bytes.subarray(start, end < 0 ? bytes.length : end); + if (this.pending.length + part.length > MAX_FRAME_BYTES) throw invalid(); + this.pending = Buffer.concat([this.pending, part]); + if (end < 0) break; + this.line(this.pending); this.pending = Buffer.alloc(0); start = end + 1; + } + } catch { throw invalid(); } + } + + private line(bytes: Buffer): void { + if (this.complete) throw invalid(); + const value: unknown = JSON.parse(bytes.toString("utf8")); + if (!object(value)) throw invalid(); + if (!this.header) { + if (value.schema !== "paperclip.indexed-inspection.v1" || !Number.isSafeInteger(value.byteLength) || + Number(value.byteLength) < 1 || Number(value.byteLength) > MAX_SNAPSHOT_BYTES || !digest(value.sha256)) throw invalid(); + this.header = { byteLength: Number(value.byteLength), sha256: value.sha256 }; + } else if (value.complete === true) { + if (Object.keys(value).length !== 1 || this.length !== this.header.byteLength) throw invalid(); + this.complete = true; + } else { + if (value.index !== this.chunks.length || typeof value.bytes !== "string" || value.bytes.length > CHUNK_BYTES * 4 / 3) throw invalid(); + const chunk = Buffer.from(value.bytes, "base64"); + if (chunk.toString("base64") !== value.bytes || chunk.length !== Math.min(CHUNK_BYTES, this.header.byteLength - this.length) || chunk.length === 0) throw invalid(); + this.hash.update(chunk); this.chunks.push(chunk); this.length += chunk.length; + } + } + + /** Call only after a clean native process exit and the complete stdout stream. */ + finish(): IndexedLocalSnapshot { + if (this.finished) throw invalid(); + this.finished = true; + try { + if (this.pending.length || !this.complete || !this.header || this.hash.digest("hex") !== this.header.sha256) throw invalid(); + const value: unknown = JSON.parse(Buffer.concat(this.chunks, this.length).toString("utf8")); + if (!object(value) || !object(value.state) || typeof value.generation !== "string" || value.generation !== value.generation.trim() || + value.generation === "0" || !digest(value.stateDigest) || + (value.preparation !== null && !object(value.preparation))) throw invalid(); + authorityGeneration(value.generation); + return value as unknown as IndexedLocalSnapshot; + } catch { throw invalid(); } + finally { this.chunks = []; } + } +} diff --git a/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.test.ts b/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.test.ts new file mode 100644 index 00000000000..e4a4784390c --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.test.ts @@ -0,0 +1,97 @@ +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, it } from "vitest"; +import { readIndexedLocalState, suspendIndexedRunnerState } from "./indexed-local-state-reader.js"; + +it.each([1, 2])("schema %s seals only the verified empty runner generation without replacing its locator or receipts", async (schema) => { + const root = mkdtempSync(join(tmpdir(), "runner-indexed-proof-")); + const path = join(root, "runner-state.json"); + const database = join(root, "runner-state.sqlite"); + const locator = JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/one/session" }); + writeFileSync(path, locator, { mode: 0o600 }); + const db = new DatabaseSync(database); + chmodSync(database, 0o600); + writeFileSync(`${database}.lifetime`, "", { mode: 0o600 }); + try { + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB); CREATE TABLE receipts(key TEXT PRIMARY KEY,bytes BLOB)"); + db.prepare("INSERT INTO store_binding VALUES (1,?,?)").run(schema, "runner/one/session"); + if (schema === 2) db.exec("CREATE TABLE receipt_prepare(singleton INTEGER PRIMARY KEY,state_key TEXT,generation INTEGER,body BLOB,digest BLOB)"); + db.prepare("INSERT INTO receipts VALUES ('ancient',?)").run(Buffer.from("original outcome")); + const activeInput = schema === 2 ? "界".repeat(500_000) : "small"; + const write = (generation: number | bigint | string, ackedSourceSeq: number) => { + const body = Buffer.from(JSON.stringify({ lifecycle: "ready", nextSourceSeq: 101, ackedSourceSeq, pendingTerminalDelivery: null, activeInput })); + db.prepare("INSERT OR REPLACE INTO current_state VALUES ('runner',?,?,?)").run(generation, body, createHash("sha256").update(body).digest()); + }; + write(1, 100); + expect(await readIndexedLocalState(path)).toMatchObject({ generation: "1", state: { outbox: [], lifecycle: "ready", activeInput } }); + rmSync(`${database}.lifetime`); + await expect(readIndexedLocalState(path)).rejects.toThrow(); + expect(existsSync(`${database}.lifetime`)).toBe(false); + const foreign = join(root, "foreign-lifetime"); + writeFileSync(foreign, "foreign", { mode: 0o600 }); + symlinkSync(foreign, `${database}.lifetime`); + await expect(readIndexedLocalState(path)).rejects.toThrow(); + expect(readFileSync(foreign, "utf8")).toBe("foreign"); + rmSync(`${database}.lifetime`); + writeFileSync(`${database}.lifetime`, "", { mode: 0o600 }); + if (schema === 2) { + db.exec("CREATE TABLE backup_manifest(singleton INTEGER PRIMARY KEY,id TEXT,complete INTEGER); INSERT INTO backup_manifest VALUES(1,'backup',0)"); + await expect(readIndexedLocalState(path)).rejects.toThrow(/incomplete backup/); + await expect(suspendIndexedRunnerState(path, "1")).rejects.toThrow("backup_incomplete"); + expect(db.prepare("SELECT generation FROM current_state").get()!.generation).toBe(1); + db.exec("UPDATE backup_manifest SET complete=1"); + db.exec("INSERT INTO receipt_prepare SELECT 1,key,generation,bytes,digest FROM current_state"); + expect(await readIndexedLocalState(path)).toMatchObject({ generation: "1", state: { lifecycle: "ready" } }); + await suspendIndexedRunnerState(path, "1"); + expect(db.prepare("SELECT generation FROM receipt_prepare").get()!.generation).toBe(2); + expect(await readIndexedLocalState(path)).toMatchObject({ generation: "2", state: { lifecycle: "suspended" } }); + db.exec("UPDATE receipt_prepare SET generation=1"); + await expect(readIndexedLocalState(path)).rejects.toThrow(/conflicting prepared receipts/); + db.exec("DELETE FROM receipt_prepare"); + } + write(2, 100); + await expect(suspendIndexedRunnerState(path, "1")).rejects.toThrow("stale_or_unsettled"); + await suspendIndexedRunnerState(path, "2"); + expect(await readIndexedLocalState(path)).toMatchObject({ generation: "3", state: { outbox: [], lifecycle: "suspended" } }); + expect(readFileSync(path, "utf8")).toBe(locator); + expect(Buffer.from(db.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome"); + write(9223372036854775807n, 100); + expect((await readIndexedLocalState(path)).generation).toBe("9223372036854775807"); + await suspendIndexedRunnerState(path, "9223372036854775807"); + const opaque = (await readIndexedLocalState(path)).generation; + expect(opaque).toMatch(/^r:[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/); + await expect(suspendIndexedRunnerState(path, "9223372036854775807")).rejects.toThrow("stale_or_unsettled"); + await suspendIndexedRunnerState(path, opaque); + expect((await readIndexedLocalState(path)).generation).not.toBe(opaque); + expect(Buffer.from(db.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome"); + write(4, 99); + expect((await readIndexedLocalState(path)).state.outbox).toEqual([{ indexedPending: true }]); + await expect(suspendIndexedRunnerState(path, "4")).rejects.toThrow("stale_or_unsettled"); + db.exec("UPDATE current_state SET digest=zeroblob(32)"); + await expect(readIndexedLocalState(path)).rejects.toThrow(/digest.*mismatch/); + } finally { + db.close(); + rmSync(root, { recursive: true, force: true }); + } +}, 20_000); + +it("reaps its native reader before releasing a failed or timed-out read", async () => { + const root = mkdtempSync(join(tmpdir(), "runner-native-reader-owner-")); + try { + const binary = join(root, "reader"); + writeFileSync(binary, `#!${process.execPath}\nconst fs = require('node:fs');\nconst path = process.argv.at(-1);\nfs.writeFileSync(path + '.pid', String(process.pid));\nprocess.stdout.write(path.endsWith('invalid') ? 'invalid\\n' + 'x'.repeat(2 * 1024 * 1024) : JSON.stringify({schema:'paperclip.indexed-inspection.v1',byteLength:1,sha256:'a'.repeat(64)})+'\\n');\nsetInterval(() => {}, 1000);\n`, { mode: 0o700 }); + for (const kind of ["invalid", "stalled"]) { + const path = join(root, kind); + await expect(readIndexedLocalState(path, { runnerBinary: binary, timeoutMs: 2000 })).rejects.toThrow(kind === "invalid" ? "invalid_or_incomplete" : "read_timeout"); + const pid = Number(readFileSync(`${path}.pid`, "utf8")); + expect(() => process.kill(pid, 0)).toThrow(); + } + await expect(readIndexedLocalState(join(root, "absent"), { runnerBinary: join(root, "missing-binary") })).rejects.toThrow("ENOENT"); + // All ownership/admission slots are available after failures. + const failed = await Promise.allSettled(Array.from({ length: 4 }, () => readIndexedLocalState(join(root, "invalid"), { runnerBinary: binary }))); + expect(failed.every(result => result.status === "rejected" && String(result.reason).includes("invalid_or_incomplete"))).toBe(true); + } finally { rmSync(root, { recursive: true, force: true }); } +}, 15_000); diff --git a/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.ts b/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.ts new file mode 100644 index 00000000000..f1cb8ce9201 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/indexed-local-state-reader.ts @@ -0,0 +1,83 @@ +import { spawn } from "node:child_process"; +import { Worker } from "node:worker_threads"; +import { basename, dirname } from "node:path"; +import { defaultStorageRunnerBinary, nativeStorageCommand } from "./native-storage-command.js"; +import { authorityGeneration } from "./durable-authority-store.js"; + +let readers = 0; +export interface IndexedLocalSnapshot { + state: Record; + generation: string; + stateDigest: string; + preparation: Record | null; +} + +/** Reads one runner-owned current-state row off the JavaScript event loop. */ +export function readIndexedLocalState(path: string, options: { runnerBinary?: string; timeoutMs?: number } = {}): Promise { + return localStateJob(path, options); +} + +/** Caller must independently fence and prove the runner/process group stopped. + * Only the exact verified, empty checkpoint can be sealed for cold continuation. */ +export async function suspendIndexedRunnerState(path: string, expectedGeneration: string, runnerBinary?: string): Promise { + if (basename(path) !== "runner-state.json" || expectedGeneration === "0") throw new Error("indexed_state_seal_invalid"); + try { authorityGeneration(expectedGeneration); } catch { throw new Error("indexed_state_seal_invalid"); } + await nativeStorageCommand(["seal", "--directory", dirname(path), "--generation", expectedGeneration], runnerBinary); +} + +async function localStateJob(path: string, options: { runnerBinary?: string; timeoutMs?: number }): Promise { + if (readers >= 4) throw new Error("indexed_state_reader_busy"); + const timeoutMs = options.timeoutMs ?? 30_000; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30_000) throw new Error("indexed_state_read_timeout_invalid"); + readers += 1; + let worker: Worker | undefined; + let child: ReturnType | undefined; + let closed: Promise | undefined; + let timer: ReturnType | undefined; + try { + const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; + worker = new Worker(new URL(`./indexed-native-state-worker.${extension}`, import.meta.url), { execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 128 } }); + child = spawn(options.runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "inspect-state", "--path", path], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, env: {} }); + closed = new Promise(resolve => child!.once("close", () => resolve())); + return await new Promise((resolve, reject) => { + let settled = false, stderr = ""; + const fail = (error: Error) => { if (!settled) { settled = true; reject(error); } }; + timer = setTimeout(() => fail(new Error("indexed_state_read_timeout")), timeoutMs); + worker!.on("message", (message: { consumed?: boolean; value?: IndexedLocalSnapshot; error?: string }) => { + if (settled) return; + if (message.error) fail(new Error(message.error)); + else if (message.value) { settled = true; resolve(message.value); } + else if (message.consumed) child!.stdout!.resume(); + else fail(new Error("indexed_state_reader_invalid_message")); + }); + worker!.once("error", fail); + worker!.once("exit", () => { if (!settled) fail(new Error("indexed_state_reader_exited")); }); + child!.once("error", fail); + child!.stderr!.on("data", (chunk: Buffer) => { if (stderr.length < 8192) stderr += chunk.toString("utf8").slice(0, 8192 - stderr.length); }); + child!.stdout!.on("data", (chunk: Buffer) => { + // One unconsumed transport chunk at a time; worker message queues must + // not become a second current-state/history buffer. + child!.stdout!.pause(); + if (!settled) worker!.postMessage({ bytes: chunk }); + }); + child!.once("close", code => { + if (settled) return; + if (code !== 0) fail(new Error(stderr.trim() || "indexed_state_native_reader_failed")); + else worker!.postMessage({ complete: true }); + }); + }); + } finally { + if (timer) clearTimeout(timer); + // A decoder failure or deadline cannot leak a native SQLite reader/lifetime + // fence. Keep the admission credit until the exact owned child is reaped. + try { + if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + // Failure may leave stdout paused with additional bytes in the pipe. + // Destroy owned streams so unread output cannot delay the close/reap. + child?.stdout?.destroy(); child?.stderr?.destroy(); + await closed; + } finally { + try { await worker?.terminate(); } finally { readers -= 1; } + } + } +} diff --git a/packages/paperclip-runner/src/control-plane/indexed-local-state-worker.ts b/packages/paperclip-runner/src/control-plane/indexed-local-state-worker.ts new file mode 100644 index 00000000000..03476c20423 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/indexed-local-state-worker.ts @@ -0,0 +1,82 @@ +import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from "node:fs"; +import { dirname, resolve, basename } from "node:path"; +import { createHash } from "node:crypto"; +import { DatabaseSync } from "node:sqlite"; +import { parentPort, workerData } from "node:worker_threads"; + +function privateFile(path: string, maximum: number): Buffer { + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > maximum || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("indexed_state_file_unsafe"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { const count = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!count) throw new Error("indexed_state_file_changed"); offset += count; } + const after = fstatSync(fd); + if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("indexed_state_file_changed"); + return bytes; + } finally { closeSync(fd); } +} +try { + const path = String(workerData.path); + const directory = dirname(path); + const dir = lstatSync(directory); + if (!dir.isDirectory() || dir.isSymbolicLink() || (process.platform !== "win32" && ((dir.mode & 0o077) !== 0 || dir.uid !== process.getuid?.()))) throw new Error("indexed_state_directory_unsafe"); + const locator = JSON.parse(privateFile(path, 4096).toString("utf8")) as { schema?: string; binding?: string }; + const type = locator.schema === "paperclip.runner.durable.state.indexed.v1" ? "runner" + : locator.schema === "paperclip.runner.codex-provider-state.indexed.v1" ? "codex-provider" : null; + if (!type || typeof locator.binding !== "string" || locator.binding.length > 1024) throw new Error("indexed_state_locator_invalid"); + const databasePath = resolve(realpathSync(directory), `${basename(path, ".json")}.sqlite`); + for (const suffix of ["", "-wal", "-shm"]) { + const stat = lstatSync(databasePath + suffix, { throwIfNoEntry: false }); + if (!stat && suffix) continue; + if (!stat || !stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("indexed_state_database_unsafe"); + } + const db = new DatabaseSync(databasePath, { readOnly: true }); + let result: Record; + try { + db.exec("PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000; PRAGMA query_only=ON;"); + const binding = db.prepare("SELECT schema_version,binding FROM store_binding WHERE singleton=1").get(); + if (![1, 2].includes(Number(binding?.schema_version)) || binding?.binding !== locator.binding) throw new Error("indexed_state_binding_mismatch"); + db.exec("BEGIN"); + const backup = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_manifest'").get(); + if (backup && db.prepare("SELECT complete FROM backup_manifest WHERE singleton=1").get()?.complete !== 1) throw new Error("indexed_state_backup_incomplete"); + const importing = db.prepare("SELECT CASE WHEN length(bytes)<=16384 THEN bytes END AS bytes,digest FROM current_state WHERE key='legacy-import'").get(); + let preparation: Record | null = null; + if (importing) { + if (!(importing.bytes instanceof Uint8Array) || !(importing.digest instanceof Uint8Array) + || !createHash("sha256").update(importing.bytes).digest().equals(Buffer.from(importing.digest))) throw new Error("indexed_state_import_invalid"); + const progress = JSON.parse(Buffer.from(importing.bytes).toString()) as Record; + if (progress.schema !== "paperclip.runner.local-import.v1" || progress.binding !== locator.binding || !progress.prepared) throw new Error("indexed_state_import_incomplete"); + preparation = progress.prepared as Record; + } + const metadata = db.prepare("SELECT length(bytes) AS size FROM current_state WHERE key=?").get(type); + if (!metadata || typeof metadata.size !== "number" || metadata.size > 32 * 1024 * 1024) throw new Error("indexed_state_snapshot_invalid"); + const statement = db.prepare("SELECT generation,bytes,CASE WHEN length(digest)=32 THEN digest END AS digest,length(bytes) AS size FROM current_state WHERE key=?"); + statement.setReadBigInts(true); + const row = statement.get(type); + if (!row || typeof row.generation !== "bigint" || row.generation <= 0n || Number(row.size) > 32 * 1024 * 1024 || !(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array)) throw new Error("indexed_state_snapshot_invalid"); + const bytes = Buffer.from(row.bytes); + if (!createHash("sha256").update(bytes).digest().equals(Buffer.from(row.digest))) throw new Error("indexed_state_digest_mismatch"); + const state = JSON.parse(bytes.toString("utf8")) as Record; + const stateDigest = Buffer.from(row.digest).toString("hex"); + const generation = row.generation; + if (binding!.schema_version === 2) { + const pendingQuery = db.prepare("SELECT generation,CASE WHEN length(body)<=33554432 THEN body END AS body,digest FROM receipt_prepare WHERE state_key=?"); + pendingQuery.setReadBigInts(true); + const pending = pendingQuery.get(type); + if (pending && (pending.generation !== generation || !(pending.body instanceof Uint8Array) || !(pending.digest instanceof Uint8Array) || !bytes.equals(Buffer.from(pending.body)) || !Buffer.from(row.digest).equals(Buffer.from(pending.digest)))) throw new Error("indexed_state_prepared_commit_mismatch"); + } + if (type === "runner") { + const next = state.nextSourceSeq, acked = state.ackedSourceSeq; + if (!Number.isSafeInteger(next) || !Number.isSafeInteger(acked) || Number(next) <= Number(acked)) throw new Error("indexed_state_cursor_invalid"); + // This is a proof view, not an outbox restore. A non-empty outbox remains + // a negative fence without loading the pending payloads into the server. + state.outbox = Number(next) === Number(acked) + 1 ? [] : [{ indexedPending: true }]; + } + result = { state, generation: String(generation), stateDigest, preparation }; + } finally { db.close(); } + parentPort!.postMessage(result); +} catch (error) { + parentPort!.postMessage({ error: error instanceof Error ? error.message : "indexed_state_reader_failed" }); +} diff --git a/packages/paperclip-runner/src/control-plane/indexed-native-state-worker.ts b/packages/paperclip-runner/src/control-plane/indexed-native-state-worker.ts new file mode 100644 index 00000000000..0abb58ddfb0 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/indexed-native-state-worker.ts @@ -0,0 +1,22 @@ +import { parentPort } from "node:worker_threads"; + +// Source-mode workers run through Node's type stripping, compiled workers use +// the adjacent JS module. The worker owns only bounded decoding, never SQLite +// or a child process that could outlive worker termination. +const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; +const { IndexedInspectionDecoder } = await import(new URL(`./indexed-inspection-codec.${extension}`, import.meta.url).href); +const decoder = new IndexedInspectionDecoder(); +parentPort!.on("message", (message: { bytes?: Uint8Array; complete?: boolean }) => { + try { + if (message.bytes) { + decoder.push(message.bytes); + parentPort!.postMessage({ consumed: true }); + } else if (message.complete) { + parentPort!.postMessage({ value: decoder.finish() }); + parentPort!.close(); + } else throw new Error("indexed_state_reader_invalid_message"); + } catch (error) { + parentPort!.postMessage({ error: error instanceof Error ? error.message : "indexed_state_reader_failed" }); + parentPort!.close(); + } +}); diff --git a/packages/paperclip-runner/src/control-plane/legacy-authority-activation.ts b/packages/paperclip-runner/src/control-plane/legacy-authority-activation.ts new file mode 100644 index 00000000000..7847b377a43 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-authority-activation.ts @@ -0,0 +1,205 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, rename } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { authorityDirectlyFollows, authorityJson, type DurableAuthorityStore } from "./durable-authority-store.js"; +import { legacyControllerActivation, stageLegacyControlPlaneAuthority, type StoredCoreState } from "./durable-prp-control-plane.js"; +import { readIndexedLocalState } from "./indexed-local-state-reader.js"; +import type { DurableRecoveryIdentity } from "./prp-transport-types.js"; + +export interface PreparedLocalAuthority { + schema: "paperclip.runner.prepared-legacy-authority.v1"; + binding: string; + fenceId: string; + sourceDigest: string; + currentDigest: string; + currentKey: "runner" | "codex-provider"; + generation: 1; + receipts: number; +} +export interface PreparedLocalSession { runner: PreparedLocalAuthority; provider: PreparedLocalAuthority } +type SourceIdentity = { dev: string; ino: string; size: string; mtime: string; ctime: string }; +type Preparation = PreparedLocalSession & { controllerGeneration: string; controllerDigest: string; files: string[]; sources: Record }; +interface Marker { + schema: "paperclip.runner.legacy-activation.v1"; + identity: DurableRecoveryIdentity; + fenceId: string; + phase: "copying" | "activating" | "active"; + preparation: Preparation | null; + preparationDigest: string | null; +} +export interface LegacySessionMigrationOptions { + root: string; + identity: DurableRecoveryIdentity; + authority: DurableAuthorityStore; + fenceId: string; + /** Use the same qualified native artifact for staging and current reads. */ + runnerBinary?: string; + /** Must prove exclusive controller ownership AND every old runner/provider + * writer stopped. An expired lease or a missing PID alone is insufficient. */ + assertExclusiveFence(): Promise; + /** Run the qualified runnerd `storage stage-legacy` command with the exact + * source invocation. It writes only this private destination. */ + prepareLocal(destination: string): Promise; + onBoundary?(name: string): Promise; +} +const digest = (value: unknown) => createHash("sha256").update(authorityJson(value)).digest("hex"); +const prefixes = ["runner-state", "codex-provider-state"]; +const dataFiles = prefixes.flatMap(prefix => [".sqlite", ".sqlite-wal", ".sqlite-shm", ".sqlite.receipts", ".sqlite.routing", ".sqlite.lifetime", ".sqlite.lock"].map(suffix => prefix + suffix)); +const locators = prefixes.map(prefix => `${prefix}.json`); +const isStoreDirectory = (file: string) => file.endsWith(".receipts") || file.endsWith(".routing"); + +async function stat(path: string) { try { return await lstat(path); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; } } +async function privatePath(path: string, directory = false): Promise { + const value = await lstat(path); + if (value.isSymbolicLink() || (directory ? !value.isDirectory() : !value.isFile()) || + (process.platform !== "win32" && ((value.mode & 0o077) !== 0 || value.uid !== process.getuid?.()))) throw new Error("native_legacy_migration_unsafe"); +} +async function sourceIdentity(path: string): Promise { + await privatePath(path); + const value = await lstat(path, { bigint: true }); + return { dev: String(value.dev), ino: String(value.ino), size: String(value.size), mtime: String(value.mtimeNs), ctime: String(value.ctimeNs) }; +} +function sameSource(a: SourceIdentity, b: SourceIdentity, linked = false) { + return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtime === b.mtime && (linked || a.ctime === b.ctime); +} +async function syncDirectory(path: string) { + if (process.platform === "win32") return; + const fd = await open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { await fd.sync(); } finally { await fd.close(); } +} +async function publish(path: string, value: unknown) { + if (await stat(path)) await privatePath(path); + const temporary = `${path}.${randomUUID()}.tmp`; + const file = await open(temporary, "wx", 0o600); + try { await file.writeFile(JSON.stringify(value)); await file.sync(); } finally { await file.close(); } + await rename(temporary, path); + await syncDirectory(dirname(path)); +} +async function readMarker(path: string): Promise { + if (!await stat(path)) return null; + await privatePath(path); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await file.stat(); + if (before.size > 16 * 1024) throw new Error("native_legacy_migration_marker_oversized"); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { const { bytesRead } = await file.read(bytes, offset, bytes.length - offset, offset); if (!bytesRead) throw new Error("native_legacy_migration_changed"); offset += bytesRead; } + const after = await file.stat(); + if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("native_legacy_migration_changed"); + return JSON.parse(bytes.toString()) as Marker; + } finally { await file.close(); } +} +function validateLocal(proof: PreparedLocalAuthority, kind: "runner" | "codex-provider", identity: DurableRecoveryIdentity, fenceId: string) { + const binding = `${kind === "runner" ? "runner" : "provider"}/${identity.runnerInstanceId}/${identity.normalizedSessionId}`; + if (proof.schema !== "paperclip.runner.prepared-legacy-authority.v1" || proof.binding !== binding || proof.fenceId !== fenceId || proof.currentKey !== kind || proof.generation !== 1 || + !Number.isSafeInteger(proof.receipts) || proof.receipts < 0 || !/^[a-f0-9]{64}$/.test(proof.sourceDigest) || !/^[a-f0-9]{64}$/.test(proof.currentDigest)) throw new Error("native_legacy_preparation_invalid"); +} +async function verifyLocal(directory: string, prepared: PreparedLocalSession, identity: DurableRecoveryIdentity, fenceId: string, runnerBinary?: string) { + for (const [name, proof] of [["runner-state", prepared.runner], ["codex-provider-state", prepared.provider]] as const) { + validateLocal(proof, name === "runner-state" ? "runner" : "codex-provider", identity, fenceId); + const snapshot = await readIndexedLocalState(resolve(directory, `${name}.json`), { runnerBinary }); + if (snapshot.generation !== "1" || snapshot.stateDigest !== proof.currentDigest || authorityJson(snapshot.preparation) !== authorityJson(proof)) throw new Error("native_legacy_preparation_changed"); + if (name === "runner-state" && Object.entries(identity).some(([key, value]) => snapshot.state[key] !== value)) throw new Error("native_legacy_runner_binding_changed"); + } +} + +/** Roll forward a three-store migration. The durable intent gates BOTH formats + * until the exact controller commit and all local stores have been published. + * Originals remain private and inspectable; no rollback guesses from history. */ +export async function migrateLegacySessionAuthority(options: LegacySessionMigrationOptions): Promise { + const { identity, fenceId, authority } = options; + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/.test(fenceId)) throw new Error("native_legacy_migration_fence_invalid"); + const root = resolve(options.root), controller = resolve(root, "control-plane"), runner = resolve(root, "runner"); + const staging = resolve(root, "indexed-migration"), local = resolve(staging, "runner"), originals = resolve(staging, "legacy"); + const markerPath = resolve(root, "indexed-migration.json"), sourcePath = resolve(controller, "control-plane-state.json"); + await options.assertExclusiveFence(); + for (const path of [root, controller, runner]) await privatePath(path, true); + let marker = await readMarker(markerPath); + if (marker && (marker.schema !== "paperclip.runner.legacy-activation.v1" || marker.fenceId !== fenceId || authorityJson(marker.identity) !== authorityJson(identity) + || !["copying", "activating", "active"].includes(marker.phase))) throw new Error("native_legacy_migration_conflict"); + if (!marker) { + marker = { schema: "paperclip.runner.legacy-activation.v1", identity, fenceId, phase: "copying", preparation: null, preparationDigest: null }; + await publish(markerPath, marker); + await options.onBoundary?.("intent"); + } + // Completed execution may have advanced since migration. Never rewrite it. + if (marker.phase === "active") return; + for (const path of [staging, originals]) { + try { await mkdir(path, { mode: 0o700 }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } + await privatePath(path, true); await syncDirectory(dirname(path)); + } + if (marker.phase === "copying") { + const sources: Record = {}; + for (const name of ["control-plane-state.json", ...locators]) sources[name] = await sourceIdentity(name === "control-plane-state.json" ? sourcePath : resolve(runner, name)); + const prepared = await stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId, assertExclusiveFence: options.assertExclusiveFence }); + const peers = await options.prepareLocal(local); + await verifyLocal(local, peers, identity, fenceId, options.runnerBinary); + const runnerState = (await readIndexedLocalState(resolve(local, "runner-state.json"), { runnerBinary: options.runnerBinary })).state; + const projection = prepared.state.projection as unknown as StoredCoreState; + if (projection.warmTransition || Number(runnerState.ackedSourceSeq) > projection.ackedSourceSeq || Number(runnerState.nextSourceSeq) <= projection.ackedSourceSeq || + Number(runnerState.lastControllerCommandSeq) >= projection.indexedState!.nextControllerSeq) throw new Error("native_legacy_peer_cursors_conflict"); + await options.assertExclusiveFence(); + // Preparation of local peers can take time. Revalidate the original + // controller identity after it, without scanning its copied prefix again. + await stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId, assertExclusiveFence: options.assertExclusiveFence }); + for (const name of Object.keys(sources)) if (!sameSource(sources[name]!, await sourceIdentity(name === "control-plane-state.json" ? sourcePath : resolve(runner, name)))) throw new Error("native_legacy_source_changed"); + const files: string[] = []; + for (const file of [...dataFiles, ...locators]) if (await stat(resolve(local, file))) { await privatePath(resolve(local, file), isStoreDirectory(file)); files.push(file); } + if (![...prefixes.flatMap(prefix => [`${prefix}.sqlite`, `${prefix}.sqlite.routing`]), ...locators].every(file => files.includes(file))) throw new Error("native_legacy_preparation_incomplete"); + marker.preparation = { ...peers, controllerGeneration: prepared.generation, controllerDigest: digest(prepared.state), files, sources }; + marker.preparationDigest = digest(marker.preparation); + marker.phase = "activating"; + await publish(markerPath, marker); + await options.onBoundary?.("prepared"); + } + const prepared = marker.preparation; + if (!prepared || marker.preparationDigest !== digest(prepared) || !Array.isArray(prepared.files) || new Set(prepared.files).size !== prepared.files.length + || prepared.files.some(file => ![...dataFiles, ...locators].includes(file))) throw new Error("native_legacy_migration_preparation_invalid"); + validateLocal(prepared.runner, "runner", identity, fenceId); validateLocal(prepared.provider, "codex-provider", identity, fenceId); + for (const name of ["control-plane-state.json", ...locators]) { + const backup = resolve(originals, name), linked = !!await stat(backup); + const source = linked ? backup : name === "control-plane-state.json" ? sourcePath : resolve(runner, name); + // Our hard link changes ctime but preserves the original inode, bytes and + // modification time. Before that link, require the full source identity. + if (!prepared.sources?.[name] || !sameSource(prepared.sources[name]!, await sourceIdentity(source), linked)) throw new Error("native_legacy_source_changed"); + } + await options.assertExclusiveFence(); + const current = await authority.load(); + if (!current) throw new Error("native_legacy_controller_missing"); + const receipt = (current.state.indexedState as StoredCoreState["indexedState"])?.legacyActivation; + if (receipt?.fenceId === fenceId && receipt.preparationDigest === marker.preparationDigest && !authorityDirectlyFollows(current, prepared.controllerGeneration)) throw new Error("native_legacy_controller_advanced_before_publication"); + if (receipt?.fenceId !== fenceId || receipt.preparationDigest !== marker.preparationDigest) { + if (current.generation !== prepared.controllerGeneration || digest(current.state) !== prepared.controllerDigest) throw new Error("native_legacy_controller_changed"); + const projection = legacyControllerActivation(current, identity, fenceId, marker.preparationDigest!); + await authority.commit({ expectedGeneration: current.generation, state: projection as unknown as Record, records: [] }); + await options.onBoundary?.("controller-commit"); + } + // Preserve exact original bytes with no-clobber hard links before replacing + // any locator. All peers are stopped; the external fence is rechecked below. + for (const [name, path] of [["control-plane-state.json", sourcePath], ...locators.map(file => [file, resolve(runner, file)])] as const) { + const backup = resolve(originals, name!); + if (!await stat(backup)) { await privatePath(path!); await link(path!, backup); await syncDirectory(originals); } + else await privatePath(backup); + } + for (const file of prepared.files) { + await options.assertExclusiveFence(); + const source = resolve(local, file), target = resolve(runner, file); + const sourceStat = await stat(source), targetStat = await stat(target); + if (sourceStat) { + await privatePath(source, isStoreDirectory(file)); + if (targetStat && !locators.includes(file)) throw new Error("native_legacy_destination_conflict"); + if (targetStat) await privatePath(target, isStoreDirectory(file)); + await rename(source, target); await syncDirectory(local); await syncDirectory(runner); + await options.onBoundary?.(`publish:${file}`); + } else await privatePath(target, isStoreDirectory(file)); + } + await verifyLocal(runner, prepared, identity, fenceId, options.runnerBinary); + await options.assertExclusiveFence(); + await publish(sourcePath, { schema: "paperclip.runner.authority-locator.v1", location: authority.location }); + await options.onBoundary?.("controller-locator"); + marker.phase = "active"; + await publish(markerPath, marker); + await options.onBoundary?.("active"); +} diff --git a/packages/paperclip-runner/src/control-plane/legacy-authority-import.test.ts b/packages/paperclip-runner/src/control-plane/legacy-authority-import.test.ts new file mode 100644 index 00000000000..35ca9466a04 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-authority-import.test.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { DurablePrpControlPlane, stageLegacyControlPlaneAuthority, type StoredCoreState } from "./durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "./sqlite-authority-store.js"; +import type { DurableRecoveryCoreCommand, DurableRecoveryIdentity } from "./prp-transport-types.js"; +const identity: DurableRecoveryIdentity = { runnerInstanceId: "migration-runner", environmentLeaseId: "migration-environment", normalizedSessionId: "migration-session", runId: "migration-run", turnId: "migration-turn", itemId: "migration-item" }; +const command = (seq: number): DurableRecoveryCoreCommand => ({ schema: "paperclip.prp.command.v1",commandId:`command-${seq}`,controllerSeq:seq,type:"run.prepare",issuedAt:"2026-09-28T00:00:00.000Z",payload:{},status:"completed",result:{status:"completed",commandId:`command-${seq}`,controllerSeq:seq,commandType:"run.prepare"} }); +const coreOptions = (directory: string) => ({ stateDirectory: directory, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); +function event(seq: number, eventType: string, payload: Record) { + return { sourceSeq:seq,sourceEventId:`event-${seq}`,eventType,priority:1,deliveryCount:1,logicalEffectCount:1,envelope:{protocol:"paperclip.runner",version:1,kind:"event",...identity,payload:{schema:"paperclip.prp.event.v1",schemaVersion:1,...identity,sourceSeq:seq,sourceEventId:`event-${seq}`,sourceInstanceId:identity.runnerInstanceId,sourceKind:"runner",eventType,priority:1,emittedAt:"2026-09-28T00:00:00.000Z",payload}} }; +} +it("stages a >192 MiB controller journal with bounded current state, exact old receipts, and restartable checkpoints", async () => { + const directory = await mkdtemp(join(tmpdir(), "legacy-authority-")); + const sourcePath = join(directory, "control-plane-state.json"); + const storage = { path: join(directory, "authority.sqlite"), binding: "legacy-import-test", create: true }; + let authority = await SqliteAuthorityStore.open(storage); + try { + const legacy = await DurablePrpControlPlane.open(coreOptions(directory)); + await legacy.stop(); + const initial = JSON.parse(await readFile(sourcePath, "utf8")); + const fd = await open(sourcePath, "w", 0o600); + await fd.write('{"committedEvents":['); // history precedes commands deliberately + const payload = { code: "history", text: "x".repeat(100 * 1024) }; + for (let seq = 1; seq <= 2000; seq++) await fd.write(`${seq > 1 ? "," : ""}${JSON.stringify(event(seq,"harness.diagnostic",payload))}`); + const tail = { ...initial, commands:[command(1),command(2)],ackedSourceSeq:2000,commandDeliveryCounts:{"command-1":1,"command-2":3} }; + delete tail.committedEvents; + await fd.write(`],${JSON.stringify(tail).slice(1)}`); + await fd.close(); + let checkpoints=0; + const input={sourcePath,identity,authority,fenceId:"exclusive-migration-1",assertExclusiveFence:async()=>{}}; + await expect(stageLegacyControlPlaneAuthority({...input,onCheckpoint:async()=>{if(++checkpoints===3)throw new Error("simulated controller death");}})).rejects.toThrow("simulated controller death"); + const interrupted=await authority.load(); + expect((interrupted!.state.cursor as {offset:number}).offset).toBeGreaterThan(16*1024*1024); + await authority.close(); + authority=await SqliteAuthorityStore.open({...storage,create:false}); + const prepared=await stageLegacyControlPlaneAuthority({...input,authority}); + expect(prepared.state.phase).toBe("prepared"); + expect((prepared.state.cursor as {offset:number}).offset).toBeGreaterThan(192*1024*1024); + expect(JSON.stringify(prepared.state).length).toBeLessThan(16*1024); + const current=prepared.state.projection as unknown as StoredCoreState; + expect(current.commands.map(command=>command.commandId)).toEqual(["command-2"]); + expect(current.commandDeliveryCounts).toEqual({"command-2":3}); + expect(current.indexedState!.nextControllerSeq).toBe(3); + expect((await authority.getRecord(identity.runId,"command","command-1"))!.body).toEqual(command(1)); + expect((await authority.getRecord(identity.runId,"event","event-1"))!.body).toEqual(event(1,"harness.diagnostic",payload)); + // Preparation must not replace the legacy locator or activate staging. + const beginning=await open(sourcePath,"r"); + const bytes=Buffer.alloc(24);await beginning.read(bytes,0,bytes.length,0);await beginning.close(); + expect(bytes.toString()).toContain('"committedEvents"'); + await expect(DurablePrpControlPlane.open({...coreOptions(directory),authorityStore:authority})).rejects.toThrow(); + await expect(stageLegacyControlPlaneAuthority({...input,authority,fenceId:"wrong-owner"})).rejects.toThrow("different authority or fence"); + } finally {await authority.close();await rm(directory,{recursive:true,force:true});} +},180_000); + +it("reconciles semantic inputs after all commands and refuses a lost ownership fence",async()=>{ + const directory=await mkdtemp(join(tmpdir(),"legacy-semantic-import-")); + const sourcePath=join(directory,"control-plane-state.json"); + const authority=await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"semantic-import",create:true}); + try{ + const legacy=await DurablePrpControlPlane.open(coreOptions(directory));await legacy.stop(); + const initial=JSON.parse(await readFile(sourcePath,"utf8")); + const correlation={runId:identity.runId,normalizedSessionId:identity.normalizedSessionId,turnId:identity.turnId,itemId:identity.itemId}; + const semantic=(callId:string)=>({semantic_tool:{callId,operationId:"get_task_context",correlation,input:{}}}); + const commandId=`command_tool_${createHash("sha256").update(`${identity.runId}\0settled-call`).digest("hex").slice(0,32)}`; + const settled={...command(1),commandId,type:"semantic_tool.result",payload:{callId:"settled-call",operationId:"get_task_context",sourceEventId:"event-1",sourceEventType:"semantic_tool.input",correlation,input:{}},result:{commandId,controllerSeq:1,commandType:"semantic_tool.result",status:"completed"}}; + const {committedEvents:_events,commands:_commands,...fields}=initial; + await writeFile(sourcePath,JSON.stringify({committedEvents:[event(1,"semantic_tool.input",semantic("settled-call")),event(2,"semantic_tool.input",semantic("pending-call"))],...fields,commands:[settled],ackedSourceSeq:2}),{mode:0o600}); + const input={sourcePath,identity,authority,fenceId:"semantic-fence",assertExclusiveFence:async()=>{}}; + await expect(stageLegacyControlPlaneAuthority({...input,assertExclusiveFence:async()=>{throw new Error("lease lost");}})).rejects.toThrow("lease lost"); + expect(await authority.load()).toBeNull(); + const prepared=await stageLegacyControlPlaneAuthority(input); + const current=prepared.state.projection as unknown as StoredCoreState; + expect(current.indexedState!.pendingSemanticInputIds).toEqual(["event-2"]); + expect(current.committedEvents.map(event=>event.sourceEventId)).toEqual(["event-2"]); + await writeFile(sourcePath,'{"changed":true}',{mode:0o600}); + await expect(stageLegacyControlPlaneAuthority(input)).rejects.toThrow("source changed"); + }finally{await authority.close();await rm(directory,{recursive:true,force:true});} +}); + +it("does not invent execution authority from missing legacy fields", async () => { + const directory = await mkdtemp(join(tmpdir(), "legacy-import-missing-")); + const sourcePath = join(directory, "control-plane-state.json"); + const authority = await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "missing-fields", create: true }); + try { + await writeFile(sourcePath, JSON.stringify({ identity }), { mode: 0o600 }); + await expect(stageLegacyControlPlaneAuthority({ sourcePath, identity, authority, fenceId: "missing-fence", assertExclusiveFence: async () => {} })).rejects.toThrow("missing required source fields"); + expect(await authority.load()).toBeNull(); + } finally { await authority.close(); await rm(directory, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/src/control-plane/legacy-json-reader.test.ts b/packages/paperclip-runner/src/control-plane/legacy-json-reader.test.ts new file mode 100644 index 00000000000..4c117971078 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-json-reader.test.ts @@ -0,0 +1,58 @@ +import { mkdtemp, open, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { LegacyJsonReader, type LegacyJsonCursor } from "./legacy-json-reader.js"; + +it("streams more than 192 MiB losslessly and resumes from a durable page cursor", async () => { + const directory = await mkdtemp(join(tmpdir(), "legacy-import-")); + const path = join(directory, "legacy.json"); + const fd = await open(path, "wx", 0o600); + const payload = "A".repeat(128 * 1024) + ' escaped: \\" ] } λ'; + await fd.write('{"schema":"legacy-test","commands":[],"committedEvents":['); + for (let index = 0; index < 1600; index++) await fd.write(`${index ? "," : ""}${JSON.stringify({ index, payload })}`); + await fd.write('],"commandDeliveryCounts":{"first":2},"identity":{"session":"same"}}'); + await fd.close(); + let reader = new LegacyJsonReader(path); + let cursor: LegacyJsonCursor | null = null; + let source: string | null = null; + let seen = 0, pages = 0; + const fields: Record = {}; + try { + for (;;) { + const page = await reader.page(source, cursor); + expect(page.entries.length).toBeLessThanOrEqual(128); + for (const entry of page.entries) { + if (entry.field === "committedEvents") { expect(entry.key).toBe(String(seen)); expect(entry.value).toEqual({ index: seen++, payload }); } + else fields[entry.field] = entry.value; + } + const before = cursor?.offset ?? 0; + cursor = page.cursor; source = page.source; pages++; + expect(cursor.offset).toBeGreaterThan(before); + if (page.done) break; + if (pages === 5) { await reader.close(); reader = new LegacyJsonReader(path); } + } + expect(seen).toBe(1600); + expect(cursor!.offset).toBeGreaterThan(192 * 1024 * 1024); + expect(fields).toEqual({ schema: "legacy-test", commandDeliveryCounts: 2, identity: { session: "same" } }); + } finally { await reader.close(); await rm(directory, { recursive: true, force: true }); } +}, 60_000); + +it("rejects a changed source on resume and malformed boundaries", async () => { + const directory = await mkdtemp(join(tmpdir(), "legacy-import-invalid-")); + const path = join(directory, "legacy.json"); + try { + for (const content of ['{"commands":[1,]}', '{"a":1,"a":2}', '{"commands":[{"a":]}', '{"a":"unterminated}', '{"a":1} trailing', '{"a":1,}']) { + await writeFile(path, content, { mode: 0o600 }); + const reader = new LegacyJsonReader(path); + try { await expect(reader.page(null, null)).rejects.toThrow(); } finally { await reader.close(); } + } + await writeFile(path, JSON.stringify({ committedEvents: Array.from({ length: 500 }, (_, i) => ({ i })) }), { mode: 0o600 }); + const reader = new LegacyJsonReader(path); + try { + const page = await reader.page(null, null); + await writeFile(path, '{"changed":true}', { mode: 0o600 }); + await expect(reader.page(page.source, page.cursor)).rejects.toThrow("source changed"); + } finally { await reader.close(); } + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/src/control-plane/legacy-json-reader.ts b/packages/paperclip-runner/src/control-plane/legacy-json-reader.ts new file mode 100644 index 00000000000..a7c0c789469 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-json-reader.ts @@ -0,0 +1,30 @@ +import { Worker } from "node:worker_threads"; +import type { LegacyJsonCursor, LegacyJsonPage } from "./legacy-json-scanner.js"; +export type { LegacyJsonCursor, LegacyJsonPage, LegacyJsonEntry } from "./legacy-json-scanner.js"; + +/** One bounded page at a time. Import has no whole-job deadline; an interrupted + * caller resumes at its last transactionally committed source cursor. */ +export class LegacyJsonReader { + private readonly worker: Worker; + private pending: { resolve(page: LegacyJsonPage): void; reject(error: Error): void } | null = null; + private failure: Error | null = null; + constructor(private readonly path: string) { + const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; + this.worker = new Worker(new URL(`./legacy-json-worker.${extension}`, import.meta.url), { execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 128 } }); + this.worker.on("message", (result: { page?: LegacyJsonPage; error?: string }) => { + const pending = this.pending; this.pending = null; + if (!pending) return; + if (result.error || !result.page) pending.reject(new Error(result.error ?? "missing legacy import page")); + else pending.resolve(result.page); + }); + this.worker.on("error", (error) => this.fail(error)); + this.worker.on("exit", () => this.fail(new Error("legacy import reader exited"))); + } + private fail(error: Error): void { this.failure ??= error; this.pending?.reject(this.failure); this.pending = null; } + page(source: string | null, cursor: LegacyJsonCursor | null): Promise { + if (this.failure) return Promise.reject(this.failure); + if (this.pending) return Promise.reject(new Error("legacy import already has a pending page")); + return new Promise((resolve, reject) => { this.pending = { resolve, reject }; this.worker.postMessage({ path: this.path, source, cursor }); }); + } + async close(): Promise { await this.worker.terminate(); } +} diff --git a/packages/paperclip-runner/src/control-plane/legacy-json-scanner.ts b/packages/paperclip-runner/src/control-plane/legacy-json-scanner.ts new file mode 100644 index 00000000000..9d63e9725d7 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-json-scanner.ts @@ -0,0 +1,144 @@ +import { createHash } from "node:crypto"; +import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs"; + +export interface LegacyJsonCursor { + offset: number; + phase: "start" | "field" | "entry" | "entry_separator" | "field_separator" | "done"; + fields: string[]; + collection: { field: string; kind: "array" | "object"; index: string } | null; + prefixDigest: string; +} +export interface LegacyJsonEntry { field: string; key: string | null; value: unknown } +export interface LegacyJsonPage { source: string; cursor: LegacyJsonCursor; entries: LegacyJsonEntry[]; done: boolean } +const CHUNK = 64 * 1024; +const MAX_VALUE = 16 * 1024 * 1024; +const MAX_PAGE = 8 * 1024 * 1024; +const collections: Record = { commands: "array", committedEvents: "array", commandDeliveryCounts: "object" }; +const invalid = () => new Error("invalid legacy migration JSON or cursor"); + +class Scanner { + offset: number; + private buffer = Buffer.allocUnsafe(CHUNK); + private begin = -1; + private length = 0; + private fd: number; + private size: number; + constructor(fd: number, size: number, offset: number) { this.fd = fd; this.size = size; this.offset = offset; } + peek(): number { + if (this.offset >= this.size) return -1; + if (this.offset < this.begin || this.offset >= this.begin + this.length) { + this.begin = this.offset; + this.length = readSync(this.fd, this.buffer, 0, Math.min(CHUNK, this.size - this.offset), this.offset); + if (!this.length) throw new Error("legacy migration source changed"); + } + return this.buffer[this.offset - this.begin]!; + } + space(): void { while ([32, 9, 10, 13].includes(this.peek())) this.offset++; } + expect(byte: number): void { this.space(); if (this.peek() !== byte) throw invalid(); this.offset++; } + /** Locate one JSON value with bounded memory, then let the JSON parser + * validate its complete grammar. Delimiters inside strings remain data. */ + value(maximum = MAX_VALUE): { value: unknown; bytes: number } { + this.space(); + const start = this.offset; + const first = this.peek(); + if (first < 0 || [44, 93, 125].includes(first)) throw invalid(); + let depth = 0, quoted = false, escaped = false; + const container = first === 91 || first === 123; + for (;;) { + const byte = this.peek(); + if (byte < 0) { + if (quoted || depth > 0 || container) throw invalid(); + break; + } + if (!quoted && depth === 0 && this.offset > start && [32, 9, 10, 13, 44, 93, 125].includes(byte)) break; + this.offset++; + if (this.offset - start > maximum) throw new Error("legacy migration record exceeds bounded admission capacity"); + if (quoted) { + if (escaped) escaped = false; + else if (byte === 92) escaped = true; + else if (byte === 34) { quoted = false; if (depth === 0) break; } + } else if (byte === 34) quoted = true; + else if (byte === 91 || byte === 123) { if (++depth > 128) throw invalid(); } + else if (byte === 93 || byte === 125) { if (--depth === 0) break; if (depth < 0) throw invalid(); } + } + const length = this.offset - start; + const bytes = Buffer.allocUnsafe(length); + let offset = 0; + while (offset < length) { + const count = readSync(this.fd, bytes, offset, length - offset, start + offset); + if (!count) throw new Error("legacy migration source changed"); + offset += count; + } + return { value: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), bytes: length }; + } +} + +/** Worker-only lossless input. No total file limit; each page has a durable, + * seekable cursor. Callers must hold the migration's exclusive ownership fence + * and persist the page's records and cursor in the same staging transaction. */ +export function scanLegacyJsonPage(path: string, expectedSource: string | null, previous: LegacyJsonCursor | null): LegacyJsonPage { + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = fstatSync(fd, { bigint: true }); + if (!stat.isFile() || (process.platform !== "win32" && ((stat.mode & 0o077n) !== 0n || stat.uid !== BigInt(process.getuid!())))) throw new Error("unsafe legacy migration source"); + const identity = (value: typeof stat) => JSON.stringify([value.dev, value.ino, value.size, value.mtimeNs, value.ctimeNs].map(String)); + const source = identity(stat); + if (expectedSource !== null && expectedSource !== source) throw new Error("legacy migration source changed"); + const size = Number(stat.size); + if (!Number.isSafeInteger(size)) throw new Error("legacy migration source exceeds filesystem seek range"); + const cursor: LegacyJsonCursor = structuredClone(previous ?? { offset: 0, phase: "start", fields: [], collection: null, prefixDigest: "0".repeat(64) }); + if (!Number.isSafeInteger(cursor.offset) || cursor.offset < 0 || cursor.offset > size || !Array.isArray(cursor.fields) || cursor.fields.length > 128 || !/^[0-9a-f]{64}$/.test(cursor.prefixDigest)) throw invalid(); + const scanner = new Scanner(fd, size, cursor.offset); + const begin = cursor.offset; + const entries: LegacyJsonEntry[] = []; + let bytes = 0; + while (entries.length < 128 && bytes < MAX_PAGE && cursor.phase !== "done") { + if (cursor.phase === "start") { scanner.expect(123); cursor.phase = "field"; } + else if (cursor.phase === "field") { + scanner.space(); + if (scanner.peek() === 125) { scanner.offset++; cursor.phase = "done"; continue; } + const key = scanner.value(4096).value; + if (typeof key !== "string" || cursor.fields.includes(key) || cursor.fields.length >= 128) throw invalid(); + cursor.fields.push(key); scanner.expect(58); + const kind = Object.hasOwn(collections, key) ? collections[key]! : null; + if (kind) { scanner.expect(kind === "array" ? 91 : 123); cursor.collection = { field: key, kind, index: "0" }; cursor.phase = "entry"; } + else { const value = scanner.value(); entries.push({ field: key, key: null, value: value.value }); bytes += value.bytes; cursor.phase = "field_separator"; } + } else if (cursor.phase === "entry") { + const collection = cursor.collection; + if (!collection || collections[collection.field] !== collection.kind || !/^(0|[1-9][0-9]*)$/.test(collection.index) || collection.index.length > 32) throw invalid(); + scanner.space(); + if (scanner.peek() === (collection.kind === "array" ? 93 : 125)) { scanner.offset++; cursor.collection = null; cursor.phase = "field_separator"; continue; } + let key = collection.index; + if (collection.kind === "object") { + const parsed = scanner.value(4096).value; + if (typeof parsed !== "string") throw invalid(); + key = parsed; scanner.expect(58); + } + const value = scanner.value(); + entries.push({ field: collection.field, key, value: value.value }); bytes += value.bytes; + collection.index = String(BigInt(collection.index) + 1n); cursor.phase = "entry_separator"; + } else if (cursor.phase === "entry_separator") { + const collection = cursor.collection; + if (!collection) throw invalid(); + scanner.space(); + if (scanner.peek() === (collection.kind === "array" ? 93 : 125)) { scanner.offset++; cursor.collection = null; cursor.phase = "field_separator"; } + else { scanner.expect(44); scanner.space(); if ([93, 125].includes(scanner.peek())) throw invalid(); cursor.phase = "entry"; } + } else if (cursor.phase === "field_separator") { + scanner.space(); + if (scanner.peek() === 125) { scanner.offset++; cursor.phase = "done"; } + else { scanner.expect(44); scanner.space(); if (scanner.peek() === 125) throw invalid(); cursor.phase = "field"; } + } else throw invalid(); + } + if (cursor.phase === "done") { scanner.space(); if (scanner.offset !== size) throw invalid(); } + const digest = createHash("sha256").update(Buffer.from(cursor.prefixDigest, "hex")); + const buffer = Buffer.allocUnsafe(CHUNK); + for (let offset = begin; offset < scanner.offset;) { + const count = readSync(fd, buffer, 0, Math.min(CHUNK, scanner.offset - offset), offset); + if (!count) throw new Error("legacy migration source changed"); + digest.update(buffer.subarray(0, count)); offset += count; + } + if (source !== identity(fstatSync(fd, { bigint: true }))) throw new Error("legacy migration source changed"); + cursor.offset = scanner.offset; cursor.prefixDigest = digest.digest("hex"); + return { source, cursor, entries, done: cursor.phase === "done" }; + } finally { closeSync(fd); } +} diff --git a/packages/paperclip-runner/src/control-plane/legacy-json-worker.ts b/packages/paperclip-runner/src/control-plane/legacy-json-worker.ts new file mode 100644 index 00000000000..56a57ebb0ed --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-json-worker.ts @@ -0,0 +1,8 @@ +import { parentPort } from "node:worker_threads"; +import type { LegacyJsonCursor } from "./legacy-json-scanner.js"; +const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; +const { scanLegacyJsonPage } = await import(new URL(`./legacy-json-scanner.${extension}`, import.meta.url).href) as typeof import("./legacy-json-scanner.js"); +parentPort!.on("message", (input: { path: string; source: string | null; cursor: LegacyJsonCursor | null }) => { + try { parentPort!.postMessage({ page: scanLegacyJsonPage(input.path, input.source, input.cursor) }); } + catch (error) { parentPort!.postMessage({ error: error instanceof Error ? error.message : "legacy migration read failed" }); } +}); diff --git a/packages/paperclip-runner/src/control-plane/legacy-migration-gate.test.ts b/packages/paperclip-runner/src/control-plane/legacy-migration-gate.test.ts new file mode 100644 index 00000000000..903b26463ea --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-migration-gate.test.ts @@ -0,0 +1,24 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { assertNoPendingLegacyMigration } from "./legacy-migration-gate.js"; +import { readDurableControlPlaneState } from "./authority-locator.js"; + +it("holds absent/legacy/indexed admission while migration is partial and refuses unsafe markers", async () => { + const root = await mkdtemp(join(tmpdir(), "migration-gate-")); + const controller = join(root, "control-plane"), marker = join(root, "indexed-migration.json"); + try { + await mkdir(controller, { mode: 0o700 }); + expect(() => assertNoPendingLegacyMigration(controller)).not.toThrow(); + for (const phase of ["copying", "activating", "unknown"]) { + await writeFile(marker, JSON.stringify({ schema: "paperclip.runner.legacy-activation.v1", phase }), { mode: 0o600 }); + expect(() => assertNoPendingLegacyMigration(controller)).toThrow("migration_pending"); + await expect(readDurableControlPlaneState(controller)).rejects.toThrow("migration_pending"); + } + await writeFile(marker, JSON.stringify({ schema: "paperclip.runner.legacy-activation.v1", phase: "active" }), { mode: 0o600 }); + expect(() => assertNoPendingLegacyMigration(controller)).not.toThrow(); + await rm(marker); await symlink(join(root, "missing"), marker); + expect(() => assertNoPendingLegacyMigration(controller)).toThrow(); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/src/control-plane/legacy-migration-gate.ts b/packages/paperclip-runner/src/control-plane/legacy-migration-gate.ts new file mode 100644 index 00000000000..86701b8f7bf --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-migration-gate.ts @@ -0,0 +1,22 @@ +import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +/** Checked before opening either format. An interrupted migration must never + * turn a temporarily absent file into permission to create fresh authority. */ +export function assertNoPendingLegacyMigration(stateDirectory: string): void { + const path = resolve(dirname(stateDirectory), "indexed-migration.json"); + let fd: number; + try { fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; } + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > 16 * 1024 || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_legacy_migration_unsafe"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { const count = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!count) throw new Error("native_legacy_migration_changed"); offset += count; } + const after = fstatSync(fd); + if (stat.size !== after.size || stat.mtimeMs !== after.mtimeMs || stat.ctimeMs !== after.ctimeMs) throw new Error("native_legacy_migration_changed"); + const marker = JSON.parse(bytes.toString()) as {schema?:unknown;phase?:unknown}; + if (marker.schema !== "paperclip.runner.legacy-activation.v1" || marker.phase !== "active") throw new Error("native_legacy_migration_pending"); + } finally { closeSync(fd); } +} diff --git a/packages/paperclip-runner/src/control-plane/legacy-sqlite-authority-worker.ts b/packages/paperclip-runner/src/control-plane/legacy-sqlite-authority-worker.ts new file mode 100644 index 00000000000..d9211915965 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/legacy-sqlite-authority-worker.ts @@ -0,0 +1,192 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { DatabaseSync, type StatementSync } from "node:sqlite"; +import { createHash } from "node:crypto"; +import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, realpathSync } from "node:fs"; +import { dirname, basename, resolve } from "node:path"; +import type { AuthorityCommit, AuthorityRecord, AuthorityWorkRecord } from "./durable-authority-store.js"; + +// Support the source checkout and the compiled package without inheriting a +// test runner's loader into a storage worker. These modules are type-strippable. +const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; +const { authorityInteger, authorityJson, validateAuthorityCommit, validateAuthorityPage, validateAuthorityWorkKey, validateAuthorityWorkPage, DurableAuthorityStoreError, MAX_AUTHORITY_RECORD_BYTES } = + await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js"); + +const { path, binding, create, readOnly } = workerData as { path: string; binding: string; create: boolean; readOnly?: boolean }; +let database: DatabaseSync; + +function digest(bytes: string): string { return createHash("sha256").update(bytes).digest("hex"); } +function fail(code: InstanceType["code"], message: string): never { throw new DurableAuthorityStoreError(code, message); } + +function privateFile(filePath: string, optional: boolean): void { + let fd: number; + try { fd = openSync(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } + catch (error) { if (optional && (error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; } + try { + const stat = fstatSync(fd); + if (!stat.isFile() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || (process.getuid && stat.uid !== process.getuid())))) fail("invalid_authority", "unsafe storage file"); + } finally { closeSync(fd); } +} + +function openDatabase(): DatabaseSync { + if (create && readOnly) fail("invalid_authority", "a proof reader cannot create authority"); + if (typeof binding !== "string" || !binding.length || binding.length > 1024 || /[\x00-\x1f]/.test(binding)) fail("invalid_authority", "invalid storage binding"); + if (create && !existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const directory = lstatSync(dirname(path)); + if (!directory.isDirectory() || directory.isSymbolicLink() || (process.platform !== "win32" && ((directory.mode & 0o077) !== 0 || (process.getuid && directory.uid !== process.getuid())))) fail("invalid_authority", "unsafe storage directory"); + if (create) { + try { const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); try { fsyncSync(fd); } finally { closeSync(fd); } } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } + } + for (const suffix of ["", "-wal", "-shm"]) privateFile(`${path}${suffix}`, suffix.length > 0); + const db = new DatabaseSync(resolve(realpathSync(dirname(path)), basename(path)), { enableForeignKeyConstraints: true, readOnly }); + try { + const version = String(db.prepare("SELECT sqlite_version() AS version").get()?.version).split(".").map(Number); + if ((version[0]! * 1_000_000 + version[1]! * 1_000 + version[2]!) < 3_051_003) fail("storage_unavailable", "SQLite build lacks the WAL-reset fix"); + db.exec("PRAGMA trusted_schema=OFF; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;"); + if (readOnly) db.exec("PRAGMA query_only=ON"); + else db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA wal_autocheckpoint=256; PRAGMA journal_size_limit=4194304;"); + if (create) { + db.exec("BEGIN IMMEDIATE"); + try { + db.exec(`CREATE TABLE IF NOT EXISTS authority_binding (singleton INTEGER PRIMARY KEY CHECK(singleton=1), schema_version INTEGER NOT NULL, binding TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS authority_state (singleton INTEGER PRIMARY KEY CHECK(singleton=1), generation INTEGER NOT NULL CHECK(generation>0), body TEXT NOT NULL, digest TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS authority_records (epoch TEXT NOT NULL, kind TEXT NOT NULL, id TEXT NOT NULL, sequence INTEGER NOT NULL, body TEXT NOT NULL, digest TEXT NOT NULL, PRIMARY KEY(epoch,kind,id), UNIQUE(epoch,kind,sequence)) WITHOUT ROWID; + CREATE UNIQUE INDEX IF NOT EXISTS authority_session_effect ON authority_records(id) WHERE kind='effect';`); + db.prepare("INSERT OR IGNORE INTO authority_binding VALUES (1,1,?)").run(binding); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + if (process.platform !== "win32") { + const fd = openSync(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY); + try { fsyncSync(fd); } finally { closeSync(fd); } + } + } + const stored = db.prepare("SELECT schema_version,binding FROM authority_binding WHERE singleton=1").get(); + if (stored?.schema_version !== 1 || stored.binding !== binding) fail("invalid_authority", "storage binding or version mismatch"); + if (!readOnly) db.exec("CREATE TABLE IF NOT EXISTS authority_work(collection TEXT NOT NULL,id TEXT NOT NULL,body TEXT NOT NULL,digest TEXT NOT NULL,PRIMARY KEY(collection,id)) WITHOUT ROWID"); + return db; + } catch (error) { db.close(); throw error; } +} + +function exactStatement(sql: string): StatementSync { + const statement = database.prepare(sql); + statement.setReadBigInts(true); + return statement; +} + +function decode(row: Record): AuthorityRecord { + if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "receipt digest mismatch"); + return { epoch: String(row.epoch), kind: row.kind as AuthorityRecord["kind"], id: String(row.id), sequence: String(row.sequence), body: JSON.parse(row.body) }; +} + +function dispatch(operation: string, input: Record): unknown { + switch (operation) { + case "ready": return null; + case "getWork": { + const collection = String(input.collection), id = String(input.id); + validateAuthorityWorkKey(collection, id); + if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_work'").get()) return null; + const row = database.prepare("SELECT CASE WHEN length(CAST(body AS BLOB))<=16384 THEN body END AS body,digest FROM authority_work WHERE collection=? AND id=?").get(collection, id); + if (!row) return null; + if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "outstanding-work digest mismatch"); + return { collection, id, body: JSON.parse(row.body), sha256: row.digest }; + } + case "readWorkPage": { + const collection = String(input.collection), after = String(input.after), limit = Number(input.limit), generation = String(input.expectedGeneration); + validateAuthorityWorkPage(collection, after, limit, generation); + database.exec("BEGIN"); + try { + const current = exactStatement("SELECT generation FROM authority_state WHERE singleton=1").get()?.generation; + if (String(current) !== generation) fail("stale_authority", "outstanding-work page generation changed"); + if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_work'").get()) return { records: [], nextAfter: null }; + const ids = database.prepare("SELECT id FROM authority_work WHERE collection=? AND id>? ORDER BY id LIMIT ?").all(collection, after, limit); + const records = ids.map(row => dispatch("getWork", { collection, id: row.id }) as AuthorityWorkRecord); + return { records, nextAfter: records.at(-1)?.id ?? null }; + } finally { database.exec("ROLLBACK"); } + } + case "load": { + const row = exactStatement("SELECT generation,CASE WHEN length(CAST(body AS BLOB))<=16777216 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_state WHERE singleton=1").get(); + if (!row) return null; + if (typeof row.body !== "string" || row.digest !== digest(row.body)) fail("invalid_authority", "current authority digest mismatch"); + return { generation: String(row.generation), state: JSON.parse(row.body) }; + } + case "getRecord": { + const row = exactStatement("SELECT epoch,kind,id,sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE epoch=? AND kind=? AND id=?").get(String(input.epoch), String(input.kind), String(input.id)); + return row ? decode(row) : null; + } + case "getSessionEffect": { + const row = exactStatement("SELECT epoch,kind,id,sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE kind='effect' AND id=?").get(String(input.id)); + return row ? decode(row) : null; + } + case "readEvents": { + const after = String(input.after), limit = Number(input.limit), byteBudget = Number(input.byteBudget); + validateAuthorityPage(after, limit, byteBudget); + database.exec("BEGIN"); + const records: AuthorityRecord[] = []; + try { + const rows = exactStatement("SELECT id,sequence,length(CAST(body AS BLOB)) AS bytes FROM authority_records WHERE epoch=? AND kind='event' AND sequence>? ORDER BY sequence LIMIT ?").all(String(input.epoch), authorityInteger(after), limit); + let bytes = 0; + for (const row of rows) { + const size = Number(row.bytes); + if (!Number.isSafeInteger(size) || size < 0 || size > MAX_AUTHORITY_RECORD_BYTES) fail("invalid_authority", "receipt exceeds byte capacity"); + if (bytes + size > byteBudget) break; + const record = dispatch("getRecord", { epoch: input.epoch, kind: "event", id: row.id }) as AuthorityRecord | null; + if (!record) fail("invalid_authority", "receipt disappeared"); + bytes += size; + records.push(record); + } + } finally { database.exec("ROLLBACK"); } + return { records, nextAfter: records.at(-1)?.sequence ?? null }; + } + case "commit": { + if (readOnly) fail("invalid_authority", "proof readers cannot commit authority"); + const commit = input as unknown as AuthorityCommit; + validateAuthorityCommit(commit); + if (commit.records.some(record => record.sequenceEpoch !== undefined)) fail("invalid_authority", "retained v1 store cannot rotate command sequences"); + if (commit.records.some(record => record.kind === "effect" && record.sequence === "0")) fail("invalid_authority", "retained v1 store requires ordered effect receipts"); + const expected = authorityInteger(commit.expectedGeneration), generation = expected + 1n; + authorityInteger(String(generation)); + database.exec("BEGIN IMMEDIATE"); + try { + const current = exactStatement("SELECT generation FROM authority_state WHERE singleton=1").get()?.generation ?? 0n; + if (current !== expected) fail("stale_authority", "authority generation changed"); + for (const change of commit.work ?? []) { + const previous = dispatch("getWork", { collection: change.collection, id: change.id }) as AuthorityWorkRecord | null; + if ((previous?.sha256 ?? null) !== change.expectedSha256) fail("stale_authority", "outstanding-work digest changed"); + if (change.body === null) database.prepare("DELETE FROM authority_work WHERE collection=? AND id=?").run(change.collection, change.id); + else { + const body = authorityJson(change.body); + database.prepare("INSERT INTO authority_work VALUES(?,?,?,?) ON CONFLICT(collection,id) DO UPDATE SET body=excluded.body,digest=excluded.digest").run(change.collection, change.id, body, digest(body)); + } + } + for (const record of commit.records) { + const body = authorityJson(record.body); + const existing = exactStatement("SELECT sequence,CASE WHEN length(CAST(body AS BLOB))<=1048576 THEN body END AS body,substr(digest,1,65) AS digest FROM authority_records WHERE epoch=? AND kind=? AND id=?").get(record.epoch, record.kind, record.id); + if (existing) { + if (String(existing.sequence) !== record.sequence || existing.body !== body || existing.digest !== digest(body)) fail("receipt_conflict", "exact receipt differs"); + } else { + const sequenceOwner = database.prepare("SELECT id FROM authority_records WHERE epoch=? AND kind=? AND sequence=?").get(record.epoch, record.kind, authorityInteger(record.sequence)); + if (sequenceOwner) fail("receipt_conflict", "receipt sequence already belongs to another identity"); + database.prepare("INSERT INTO authority_records(epoch,kind,id,sequence,body,digest) VALUES (?,?,?,?,?,?)").run(record.epoch, record.kind, record.id, authorityInteger(record.sequence), body, digest(body)); + } + } + const body = authorityJson(commit.state); + database.prepare("INSERT INTO authority_state(singleton,generation,body,digest) VALUES (1,?,?,?) ON CONFLICT(singleton) DO UPDATE SET generation=excluded.generation,body=excluded.body,digest=excluded.digest").run(generation, body, digest(body)); + database.exec("COMMIT"); + return String(generation); + } catch (error) { if (database.isTransaction) database.exec("ROLLBACK"); throw error; } + } + case "close": database.close(); return null; + default: fail("invalid_authority", "unknown storage operation"); + } +} + +let startupError: unknown; +try { database = openDatabase(); } catch (error) { startupError = error; } +parentPort!.on("message", ({ id, operation, input }: { id: number; operation: string; input: Record }) => { + try { + if (startupError) throw startupError; + parentPort!.postMessage({ id, value: dispatch(operation, input) }); + } catch (error) { + parentPort!.postMessage({ id, error: { code: error instanceof DurableAuthorityStoreError ? error.code : "storage_unavailable", message: error instanceof Error ? error.message : "storage worker failed" } }); + } +}); diff --git a/packages/paperclip-runner/src/control-plane/native-storage-command.ts b/packages/paperclip-runner/src/control-plane/native-storage-command.ts new file mode 100644 index 00000000000..b0a5fe58175 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/native-storage-command.ts @@ -0,0 +1,23 @@ +import { execFile } from "node:child_process"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); +export function defaultStorageRunnerBinary(): string { + const root = resolve(fileURLToPath(new URL("../..", import.meta.url))); + const name = `paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`; + const staged = resolve(root, "dist/bin", name); + return existsSync(staged) ? staged : resolve(root, "runner/target/debug", name); +} + +/** Storage-only native commands never launch a provider or inherit credentials. */ +export async function nativeStorageCommand(args: string[], runnerBinary = defaultStorageRunnerBinary()): Promise { + try { + await execute(runnerBinary, ["storage", ...args], { timeout: 30_000, maxBuffer: 8 * 1024, windowsHide: true, env: {} }); + } catch (error) { + const failure = error as Error & { stderr?: string }; + throw new Error(failure.stderr?.trim() || failure.message, { cause: error }); + } +} diff --git a/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.test.ts b/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.test.ts new file mode 100644 index 00000000000..4b632e31d65 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.test.ts @@ -0,0 +1,130 @@ +import { randomUUID } from "node:crypto"; +import { normalizedEventId } from "./event-epochs.js"; +import { expect, it, vi } from "vitest"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { applyNormalizedBatch, acknowledgeNormalizedEvent, type NormalizedDeliveryPort, type NormalizedDeliveryState } from "./normalized-delivery.js"; +import { NormalizedDeliveryWriter } from "./normalized-delivery-writer.js"; + +function event(sequence: number): PrpEvent { + return { schema: "paperclip.prp.event.v1", sourceEventId: `runner:run:${sequence}`, sourceSeq: sequence, sourceInstanceId: "runner", sourceKind: "runner", runId: "run", normalizedSessionId: "session", eventType: "harness.diagnostic", schemaVersion: 1, priority: 1, emittedAt: "2026-09-28T00:00:00.000Z", payload: { code: "test" } }; +} + +function fixture(initial: NormalizedDeliveryState | null = null) { + let current: NormalizedDeliveryState | null = initial; + const port: NormalizedDeliveryPort = { + epoch: "run", + load: () => structuredClone(current), + commit: vi.fn(async (batch) => current = applyNormalizedBatch(current, batch)), + acknowledge: async (value) => { current = acknowledgeNormalizedEvent(current!, value); }, + }; + const published: PrpEvent[] = []; + const failed = vi.fn(); + let reducer = { value: "before" }; + const writer = new NormalizedDeliveryWriter(port, () => structuredClone(reducer), (value) => published.push(value), failed); + return { port, writer, published, failed, setState: (value: string) => { reducer = { value }; } }; +} + +it("captures a whole synchronous reducer step and publishes only after durable success", async () => { + const { port, writer, published, setState } = fixture(); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + const commit = port.commit; + port.commit = async (batch) => { await gate; return commit(batch); }; + writer.emit({ ...event(1), payload: { code: "wire-optional", optional: undefined } }); + writer.emit(event(2)); + setState("after"); + writer.changed({ epoch: "run", sourceSeq: 1, ordinal: 1 }); + const done = writer.flush(); + expect(published).toEqual([]); + expect(port.load()).toBeNull(); + setState("later-uncommitted-step"); + release(); await done; + expect(port.load()).toMatchObject({ driver: { value: "after" }, raw: { epoch: "run", sourceSeq: 1, ordinal: 1 }, produced: 2 }); + expect(published[0]!.payload).toEqual({ code: "wire-optional" }); + expect(Object.hasOwn(published[0]!.payload, "optional")).toBe(false); + await writer.acknowledge(published[0]!); + const replayed: PrpEvent[] = []; + new NormalizedDeliveryWriter(port, () => ({}), (value) => replayed.push(value), () => {}); + expect(replayed).toEqual([event(2)]); +}); + +it("queues captured reducers across the old revision maximum using committed identities", async () => { + const initial: NormalizedDeliveryState = { schema: "paperclip.runner.normalized-delivery.v1", revision: Number.MAX_SAFE_INTEGER, + raw: { epoch: "run", sourceSeq: 0, ordinal: 0 }, driver: {}, produced: 0, acknowledged: 0, pending: [] }; + const { port, writer, published, setState, failed } = fixture(initial); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const commit = port.commit; + const before: Array = [], after: Array = []; + port.commit = async batch => { + await gate; + before.push(batch.expectedRevision); + const current = await commit(batch); after.push(current.revision); return current; + }; + writer.emit(event(1)); setState("first"); + await Promise.resolve(); // Capture while the first commit is still held. + writer.emit(event(2)); setState("second"); + const done = writer.flush(); + expect(published).toEqual([]); + release(); await done; + expect(before).toEqual([Number.MAX_SAFE_INTEGER, after[0]]); + expect(after[0]).toMatch(/^r:/); expect(after[1]).toMatch(/^r:/); + expect(after[0]).not.toBe(after[1]); + expect(port.load()).toMatchObject({ revision: after[1], driver: { value: "second" }, pending: [event(1), event(2)] }); + expect(failed).not.toHaveBeenCalled(); + await expect(port.commit({ expectedRevision: Number.MAX_SAFE_INTEGER, raw: initial.raw, driver: {}, events: [] })).rejects.toThrow("revision changed"); + await expect(port.commit({ expectedRevision: after[0]!, raw: initial.raw, driver: {}, events: [] })).rejects.toThrow("revision changed"); + await writer.acknowledge(event(1)); await writer.acknowledge(event(2)); + const recovered = new NormalizedDeliveryWriter(port, () => ({ recovered: true }), value => published.push(value), failed); + recovered.emit(event(3)); await recovered.flush(); + expect(published).toEqual([event(1), event(2), event(3)]); + expect(port.load()!.revision).not.toBe(after[1]); +}); + +it("fences a failed write without publishing it or emitting another diagnostic into the failed writer", async () => { + const { port, writer, published, failed } = fixture(); + port.commit = async () => { throw new Error("storage_unavailable: worker exited"); }; + writer.emit(event(1)); + await expect(writer.flush()).rejects.toThrow("worker exited"); + expect(port.load()).toBeNull(); + expect(published).toEqual([]); + expect(failed).toHaveBeenCalledTimes(1); + expect(() => writer.emit(event(2))).toThrow("worker exited"); + await expect(writer.waitForCapacity()).rejects.toThrow("worker exited"); +}); + +it("resumes admission after acknowledgements release pending capacity", async () => { + const { writer, published, port } = fixture(); + for (let sequence = 1; sequence <= 128; sequence++) writer.emit(event(sequence)); + await writer.flush(); + let admitted = false; + const waiting = writer.waitForCapacity().then(() => { admitted = true; }); + await Promise.resolve(); + expect(admitted).toBe(false); + await writer.acknowledge(published[0]!); + await waiting; + expect(admitted).toBe(true); + expect(port.load()!.pending).toHaveLength(127); +}); + +it("persists a pending queue across several source namespaces and acknowledges exact identities", async () => { + const initial: NormalizedDeliveryState = { schema: "paperclip.runner.normalized-delivery.v1", revision: 0, + raw: { epoch: "run", sourceSeq: 0, ordinal: 0 }, driver: {}, produced: Number.MAX_SAFE_INTEGER, acknowledged: Number.MAX_SAFE_INTEGER, pending: [] }; + const { port, writer, published } = fixture(initial); + let fromEpoch: string | undefined, final = Number.MAX_SAFE_INTEGER; + for (let round = 0; round < 3; round++) { + const nextEpoch = randomUUID(); + for (let seq = 1; seq <= 4; seq++) writer.emit({ ...event(seq), sourceEpoch: nextEpoch, sourceEventId: normalizedEventId("runner", "run", seq, nextEpoch), + ...(seq === 1 ? { sourceEpochTransition: { schema: "paperclip.prp.event-epoch.v1", runId: "run", transitionId: randomUUID(), fromEpoch: fromEpoch ?? null, nextEpoch, finalOrdinal: final } } : {}) }); + fromEpoch = nextEpoch; final = 4; + } + await writer.flush(); + expect(port.load()).toMatchObject({ produced: 4, producedEpoch: fromEpoch, acknowledged: Number.MAX_SAFE_INTEGER }); + const restored = fixture(port.load()); + expect(restored.published).toEqual(published); + await expect(restored.writer.acknowledge(published[4]!)).rejects.toThrow("differs"); + for (const value of published) await restored.writer.acknowledge(value); + expect(restored.port.load()).toMatchObject({ produced: 4, producedEpoch: fromEpoch, acknowledged: 4, acknowledgedEpoch: fromEpoch, pending: [] }); + expect(new Set(published.map(e => e.sourceEventId)).size).toBe(12); + await expect(restored.writer.acknowledge(published[0]!)).rejects.toThrow("differs"); +}); diff --git a/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.ts b/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.ts new file mode 100644 index 00000000000..95ac78f95bc --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/normalized-delivery-writer.ts @@ -0,0 +1,92 @@ +import type { PrpEvent } from "../protocol/replay-contract.js"; +import type { DriverHistoryReceipt, NormalizedDeliveryPort, NormalizedDeliveryRevision, RawDeliveryCursor } from "./normalized-delivery.js"; + +/** Serializes short reducer steps, not provider RPCs or waits for user input. + * A synchronous step captures its events and resulting state together. Nothing + * reaches the run-log consumer before that transaction is durable. */ +export class NormalizedDeliveryWriter { + #revision: NormalizedDeliveryRevision; + #raw: RawDeliveryCursor; + #events: PrpEvent[] = []; + #scheduled = false; + #tail: Promise = Promise.resolve(); + #failure: Error | null = null; + #queued = 0; + #capacityWaiters = new Set<() => void>(); + + constructor( + readonly port: NormalizedDeliveryPort, + private readonly snapshot: () => Record, + private readonly publish: (event: PrpEvent) => void, + private readonly fail: (error: Error) => void, + private readonly takeReceipts: () => DriverHistoryReceipt[] = () => [], + ) { + const restored = port.load(); + this.#revision = restored?.revision ?? 0; + this.#raw = restored?.raw ?? { epoch: port.epoch, sourceSeq: 0, ordinal: 0 }; + for (const event of restored?.pending ?? []) publish(structuredClone(event)); + } + + emit(event: PrpEvent): void { + if (this.#failure) throw this.#failure; + // The normalized event contract is JSON, matching the run-log boundary. + // Omitted optional fields must have identical bytes before and after replay. + this.#events.push(JSON.parse(JSON.stringify(event)) as PrpEvent); + this.changed(); + } + + get failure(): Error | null { return this.#failure; } + + changed(raw?: RawDeliveryCursor): void { + if (this.#failure) throw this.#failure; + if (raw) this.#raw = structuredClone(raw); + if (this.#scheduled) return; + this.#scheduled = true; + queueMicrotask(() => this.#capture()); + } + + #capture(): void { + if (!this.#scheduled || this.#failure) return; + this.#scheduled = false; + try { + if (++this.#queued > 64) throw new Error("storage_pressure: normalized reducer commit queue is full"); + const events = this.#events.splice(0); + const batch = { raw: structuredClone(this.#raw), driver: this.snapshot(), events, receipts: this.takeReceipts() }; + this.#tail = this.#tail.then(async () => { + if (this.#failure) throw this.#failure; + const committed = await this.port.commit({ ...batch, expectedRevision: this.#revision }); + this.#revision = committed.revision; + for (const event of events) this.publish(event); + }).catch((error: unknown) => this.#poison(error)).finally(() => { this.#queued--; }); + } catch (error) { this.#poison(error); } + } + + #poison(error: unknown): void { + if (this.#failure) return; + this.#failure = error instanceof Error ? error : new Error(String(error)); + for (const wake of this.#capacityWaiters) wake(); + this.#capacityWaiters.clear(); + this.fail(this.#failure); + } + + async acknowledge(event: PrpEvent): Promise { + await this.port.acknowledge(event); + for (const wake of this.#capacityWaiters) wake(); + this.#capacityWaiters.clear(); + } + + async waitForCapacity(): Promise { + for (;;) { + if (this.#failure) throw this.#failure; + const pending = this.port.load()?.pending ?? []; + if (pending.length < 128 && Buffer.byteLength(JSON.stringify(pending)) < 2 * 1024 * 1024) return; + await new Promise((resolve) => this.#capacityWaiters.add(resolve)); + } + } + + async flush(): Promise { + this.#capture(); + await this.#tail; + if (this.#failure) throw this.#failure; + } +} diff --git a/packages/paperclip-runner/src/control-plane/normalized-delivery.ts b/packages/paperclip-runner/src/control-plane/normalized-delivery.ts new file mode 100644 index 00000000000..5f43c18ee8d --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/normalized-delivery.ts @@ -0,0 +1,143 @@ +import { isCommandEpoch } from "./command-epochs.js"; +import { advanceSourceCursor, isEventEpochTransition, type EventEpochTransition, type SourceCursor } from "./event-epochs.js"; +import { randomUUID } from "node:crypto"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { validatePrpEvent } from "../protocol/replay-contract.js"; +import { authorityGeneration, authorityJson, DurableAuthorityStoreError } from "./durable-authority-store.js"; + +/** A raw frame can expand into several driver notifications. This cursor is + * independent of both the runner ACK and the normalized run-log sequence. */ +export interface RawDeliveryCursor { epoch: string; sourceEpoch?: string; sourceSeq: number; ordinal: number } +export const RAW_DELIVERY_FRAME_COMPLETE = Number.MAX_SAFE_INTEGER; +/** Retained checkpoints use safe integers. New revisions are equality-only + * identities, independent of raw and normalized event ordering. */ +export type NormalizedDeliveryRevision = number | string; + +export interface NormalizedDeliveryState { + schema: "paperclip.runner.normalized-delivery.v1"; + revision: NormalizedDeliveryRevision; + raw: RawDeliveryCursor; + driver: Record; + produced: number; + producedEpoch?: string; + acknowledged: number; + acknowledgedEpoch?: string; + lastSourceEpochTransition?: EventEpochTransition; + /** Only outstanding delivery, never completed history. */ + pending: PrpEvent[]; +} + +export interface NormalizedDeliveryBatch { + expectedRevision: NormalizedDeliveryRevision; + raw: RawDeliveryCursor; + driver: Record; + events: PrpEvent[]; + receipts?: DriverHistoryReceipt[]; +} + +export type DriverHistoryCollection = "terminal" | "file" | "steering" | "lineage"; +export interface DriverHistoryReceipt { + collection: DriverHistoryCollection; + key: string; + value: unknown; +} + +export interface NormalizedDeliveryPort { + readonly epoch: string; + readonly eventEpochs?: { limit: number }; + load(): NormalizedDeliveryState | null; + commit(batch: NormalizedDeliveryBatch): Promise; + acknowledge(event: PrpEvent): Promise; + history?: { get(collection: DriverHistoryCollection, key: string): Promise }; +} + +export const MAX_NORMALIZED_PENDING_EVENTS = 256; +export const MAX_NORMALIZED_PENDING_BYTES = 4 * 1024 * 1024; + +function invalid(message: string): never { + throw new DurableAuthorityStoreError("invalid_authority", `normalized delivery: ${message}`); +} + +export function compareRawCursor(a: RawDeliveryCursor, b: RawDeliveryCursor): number { + if (a.epoch !== b.epoch || a.sourceEpoch !== b.sourceEpoch) invalid("raw cursor namespaces cannot be numerically compared"); + return a.sourceSeq - b.sourceSeq || a.ordinal - b.ordinal; +} + +export function validateNormalizedDelivery(value: NormalizedDeliveryState): void { + if (typeof value.revision === "number") { + if (!Number.isSafeInteger(value.revision) || value.revision < 0) invalid("invalid checkpoint revision"); + } else if (typeof value.revision !== "string" || !value.revision.startsWith("r:")) invalid("invalid checkpoint revision"); + else authorityGeneration(value.revision); + if (value.schema !== "paperclip.runner.normalized-delivery.v1" + || typeof value.raw?.epoch !== "string" || !value.raw.epoch || value.raw.epoch.length > 240 + || (value.raw.sourceEpoch !== undefined && !isCommandEpoch(value.raw.sourceEpoch)) + || ![value.raw?.sourceSeq, value.raw?.ordinal, value.produced, value.acknowledged].every((n) => Number.isSafeInteger(n) && n >= 0) + || (value.producedEpoch !== undefined && !isCommandEpoch(value.producedEpoch)) + || (value.acknowledgedEpoch !== undefined && !isCommandEpoch(value.acknowledgedEpoch)) + || !Array.isArray(value.pending) + || !value.driver || typeof value.driver !== "object" || Array.isArray(value.driver)) invalid("invalid checkpoint"); + if (value.pending.length > MAX_NORMALIZED_PENDING_EVENTS || Buffer.byteLength(authorityJson(value.pending)) > MAX_NORMALIZED_PENDING_BYTES) { + throw new DurableAuthorityStoreError("storage_pressure", "normalized delivery awaiting run-log acknowledgement"); + } + if (value.producedEpoch !== undefined || value.lastSourceEpochTransition !== undefined) { + const t = value.lastSourceEpochTransition; + if (!isEventEpochTransition(t) || t.nextEpoch !== value.producedEpoch || t.runId !== value.raw.epoch) invalid("invalid normalized head receipt"); + } + let cursor: SourceCursor = { sourceEpoch: value.acknowledgedEpoch, sourceSeq: value.acknowledged }; + for (const event of value.pending) { + if (!validatePrpEvent(event).ok || event.runId !== value.raw.epoch) invalid("invalid pending event"); + try { cursor = advanceSourceCursor(cursor, event); } catch { invalid("pending event sequence is not contiguous"); } + } + if (cursor.sourceSeq !== value.produced || cursor.sourceEpoch !== value.producedEpoch) invalid("missing pending event"); +} + +/** Build a candidate without changing published state. The owner commits it + * with the raw cursor and only then exposes its events to the consumer. */ +export function applyNormalizedBatch(previous: NormalizedDeliveryState | null, batch: NormalizedDeliveryBatch, rawTransition?: EventEpochTransition): NormalizedDeliveryState { + if (previous) validateNormalizedDelivery(previous); + if (batch.expectedRevision !== (previous?.revision ?? 0)) { + throw new DurableAuthorityStoreError("stale_authority", "normalized consumer revision changed"); + } + if (previous && batch.raw.epoch !== previous.raw.epoch) invalid("raw cursor belongs to another run epoch"); + const crossing = previous?.raw.sourceEpoch !== batch.raw.sourceEpoch; + if (crossing) { + if (!previous || !isEventEpochTransition(rawTransition) || rawTransition.runId !== batch.raw.epoch + || rawTransition.fromEpoch !== (previous.raw.sourceEpoch ?? null) || rawTransition.nextEpoch !== batch.raw.sourceEpoch + || previous.raw.sourceSeq !== rawTransition.finalOrdinal || previous.raw.ordinal !== RAW_DELIVERY_FRAME_COMPLETE + || batch.raw.sourceSeq !== 0 || batch.raw.ordinal !== 0 || batch.events.length !== 0) invalid("raw epoch boundary has no exact consumed predecessor"); + } + if (!crossing && compareRawCursor(batch.raw, previous?.raw ?? { epoch: batch.raw.epoch, sourceSeq: 0, ordinal: 0 }) < 0) invalid(`raw cursor moved backwards (${previous!.raw.sourceSeq}:${previous!.raw.ordinal} -> ${batch.raw.sourceSeq}:${batch.raw.ordinal})`); + const initialSequence = previous?.produced ?? (batch.events[0]?.sourceSeq ?? 1) - 1; + let cursor: SourceCursor = { sourceSeq: initialSequence, sourceEpoch: previous?.producedEpoch }; + let lastTransition = previous?.lastSourceEpochTransition; + for (const event of batch.events) { + try { cursor = advanceSourceCursor(cursor, event); } catch { invalid("produced event sequence is not contiguous"); } + if (event.sourceEpochTransition) lastTransition = event.sourceEpochTransition; + } + const state: NormalizedDeliveryState = { + schema: "paperclip.runner.normalized-delivery.v1", + revision: `r:${randomUUID()}`, + raw: structuredClone(batch.raw), + driver: structuredClone(batch.driver), + produced: cursor.sourceSeq, + ...(cursor.sourceEpoch ? { producedEpoch: cursor.sourceEpoch } : {}), + ...(previous?.acknowledgedEpoch ? { acknowledgedEpoch: previous.acknowledgedEpoch } : {}), + ...(lastTransition ? { lastSourceEpochTransition: structuredClone(lastTransition) } : {}), + acknowledged: previous?.acknowledged ?? initialSequence, + pending: [...(previous?.pending ?? []), ...structuredClone(batch.events)], + }; + validateNormalizedDelivery(state); + return state; +} + +/** ACKs are accepted only for the exact next event, after the run-log writer + * reports durable success. A crash before this commit replays identical bytes. */ +export function acknowledgeNormalizedEvent(previous: NormalizedDeliveryState, event: PrpEvent): NormalizedDeliveryState { + const next = previous.pending[0]; + if (!next || authorityJson(next) !== authorityJson(event)) invalid("run-log acknowledgement differs from pending event"); + const state = { ...previous, acknowledged: event.sourceSeq, pending: previous.pending.slice(1) }; + if (event.sourceEpoch) state.acknowledgedEpoch = event.sourceEpoch; + else delete state.acknowledgedEpoch; + validateNormalizedDelivery(state); + return state; +} diff --git a/packages/paperclip-runner/src/control-plane/normalized-event-epochs.test.ts b/packages/paperclip-runner/src/control-plane/normalized-event-epochs.test.ts new file mode 100644 index 00000000000..ebaf25eacab --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/normalized-event-epochs.test.ts @@ -0,0 +1,51 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "./durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "./sqlite-authority-store.js"; +import { normalizedEpochCloseId, normalizedEventId } from "./event-epochs.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; + +it("reopens bounded normalized heads across numeric exhaustion, keeps exact closes and refuses reused namespaces", async () => { + const root = await mkdtemp(join(tmpdir(), "normalized-epochs-")); + const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", runId: "run", turnId: "turn", itemId: "item" }; + const storage = { path: join(root, "authority.sqlite"), binding: "normalized-epochs", create: true }; + let store = await SqliteAuthorityStore.open(storage); + const options = { stateDirectory: root, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + const event = (sourceSeq: number, sourceEpoch?: string): PrpEvent => ({ schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceEventId: normalizedEventId("runner", "run", sourceSeq, sourceEpoch), + sourceSeq, ...(sourceEpoch ? { sourceEpoch } : {}), sourceInstanceId: "runner", sourceKind: "runner", runId: "run", normalizedSessionId: "session", eventType: "harness.diagnostic", priority: 1, + emittedAt: "2026-09-29T00:00:00.000Z", payload: { code: "epoch-test" } }); + const epochs = [randomUUID(), randomUUID(), randomUUID()]; + const raw = { epoch: "run", sourceSeq: 0, ordinal: 0 }; + try { + let port = core.normalizedDelivery()!; + expect(port.eventEpochs?.limit).toBe(1_048_576); + const lastLegacy = event(Number.MAX_SAFE_INTEGER); + await port.commit({ expectedRevision: 0, raw, driver: {}, events: [lastLegacy] }); + const pending = [lastLegacy]; + let previous: string | null = null, finalOrdinal = Number.MAX_SAFE_INTEGER; + for (const nextEpoch of epochs) { + const transition = { schema: "paperclip.prp.event-epoch.v1" as const, runId: "run", transitionId: randomUUID(), fromEpoch: previous, nextEpoch, finalOrdinal }; + const events = [1, 2, 3, 4].map(n => ({ ...event(n, nextEpoch), ...(n === 1 ? { sourceEpochTransition: transition } : {}) })); + await port.commit({ expectedRevision: port.load()!.revision, raw, driver: { current: nextEpoch }, events }); pending.push(...events); + expect(await store.getRecord("run", "effect", normalizedEpochCloseId("run", "runner", previous))).toMatchObject({ sequence: "0", body: transition }); + await core.stop(); await store.close(); store = await SqliteAuthorityStore.open({ ...storage, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); port = core.normalizedDelivery()!; + expect(port.load()!.pending).toEqual(pending); + previous = nextEpoch; finalOrdinal = 4; + } + const before = port.load(); + await expect(port.commit({ expectedRevision: before!.revision, raw, driver: {}, events: [{ ...event(1, epochs[0]), sourceEpochTransition: { + schema: "paperclip.prp.event-epoch.v1", runId: "run", transitionId: randomUUID(), fromEpoch: epochs[2]!, nextEpoch: epochs[0]!, finalOrdinal: 4 } }] })).rejects.toThrow("reused"); + expect(port.load()).toEqual(before); + for (const value of pending) await port.acknowledge(value); + expect(port.load()).toMatchObject({ producedEpoch: epochs[2], acknowledgedEpoch: epochs[2], produced: 4, acknowledged: 4, pending: [] }); + await core.stop(); + const get = store.getRecord.bind(store); + vi.spyOn(store, "getRecord").mockImplementation(async (epoch, kind, id) => id.startsWith("normalized-epoch-from-") ? null : get(epoch, kind, id)); + await expect(DurablePrpControlPlane.open({ ...options, authorityStore: store })).rejects.toThrow("current recovery evidence"); + } finally { await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/src/control-plane/process-generation.test.ts b/packages/paperclip-runner/src/control-plane/process-generation.test.ts new file mode 100644 index 00000000000..d6e8379117f --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/process-generation.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { + isProviderGenerationSuccessor, + isProviderProcessGeneration, + provesProviderGenerationTransition, +} from "./process-generation.js"; + +const maxLegacy = Number.MAX_SAFE_INTEGER; +const firstOpaque = "p:00000000-0000-4000-8000-000000000001"; +const nextOpaque = "p:00000000-0000-4000-8000-000000000002"; +const launchId = "00000000-0000-4000-8000-000000000010"; + +describe("provider process generation", () => { + it("switches from the exact legacy ceiling to distinct opaque lifetimes", () => { + expect(isProviderProcessGeneration(maxLegacy)).toBe(true); + expect(isProviderProcessGeneration(firstOpaque)).toBe(true); + expect(isProviderGenerationSuccessor(maxLegacy - 1, maxLegacy)).toBe(true); + expect(isProviderGenerationSuccessor(maxLegacy, firstOpaque)).toBe(true); + expect(isProviderGenerationSuccessor(firstOpaque, nextOpaque)).toBe(true); + expect(isProviderGenerationSuccessor(firstOpaque, firstOpaque)).toBe(false); + expect(isProviderGenerationSuccessor(firstOpaque, 1)).toBe(false); + expect(isProviderGenerationSuccessor(maxLegacy, 1)).toBe(false); + }); + + it("requires a canonical opaque UUID token and exact launch transition", () => { + for (const malformed of [ + "p:00000000-0000-0000-8000-000000000001", + "p:00000000-0000-4000-7000-000000000001", + "p:00000000-0000-4000-8000-00000000000Z", + "p:00000000-0000-4000-8000-000000000001 ", + ]) { + expect(isProviderProcessGeneration(malformed)).toBe(false); + expect(isProviderGenerationSuccessor(maxLegacy, malformed)).toBe(false); + } + + const transition = { from: maxLegacy, to: firstOpaque, launchId }; + expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, transition)).toBe(true); + expect(provesProviderGenerationTransition(maxLegacy - 1, firstOpaque, transition)).toBe(false); + expect(provesProviderGenerationTransition(maxLegacy, nextOpaque, transition)).toBe(false); + expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, { ...transition, launchId: "bad" })).toBe(false); + expect(provesProviderGenerationTransition(maxLegacy, firstOpaque, { ...transition, extra: true })).toBe(false); + expect(provesProviderGenerationTransition(firstOpaque, 1, { from: firstOpaque, to: 1, launchId })).toBe(false); + }); +}); diff --git a/packages/paperclip-runner/src/control-plane/process-generation.ts b/packages/paperclip-runner/src/control-plane/process-generation.ts new file mode 100644 index 00000000000..df01905fe03 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/process-generation.ts @@ -0,0 +1,21 @@ +/** Process lifetimes use equality; retained small integers preserve old bytes. */ +export type ProviderProcessGeneration = number | string; +export function isProviderProcessGeneration(value: unknown, allowZero = false): value is ProviderProcessGeneration { + return (typeof value === "number" && Number.isSafeInteger(value) && value >= (allowZero ? 0 : 1)) + || (typeof value === "string" && /^p:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value)); +} +export function isProviderGenerationSuccessor(previous: unknown, next: unknown): boolean { + if (!isProviderProcessGeneration(previous, true) || !isProviderProcessGeneration(next)) return false; + if (typeof previous === "number" && previous < Number.MAX_SAFE_INTEGER) return next === previous + 1; + return typeof next === "string" && next !== previous; +} +/** Numeric successors are retained compatibility. An opaque successor must + * carry the exact predecessor/startup binding committed by the native owner. */ +export function provesProviderGenerationTransition(previous: unknown, next: unknown, value: unknown): boolean { + if (!isProviderGenerationSuccessor(previous, next)) return false; + if (typeof next === "number") return true; + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const transition = value as Record; + return Object.keys(transition).length === 3 && transition.from === previous && transition.to === next + && typeof transition.launchId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(transition.launchId); +} diff --git a/packages/paperclip-runner/src/control-plane/process-owner-evidence.test.ts b/packages/paperclip-runner/src/control-plane/process-owner-evidence.test.ts new file mode 100644 index 00000000000..9848b1519fb --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/process-owner-evidence.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + DurableAuthorityStoreError, + type DurableAuthorityStore, +} from "./durable-authority-store.js"; +import { eventEpochCloseId } from "./event-epochs.js"; +import type { + DurableRecoveryCommittedEvent, + DurableRecoveryIdentity, +} from "./prp-transport-types.js"; +import { + processOwnerChanges, + processOwnerEvidence, + type ProcessOwnerEvidence, +} from "./process-owner-evidence.js"; + +const identity: DurableRecoveryIdentity = { + runnerInstanceId: "runner-owner-test", + environmentLeaseId: "lease-owner-test", + runId: "run-owner-test", + normalizedSessionId: "session-owner-test", + turnId: "turn-owner-test", + itemId: "item-owner-test", +}; +const previousEpoch = "00000000-0000-4000-8000-000000000101"; +const nextEpoch = "00000000-0000-4000-8000-000000000102"; +const launchId = "00000000-0000-4000-8000-000000000103"; +const opaqueGeneration = "p:00000000-0000-4000-8000-000000000104"; + +function event(options: { + phase?: "intent" | "spawned" | "initialization_failed"; + sourceSeq?: number; + sourceEpoch?: string; + launch?: string; + previousGeneration?: number; + attemptedGeneration?: number | string; + owner?: DurableRecoveryIdentity; +} = {}): DurableRecoveryCommittedEvent { + const phase = options.phase ?? "intent"; + const owner = options.owner ?? identity; + const startup = { + schema: "paperclip.provider_startup.v1", + launchId: options.launch ?? launchId, + phase, + attemptedProcessGeneration: options.attemptedGeneration ?? 1, + ...(options.previousGeneration !== undefined + ? { previousProcessGeneration: options.previousGeneration } + : {}), + configurationFingerprint: `sha256:${"a".repeat(64)}`, + origin: { + runnerInstanceId: owner.runnerInstanceId, + runId: owner.runId, + normalizedSessionId: owner.normalizedSessionId, + turnId: owner.turnId, + itemId: owner.itemId, + }, + ...(phase === "spawned" ? { processId: 1200, processGroupId: 1200 } : {}), + processTreeRetired: false, + }; + return { + sourceSeq: options.sourceSeq ?? 1, + sourceEventId: `event-${options.sourceSeq ?? 1}`, + ...(options.sourceEpoch ? { sourceEpoch: options.sourceEpoch } : {}), + eventType: "harness.diagnostic", + priority: 1, + envelope: { + payload: { + payload: { code: "provider_startup_ownership", startup }, + }, + }, + deliveryCount: 1, + logicalEffectCount: 1, + }; +} + +function ownerFact(options: Parameters[0] = {}): ProcessOwnerEvidence { + return processOwnerEvidence(event(options), identity)!; +} + +function authorityWithOwner( + prior: ProcessOwnerEvidence, + receiptBody: Record, +) { + const getWork = vi.fn(async () => ({ + collection: "process-owner" as const, + id: launchId, + body: prior as unknown as Record, + sha256: "a".repeat(64), + })); + const getRecord = vi.fn(async () => ({ + epoch: identity.runId, + kind: "effect" as const, + id: eventEpochCloseId({ runId: identity.runId, fromEpoch: previousEpoch }), + sequence: "0", + body: receiptBody, + })); + return { + authority: { getWork, getRecord } as unknown as DurableAuthorityStore, + getWork, + getRecord, + }; +} + +function validCloseReceipt(): Record { + return { + schema: "paperclip.prp.event-epoch.v1", + runId: identity.runId, + transitionId: "00000000-0000-4000-8000-000000000105", + fromEpoch: previousEpoch, + nextEpoch, + finalOrdinal: 2, + }; +} + +describe("provider process owner evidence", () => { + it("accepts opaque generation startup only with its predecessor and compatible launch UUIDs", () => { + const opaque = processOwnerEvidence( + event({ + attemptedGeneration: opaqueGeneration, + previousGeneration: Number.MAX_SAFE_INTEGER, + }), + identity, + ); + expect(opaque?.startup).toMatchObject({ + attemptedProcessGeneration: opaqueGeneration, + previousProcessGeneration: Number.MAX_SAFE_INTEGER, + }); + + // Retained startup launch IDs are UUID-shaped identities, not process + // generation tokens; accept a valid older UUID version as native parsing does. + expect( + processOwnerEvidence( + event({ launch: "00000000-0000-1000-8000-000000000103" }), + identity, + ), + ).not.toBeNull(); + expect(() => + processOwnerEvidence( + event({ attemptedGeneration: opaqueGeneration }), + identity, + ), + ).toThrow(DurableAuthorityStoreError); + expect(() => + processOwnerEvidence( + event({ attemptedGeneration: "p:malformed", previousGeneration: 1 }), + identity, + ), + ).toThrow(DurableAuthorityStoreError); + }); + + it("rejects evidence whose startup origin belongs to a different task identity", () => { + for (const key of [ + "runnerInstanceId", + "runId", + "normalizedSessionId", + "turnId", + "itemId", + ] as const) { + const foreign = { ...identity, [key]: `other-${identity[key]}` }; + expect(() => processOwnerEvidence(event({ owner: foreign }), identity)).toThrow( + DurableAuthorityStoreError, + ); + } + }); + + it("rejects advancement that changes the launch or its recorded predecessor", async () => { + const prior = ownerFact({ + phase: "intent", + attemptedGeneration: opaqueGeneration, + previousGeneration: Number.MAX_SAFE_INTEGER, + }); + const differentLaunch = ownerFact({ + phase: "spawned", + attemptedGeneration: opaqueGeneration, + previousGeneration: Number.MAX_SAFE_INTEGER, + launch: "00000000-0000-4000-8000-000000000106", + }); + const validSuccessor = ownerFact({ + phase: "spawned", + attemptedGeneration: opaqueGeneration, + previousGeneration: Number.MAX_SAFE_INTEGER, + }); + const differentPredecessor: ProcessOwnerEvidence = { + ...validSuccessor, + startup: { + ...validSuccessor.startup, + previousProcessGeneration: Number.MAX_SAFE_INTEGER - 1, + }, + }; + const { authority } = authorityWithOwner(prior, validCloseReceipt()); + + await expect( + processOwnerChanges(authority, [differentLaunch], { + runId: identity.runId, + }), + ).rejects.toMatchObject({ code: "invalid_authority" }); + await expect( + processOwnerChanges(authority, [differentPredecessor], { + runId: identity.runId, + }), + ).rejects.toMatchObject({ code: "invalid_authority" }); + }); + + it("advances maintenance ownership only through the current run's exact raw epoch close", async () => { + const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch }); + const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch }); + const { authority, getRecord } = authorityWithOwner(prior, validCloseReceipt()); + + const changes = await processOwnerChanges(authority, [next], { + runId: identity.runId, + sourceEpoch: nextEpoch, + }); + + expect(changes).toHaveLength(1); + expect(changes[0]).toMatchObject({ + collection: "process-owner", + id: launchId, + expectedSha256: "a".repeat(64), + body: next, + }); + expect(getRecord).toHaveBeenCalledWith( + identity.runId, + "effect", + eventEpochCloseId({ runId: identity.runId, fromEpoch: previousEpoch }), + ); + }); + + it("rejects a rollover backed by a close receipt from another raw run", async () => { + const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch }); + const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch }); + const receipt = { ...validCloseReceipt(), runId: "other-run" }; + const { authority } = authorityWithOwner(prior, receipt); + + await expect( + processOwnerChanges(authority, [next], { + runId: identity.runId, + sourceEpoch: nextEpoch, + }), + ).rejects.toMatchObject({ code: "invalid_authority" }); + }); + + it("rejects maintenance facts outside the current raw run and epoch", async () => { + const prior = ownerFact({ phase: "intent", sourceSeq: 2, sourceEpoch: previousEpoch }); + const next = ownerFact({ phase: "spawned", sourceSeq: 1, sourceEpoch: nextEpoch }); + const { authority, getRecord } = authorityWithOwner(prior, validCloseReceipt()); + + await expect( + processOwnerChanges(authority, [next], { + runId: "another-run", + sourceEpoch: nextEpoch, + }), + ).rejects.toMatchObject({ code: "invalid_authority" }); + await expect( + processOwnerChanges(authority, [next], { + runId: identity.runId, + sourceEpoch: previousEpoch, + }), + ).rejects.toMatchObject({ code: "invalid_authority" }); + expect(getRecord).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/paperclip-runner/src/control-plane/process-owner-evidence.ts b/packages/paperclip-runner/src/control-plane/process-owner-evidence.ts new file mode 100644 index 00000000000..b24d2be775a --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/process-owner-evidence.ts @@ -0,0 +1,89 @@ +import { isProviderProcessGeneration, isProviderGenerationSuccessor } from "./process-generation.js"; +import { eventEpochCloseId, isEventEpochTransition } from "./event-epochs.js"; +import { authorityJson, DurableAuthorityStoreError, type AuthorityWorkChange, type DurableAuthorityStore } from "./durable-authority-store.js"; +import type { DurableRecoveryCommittedEvent, DurableRecoveryIdentity } from "./prp-transport-types.js"; + +export interface ProcessOwnerEvidence { + schema: "paperclip.process-owner.v1"; + identity: DurableRecoveryIdentity; + sourceEventId: string; + sourceSeq: number; + sourceEpoch?: string; + startup: Record; +} +const object = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +function invalid(): never { throw new DurableAuthorityStoreError("invalid_authority", "provider process ownership evidence is invalid or conflicts with its launch"); } + +/** A diagnostic is authoritative only after authenticated PRP validation. The + * relation retains unresolved launch facts, independently of transcript windows. + * Direct-child exit is deliberately not accepted as process-tree retirement. */ +export function processOwnerEvidence(event: DurableRecoveryCommittedEvent, identity: DurableRecoveryIdentity): ProcessOwnerEvidence | null { + const wire = event.envelope.payload; + const body = object(wire) && object(wire.payload) ? wire.payload : {}; + if (event.eventType !== "harness.diagnostic" || body.code !== "provider_startup_ownership") return null; + const startup = body.startup; + if (!object(startup) || startup.schema !== "paperclip.provider_startup.v1" || + typeof startup.launchId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(startup.launchId) || + !["intent", "spawned", "initialization_failed"].includes(String(startup.phase)) || + !isProviderProcessGeneration(startup.attemptedProcessGeneration) || + (typeof startup.attemptedProcessGeneration === "string" && !isProviderGenerationSuccessor(startup.previousProcessGeneration, startup.attemptedProcessGeneration)) || + typeof startup.configurationFingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(startup.configurationFingerprint) || + startup.processTreeRetired !== false || !object(startup.origin) || ["runnerInstanceId", "runId", "normalizedSessionId", "turnId", "itemId"].some(key => (startup.origin as Record)[key] !== identity[key as keyof DurableRecoveryIdentity])) invalid(); + const hasProcess = Number.isSafeInteger(startup.processId) && Number(startup.processId) > 0 && startup.processId === startup.processGroupId; + if (startup.phase === "intent" ? startup.processId != null || startup.processGroupId != null : + startup.failedStage === "spawn" ? startup.processId != null || startup.processGroupId != null : !hasProcess) invalid(); + return { schema: "paperclip.process-owner.v1", identity: structuredClone(identity), sourceEventId: event.sourceEventId, sourceSeq: event.sourceSeq, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), startup: structuredClone(startup) }; +} + +/** At most one page of new facts; repeated phases for a launch collapse in the + * transaction, while every original event is still an immutable receipt. */ +export async function processOwnerChanges(authority: DurableAuthorityStore, facts: readonly ProcessOwnerEvidence[], current: { runId: string; sourceEpoch?: string }): Promise { + // Collapse the source page before consulting the index. On interrupted + // backfill the index may already contain a later phase from this same page; + // comparing its first intent against that phase would reject a safe replay. + const latest = new Map(); + for (const fact of facts) { + const id = String(fact.startup.launchId); + const prior = latest.get(id); + if (prior) validateAdvance(prior, fact); + latest.set(id, fact); + } + const changes = new Map(); + for (const fact of latest.values()) { + const id = String(fact.startup.launchId); + const pending = changes.get(id); + const stored = pending ? null : await authority.getWork("process-owner", id); + const prior = (pending?.body ?? stored?.body) as unknown as ProcessOwnerEvidence | undefined; + if (prior) { + validateAdvance(prior, fact); + if (prior.sourceEpoch !== fact.sourceEpoch) { + if (fact.identity.runId !== current.runId || fact.sourceEpoch !== current.sourceEpoch) invalid(); + const receipt = await authority.getRecord(prior.identity.runId, "effect", eventEpochCloseId({ runId: prior.identity.runId, fromEpoch: prior.sourceEpoch ?? null })); + if (!receipt || !isEventEpochTransition(receipt.body) || receipt.body.runId !== prior.identity.runId || receipt.body.fromEpoch !== (prior.sourceEpoch ?? null) + || receipt.body.finalOrdinal < prior.sourceSeq) invalid(); + } + if (authorityJson(prior) === authorityJson(fact)) continue; + } + // An authenticated failed spawn proves that this launch created no child. + // Remove its pending intent in the same transaction as the immutable event + // receipt. A child exit after spawn does not prove a stopped process tree. + if (fact.startup.phase === "initialization_failed" && fact.startup.failedStage === "spawn") { + if (stored) changes.set(id, { collection: "process-owner", id, expectedSha256: stored.sha256, body: null }); + continue; + } + changes.set(id, { collection: "process-owner", id, expectedSha256: pending?.expectedSha256 ?? stored?.sha256 ?? null, body: fact as unknown as Record }); + } + return [...changes.values()]; +} + +function validateAdvance(prior: ProcessOwnerEvidence, fact: ProcessOwnerEvidence): void { + if (prior.schema !== fact.schema || authorityJson(prior.identity) !== authorityJson(fact.identity) || + prior.startup.launchId !== fact.startup.launchId || + prior.startup.previousProcessGeneration !== fact.startup.previousProcessGeneration || + prior.startup.configurationFingerprint !== fact.startup.configurationFingerprint || + prior.startup.attemptedProcessGeneration !== fact.startup.attemptedProcessGeneration || + (prior.startup.processId != null && prior.startup.processId !== fact.startup.processId) || + (prior.sourceEpoch === fact.sourceEpoch && (prior.sourceSeq > fact.sourceSeq || (prior.sourceSeq === fact.sourceSeq && authorityJson(prior) !== authorityJson(fact))))) invalid(); + const phase = ["intent", "spawned", "initialization_failed"]; + if (phase.indexOf(String(prior.startup.phase)) > phase.indexOf(String(fact.startup.phase))) invalid(); +} diff --git a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts index e90b1529d81..925bd271532 100644 --- a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts +++ b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts @@ -18,6 +18,7 @@ export interface DurableWarmRunTransition { commandFingerprint: string; resultDigest: string; oldAckedSourceSeq: number; + oldSourceEpoch?: string; connection: Record; runnerVersion: string; runnerDigest: string; @@ -30,6 +31,7 @@ export interface DurableRecoveryCoreCommand { schema: "paperclip.prp.command.v1" | "paperclip.prp.command.v2"; commandId: string; controllerSeq: number; + controllerEpoch?: string; type: string; issuedAt: string; payload: Record; @@ -44,6 +46,7 @@ export interface DurableRecoveryCoreCommand { } export interface DurableRecoveryCommittedEvent { + sourceEpoch?: string; sourceSeq: number; sourceEventId: string; eventType: string; diff --git a/packages/paperclip-runner/src/control-plane/sqlite-authority-store.test.ts b/packages/paperclip-runner/src/control-plane/sqlite-authority-store.test.ts new file mode 100644 index 00000000000..72f903f0897 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/sqlite-authority-store.test.ts @@ -0,0 +1,234 @@ +import { mkdtemp, rm, chmod } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { execFile } from "node:child_process"; +import { randomUUID, createHash } from "node:crypto"; +import { afterEach, expect, it } from "vitest"; +import { promisify } from "node:util"; +import { SqliteAuthorityStore } from "./sqlite-authority-store.js"; +import { authorityInteger, type AuthorityRecord } from "./durable-authority-store.js"; +import { defaultStorageRunnerBinary } from "./native-storage-command.js"; + +const execute = promisify(execFile); + +const stores: SqliteAuthorityStore[] = []; +const roots: string[] = []; +afterEach(async () => { + for (const store of stores.splice(0)) await store.close(); + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "runner-authority-")); + roots.push(root); + await chmod(root, 0o700); + const input = { path: join(root, "authority.sqlite"), binding: "company:session:runner:environment", create: true }; + const store = await SqliteAuthorityStore.open(input); + stores.push(store); + return { store, input }; +} +const record = (id: string, sequence: string, text = id): AuthorityRecord => ({ epoch: "run-1", kind: "event", id, sequence, body: { text } }); + +it("retains unordered effect identities without consuming a lifetime sequence", async () => { + const { store, input } = await fixture(); + expect(store.unorderedEffectReceipts).toBe(true); + const old = { ...record("old-effect", "9223372036854775807"), kind: "effect" as const }; + const first = { ...record("new-effect-1", "0"), kind: "effect" as const }; + const second = { ...record("new-effect-2", "0"), kind: "effect" as const }; + const generation = await store.commit({ expectedGeneration: "0", state: { settled: true }, records: [old, first, second] }); + await store.close(); + const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened); + for (const receipt of [old, first, second]) { + expect(await reopened.getSessionEffect(receipt.id)).toEqual(receipt); + expect(await reopened.getRecord("run-1", "effect", receipt.id)).toEqual(receipt); + } + for (const conflicting of [{ ...first, body: { text: "changed" } }, { ...old, sequence: "0" }, { ...first, epoch: "run-2" }]) { + await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [conflicting] })).rejects.toThrow("receipt_conflict"); + } + expect(await reopened.load()).toEqual({ generation, state: { settled: true } }); +}); + +it("commits current authority with exact receipts, and rejects stale writers", async () => { + const { store, input } = await fixture(); + expect(await store.load()).toBeNull(); + await store.commit({ expectedGeneration: "0", state: { cursor: "1" }, records: [record("first", "1")] }); + await expect(store.commit({ expectedGeneration: "1", state: { cursor: "2" }, records: [record("second", "2"), record("first", "1", "conflicting")] })).rejects.toThrow("receipt_conflict"); + await expect(store.commit({ expectedGeneration: "1", state: { cursor: "2" }, records: [record("alias", "1")] })).rejects.toThrow("receipt_conflict"); + expect(await store.getRecord("run-1", "event", "second")).toBeNull(); + expect(await store.load()).toEqual({ generation: "1", state: { cursor: "1" } }); + await expect(store.commit({ expectedGeneration: "0", state: {}, records: [] })).rejects.toThrow("stale_authority"); + await store.close(); + const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); + stores.push(reopened); + expect(await reopened.getRecord("run-1", "event", "first")).toEqual(record("first", "1")); + expect(await reopened.getRecord("run-2", "event", "first")).toBeNull(); + const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); + stores.push(proof); + expect(await proof.load()).toEqual(await reopened.load()); + await expect(proof.commit({ expectedGeneration: "1", state: {}, records: [] })).rejects.toThrow("proof readers cannot commit"); + await expect(SqliteAuthorityStore.open({ ...input, binding: "wrong-owner", create: false })).rejects.toThrow("binding"); +}); + +it("keeps exact integers above Number.MAX_SAFE_INTEGER and pages by bytes", async () => { + const { store } = await fixture(); + const first = "9007199254740993", next = "9007199254740994"; + const payload = "x".repeat(600_000); + await store.commit({ expectedGeneration: "0", state: { cursor: next }, records: [record("first", first, payload), record("next", next, payload)] }); + const page = await store.readEvents("run-1", "0", 128, 1024 * 1024); + expect(page.records.map((entry) => entry.sequence)).toEqual([first]); + expect((await store.readEvents("run-1", page.nextAfter!, 128, 1024 * 1024)).records.map((entry) => entry.sequence)).toEqual([next]); + expect(authorityInteger(first)).toBe(9007199254740993n); + expect(() => authorityInteger("9007199254740993.0")).toThrow(); +}); + +it("continues past the native revision boundary with exact proof and outstanding-work reads", async () => { + const { store, input } = await fixture(); + await store.commit({ expectedGeneration: "0", state: { cursor: "1" }, records: [record("first", "1")], work: [ + { collection: "process-owner", id: "owner", expectedSha256: null, body: { phase: "spawned" } }, + ] }); + await store.close(); + const db = new DatabaseSync(input.path); + try { db.prepare("UPDATE current_state SET generation=? WHERE key='authority'").run(9223372036854775807n); } + finally { db.close(); } + const active = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(active); + const first = await active.commit({ expectedGeneration: "9223372036854775807", state: { cursor: "2" }, records: [record("second", "2")] }); + expect(first).toMatch(/^r:/); + const second = await active.commit({ expectedGeneration: first, state: { cursor: "3" }, records: [] }); + expect(second).not.toBe(first); + await expect(active.commit({ expectedGeneration: first, state: { stale: true }, records: [] })).rejects.toThrow("stale_authority"); + const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(proof); + expect(await proof.load()).toEqual({ generation: second, state: { cursor: "3" } }); + expect(await proof.load()).toEqual(await active.load()); + expect(await proof.getRecord("run-1", "event", "first")).toEqual(record("first", "1")); + expect((await proof.readWorkPage("process-owner", "", 128, second)).records.map(row => row.id)).toEqual(["owner"]); + await expect(proof.readWorkPage("process-owner", "", 128, first)).rejects.toThrow("stale_authority"); + expect((await active.readEvents("run-1", "0", 128, 1024 * 1024)).records).toEqual([record("first", "1"), record("second", "2")]); +}); + +it("does not recreate a missing activated store", async () => { + const { store, input } = await fixture(); + await store.close(); + await rm(input.path); + await expect(SqliteAuthorityStore.open({ ...input, create: false })).rejects.toThrow("storage_unavailable"); +}); + +it("partitions standalone history and preserves exact event/effect indexes after reopening", async () => { + const { store, input } = await fixture(); + await store.close(); + const setup = new DatabaseSync(input.path); + try { + expect(setup.prepare("SELECT schema_version FROM store_binding").get()?.schema_version).toBe(2); + setup.exec("UPDATE receipt_storage_config SET partition_bytes=32768"); + } finally { setup.close(); } + const active = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(active); + let generation = "0"; + const text = "history ".repeat(1024); + for (let index = 1; index <= 64; index += 1) { + generation = await active.commit({ expectedGeneration: generation, state: { latest: index }, records: [record(`event-${index}`, String(index), text), + { epoch: "run-1", kind: "effect", id: `effect-${index}`, sequence: String(index), body: { accepted: index } }] }); + } + await active.close(); + const { stdout } = await execute(defaultStorageRunnerBinary(), [ + "storage", + "partitions", + "--path", + input.path, + "--binding", + input.binding, + "--limit", + "32", + ], { env: {}, maxBuffer: 64 * 1024, timeout: 30_000, windowsHide: true }); + const partitionPage = JSON.parse(stdout) as { + schema?: string; + partitions?: unknown[]; + }; + expect(partitionPage.schema).toBe("paperclip.receipt-partitions.v1"); + expect(partitionPage.partitions?.length ?? 0).toBeGreaterThan(1); + const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened); + expect(await reopened.load()).toEqual({ generation: "64", state: { latest: 64 } }); + expect(await reopened.getSessionEffect("effect-1")).toMatchObject({ epoch: "run-1", body: { accepted: 1 } }); + expect(await reopened.getRecord("run-1", "event", "event-1")).toEqual(record("event-1", "1", text)); + const page = await reopened.readEvents("run-1", "30", 8, 1024 * 1024); + expect(page.records.map(record => record.sequence)).toEqual(["31", "32", "33", "34", "35", "36", "37", "38"]); + await expect(reopened.commit({ expectedGeneration: "64", state: { wrong: true }, records: [{ epoch: "run-2", kind: "effect", id: "effect-1", sequence: "1", body: { accepted: 1 } }] })).rejects.toThrow("receipt_conflict"); + expect((await reopened.load())?.generation).toBe("64"); +}, 60_000); + +it("preserves retained version-1 controller files without silently replacing their authority", async () => { + const root = await mkdtemp(join(tmpdir(), "legacy-sqlite-authority-")); roots.push(root); + const path = join(root, "authority.sqlite"), binding = "legacy-retained", body = '{"retained":true}'; + const db = new DatabaseSync(path); + await chmod(path, 0o600); + try { + db.exec("CREATE TABLE authority_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE authority_state(singleton INTEGER PRIMARY KEY,generation INTEGER,body TEXT,digest TEXT); CREATE TABLE authority_records(epoch TEXT,kind TEXT,id TEXT,sequence INTEGER,body TEXT,digest TEXT,PRIMARY KEY(epoch,kind,id),UNIQUE(epoch,kind,sequence)); CREATE UNIQUE INDEX authority_session_effect ON authority_records(id) WHERE kind='effect'"); + db.prepare("INSERT INTO authority_binding VALUES (1,1,?)").run(binding); + db.prepare("INSERT INTO authority_state VALUES (1,7,?,?)").run(body, createHash("sha256").update(body).digest("hex")); + } finally { db.close(); } + const store = await SqliteAuthorityStore.open({ path, binding, create: false }); stores.push(store); + expect(store.unorderedEffectReceipts).toBe(false); + await expect(store.commit({ expectedGeneration: "7", state: {}, records: [{ ...record("effect", "0"), kind: "effect" }] })).rejects.toThrow("retained v1 store requires ordered effect receipts"); + expect(await store.load()).toEqual({ generation: "7", state: { retained: true } }); + await store.commit({ expectedGeneration: "7", state: { continued: true }, records: [record("legacy", "1")] }); + expect(await store.getRecord("run-1", "event", "legacy")).toEqual(record("legacy", "1")); + await store.close(); + const check = new DatabaseSync(path, { readOnly: true }); + try { expect(check.prepare("SELECT schema_version FROM authority_binding").get()?.schema_version).toBe(1); } + finally { check.close(); } +}); + +it("does not use an unfinished native backup as a current-state proof", async () => { + const { store, input } = await fixture(); + await store.commit({ expectedGeneration: "0", state: { authority: true }, records: [] }); + await store.close(); + const db = new DatabaseSync(input.path); + try { db.exec("CREATE TABLE backup_manifest(singleton INTEGER PRIMARY KEY,id TEXT,complete INTEGER); INSERT INTO backup_manifest VALUES(1,'unfinished',0)"); } + finally { db.close(); } + const proof = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(proof); + await expect(proof.load()).rejects.toThrow("backup is incomplete"); +}); + +it("pages outstanding owners at an exact generation and retires only the exact row", async () => { + const { store } = await fixture(); + const start = { collection: "process-owner" as const, id: "launch-1", expectedSha256: null, body: { phase: "spawned", processId: 9001 } }; + await store.commit({ expectedGeneration: "0", state: { owners: 1 }, records: [record("intent", "1")], work: [start] }); + const owner = (await store.getWork("process-owner", "launch-1"))!; + expect((await store.readWorkPage("process-owner", "", 1, "1")).records).toEqual([owner]); + await expect(store.commit({ expectedGeneration: "1", state: { owners: 0 }, records: [record("wrong", "2")], work: [{ ...start, expectedSha256: "0".repeat(64), body: null }] })).rejects.toThrow("stale_authority"); + expect(await store.getRecord("run-1", "event", "wrong")).toBeNull(); + expect(await store.getWork("process-owner", "launch-1")).toEqual(owner); + await store.commit({ expectedGeneration: "1", state: { owners: 0 }, records: [record("retirement-proof", "2")], work: [{ ...start, expectedSha256: owner.sha256, body: null }] }); + expect(await store.getWork("process-owner", "launch-1")).toBeNull(); + await expect(store.readWorkPage("process-owner", "", 1, "1")).rejects.toThrow("stale_authority"); + expect((await store.readWorkPage("process-owner", "", 128, "2")).records).toEqual([]); + expect(await store.getRecord("run-1", "event", "intent")).toEqual(record("intent", "1")); +}); + +it("read-only owner pages validate authority and reject a changed generation or corrupt root", async () => { + const { store, input } = await fixture(); + await store.commit({ expectedGeneration: "0", state: { owners: 1 }, records: [], work: [{ collection: "process-owner", id: "owner-1", expectedSha256: null, body: { phase: "spawned" } }] }); + const reader = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(reader); + expect((await reader.readWorkPage("process-owner", "", 128, "1")).records).toHaveLength(1); + await expect(reader.readWorkPage("process-owner", "", 128, "2")).rejects.toThrow("stale_authority"); + await reader.close(); await store.close(); + const db = new DatabaseSync(input.path); + try { db.exec("UPDATE current_state SET digest=zeroblob(32) WHERE key='authority'"); } finally { db.close(); } + const damaged = await SqliteAuthorityStore.open({ ...input, create: false, readOnly: true }); stores.push(damaged); + await expect(damaged.readWorkPage("process-owner", "", 128, "1")).rejects.toThrow("digest mismatch"); +}); + + +it("separates renewable command ordering from immutable command identity", async () => { + const { store, input } = await fixture(); + expect(store.commandEpochs).toBe(true); + const receipts: AuthorityRecord[] = [undefined, randomUUID(), randomUUID()].map((sequenceEpoch, index) => ({ + ...record(`command-${index}`, "1"), kind: "command", ...(sequenceEpoch ? { sequenceEpoch } : {}), + })); + const generation = await store.commit({ expectedGeneration: "0", state: { current: receipts[2]!.sequenceEpoch }, records: receipts }); + await store.close(); + const reopened = await SqliteAuthorityStore.open({ ...input, create: false }); stores.push(reopened); + for (const receipt of receipts) expect(await reopened.getRecord(receipt.epoch, "command", receipt.id)).toEqual(receipt); + for (const record of [{ ...receipts[1]!, id: "alias" }, { ...receipts[0]!, sequenceEpoch: randomUUID() }]) { + await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [record] })).rejects.toThrow("receipt_conflict"); + } + expect((await reopened.load())?.generation).toBe(generation); +}); diff --git a/packages/paperclip-runner/src/control-plane/sqlite-authority-store.ts b/packages/paperclip-runner/src/control-plane/sqlite-authority-store.ts new file mode 100644 index 00000000000..426781c9574 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/sqlite-authority-store.ts @@ -0,0 +1,127 @@ +import { basename } from "node:path"; +import { Worker } from "node:worker_threads"; +import { + DurableAuthorityStoreError, + validateAuthorityCommit, + validateAuthorityPage, + validateAuthorityWorkKey, + validateAuthorityWorkPage, + type AuthorityWorkRecord, + type AuthorityWorkPage, + type AuthorityCommit, + type AuthorityPage, + type AuthorityRecord, + type AuthoritySnapshot, + type DurableAuthorityStore, +} from "./durable-authority-store.js"; + +/** Standalone/eval implementation. Production Paperclip uses its Postgres + * transaction so event acceptance and current authority share one commit. */ +export class SqliteAuthorityStore implements DurableAuthorityStore { + readonly location: import("./authority-locator.js").AuthorityLocation; + readonly #worker: Worker; + readonly #pending = new Map(); + #nextRequest = 0; + #failure: Error | null = null; + #closed = false; + #unorderedEffectReceipts = false; + #eventEpochs = false; + get eventEpochs(): boolean { return this.#eventEpochs; } + #commandEpochs = false; + get commandEpochs(): boolean { return this.#commandEpochs; } + + get unorderedEffectReceipts(): boolean { return this.#unorderedEffectReceipts; } + + private constructor(readonly binding: string, path: string, create: boolean, readOnly = false, runnerBinary?: string) { + this.location = { kind: "sqlite", binding, file: basename(path) }; + const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; + this.#worker = new Worker(new URL(`./sqlite-authority-worker.${extension}`, import.meta.url), { + workerData: { binding, path, create, readOnly, runnerBinary }, + // The worker is a plain Node module, never a Vitest/tsx loader copy. + execArgv: [], + }); + this.#worker.on("message", (message: { id: number; value?: unknown; error?: { code: DurableAuthorityStoreError["code"]; message: string } }) => { + const pending = this.#pending.get(message.id); + if (!pending) return; + this.#pending.delete(message.id); + if (message.error) pending.reject(new DurableAuthorityStoreError(message.error.code, message.error.message)); + else pending.resolve(message.value); + }); + this.#worker.on("error", (error) => this.#fail(error)); + this.#worker.on("exit", (code) => { + if (!this.#closed || this.#pending.size) this.#fail(new DurableAuthorityStoreError("storage_unavailable", `storage worker exited (${code})`)); + }); + } + + static async open(input: { binding: string; path: string; create: boolean; readOnly?: boolean; runnerBinary?: string }): Promise { + const store = new SqliteAuthorityStore(input.binding, input.path, input.create, input.readOnly, input.runnerBinary); + try { + const ready = await store.#request<{ unorderedEffectReceipts?: boolean; commandEpochs?: boolean; eventEpochs?: boolean } | null>("ready", {}); + store.#unorderedEffectReceipts = ready?.unorderedEffectReceipts === true; + store.#commandEpochs = ready?.commandEpochs === true; + store.#eventEpochs = ready?.eventEpochs === true; + return store; + } + catch (error) { await store.#worker.terminate(); throw error; } + } + + #fail(error: Error): void { + this.#failure ??= error; + for (const pending of this.#pending.values()) pending.reject(this.#failure); + this.#pending.clear(); + } + + #request(operation: string, input: unknown): Promise { + if (this.#failure) return Promise.reject(this.#failure); + if (this.#closed) return Promise.reject(new DurableAuthorityStoreError("storage_unavailable", "store is closed")); + if (this.#pending.size >= 8) return Promise.reject(new DurableAuthorityStoreError("storage_pressure", "storage executor queue is full")); + // RPC correlation is local to the current bounded in-flight set, not a + // durable history sequence. Reuse settled IDs instead of overflowing a + // lifetime JavaScript counter in a continuously running storage worker. + do { this.#nextRequest = this.#nextRequest >= 0xffff_ffff ? 1 : this.#nextRequest + 1; } + while (this.#pending.has(this.#nextRequest)); + const id = this.#nextRequest; + return new Promise((resolve, reject) => { + this.#pending.set(id, { resolve: (value) => resolve(value as T), reject }); + try { this.#worker.postMessage({ id, operation, input }); } + catch (error) { this.#pending.delete(id); reject(error); } + }); + } + + load(): Promise { return this.#request("load", {}); } + + commit(input: AuthorityCommit): Promise { + validateAuthorityCommit(input); + return this.#request("commit", input); + } + + getRecord(epoch: string, kind: AuthorityRecord["kind"], id: string): Promise { + return this.#request("getRecord", { epoch, kind, id }); + } + + getSessionEffect(id: string): Promise { + return this.#request("getSessionEffect", { id }); + } + + getWork(collection: AuthorityWorkRecord["collection"], id: string): Promise { + validateAuthorityWorkKey(collection, id); + return this.#request("getWork", { collection, id }); + } + + readWorkPage(collection: AuthorityWorkRecord["collection"], after: string, limit: number, expectedGeneration: string): Promise { + validateAuthorityWorkPage(collection, after, limit, expectedGeneration); + return this.#request("readWorkPage", { collection, after, limit, expectedGeneration }); + } + + readEvents(epoch: string, after: string, limit: number, byteBudget: number, sequenceEpoch?: string): Promise { + validateAuthorityPage(after, limit, byteBudget, sequenceEpoch); + if (sequenceEpoch !== undefined && !this.eventEpochs) return Promise.reject(new DurableAuthorityStoreError("invalid_authority", "event epochs require indexed storage")); + return this.#request("readEvents", { epoch, after, limit, byteBudget, sequenceEpoch }); + } + + async close(): Promise { + if (this.#closed) return; + try { await this.#request("close", {}); } + finally { this.#closed = true; await this.#worker.terminate(); } + } +} diff --git a/packages/paperclip-runner/src/control-plane/sqlite-authority-worker.ts b/packages/paperclip-runner/src/control-plane/sqlite-authority-worker.ts new file mode 100644 index 00000000000..c8d9ec3fab6 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/sqlite-authority-worker.ts @@ -0,0 +1,228 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { spawn, execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { DatabaseSync } from "node:sqlite"; +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync, realpathSync } from "node:fs"; +import { basename, dirname, resolve } from "node:path"; +import type { AuthorityCommit, AuthorityRecord, AuthoritySnapshot, AuthorityWorkRecord } from "./durable-authority-store.js"; + +const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; +const { authorityInteger, authorityJson, validateAuthorityCommit, validateAuthorityPage, validateAuthorityWorkKey, validateAuthorityWorkPage, DurableAuthorityStoreError } = + await import(new URL(`./durable-authority-store.${extension}`, import.meta.url).href) as typeof import("./durable-authority-store.js"); +const { defaultStorageRunnerBinary } = await import(new URL(`./native-storage-command.${extension}`, import.meta.url).href) as typeof import("./native-storage-command.js"); +const { path, binding, create, readOnly, runnerBinary } = workerData as { path: string; binding: string; create: boolean; readOnly?: boolean; runnerBinary?: string }; +type Code = InstanceType["code"]; +function fail(code: Code, message: string): never { throw new DurableAuthorityStoreError(code, message); } +function privateFile(path: string): void { + // Workers share a POSIX process: opening and closing a probe descriptor can + // cancel SQLite's locks held by a different worker. SQLite owns all database + // and sidecar descriptors; the private-directory preflight is metadata only. + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) fail("invalid_authority", "unsafe storage file"); +} +let database: DatabaseSync | undefined; +let legacy = false; +let startupError: unknown; +try { + if (readOnly && create) fail("invalid_authority", "a proof reader cannot create authority"); + if (typeof binding !== "string" || !binding.length || binding.length > 1024 || /[\x00-\x1f]/.test(binding)) fail("invalid_authority", "invalid storage binding"); + if (create) mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const directory = lstatSync(dirname(path)); + if (!directory.isDirectory() || directory.isSymbolicLink() || (process.platform !== "win32" && ((directory.mode & 0o077) !== 0 || directory.uid !== process.getuid?.()))) fail("invalid_authority", "unsafe storage directory"); + const exists = lstatSync(path, { throwIfNoEntry: false }); + if (!exists && !create) fail("storage_unavailable", "activated store is missing"); + if (exists) { + for (const suffix of ["", "-wal", "-shm"]) if (!suffix || lstatSync(path + suffix, { throwIfNoEntry: false })) privateFile(path + suffix); + database = new DatabaseSync(resolve(realpathSync(dirname(path)), basename(path)), { readOnly: true }); + database.exec("PRAGMA trusted_schema=OFF; PRAGMA query_only=ON; PRAGMA cache_size=-4096; PRAGMA busy_timeout=5000;"); + legacy = !!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='authority_binding'").get(); + if (!legacy) { + const stored = database.prepare("SELECT schema_version,binding FROM store_binding WHERE singleton=1").get(); + if (stored?.schema_version !== 2 || stored.binding !== binding) fail("invalid_authority", "storage binding or version mismatch"); + } + if (legacy || !readOnly) { database.close(); database = undefined; } + } +} catch (error) { database?.close(); database = undefined; startupError = error; } + +if (legacy && !startupError) { + // Retained v1 files keep their original format; only fresh stores select v2. + await import(new URL(`./legacy-sqlite-authority-worker.${extension}`, import.meta.url).href); +} else { + type Reply = { id: number; value?: unknown; error?: { code: Code; message: string } }; + const pending = new Map(); + let requestId = 0, closing = false, stderr = ""; + let failed: Error | undefined; + let output = Buffer.alloc(0); + const child = !startupError && !readOnly ? spawn(runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "rpc", "--path", path, "--binding", binding, "--create", String(create)], { env: {}, stdio: ["pipe", "pipe", "pipe"], windowsHide: true }) : undefined; + function fence(error: Error) { + failed ??= error; + for (const request of pending.values()) request.reject(failed); + pending.clear(); + } + child?.stdout.on("data", (chunk: Buffer) => { + if (output.length + chunk.length > 96 * 1024 * 1024) { fence(new DurableAuthorityStoreError("storage_pressure", "storage response exceeds capacity")); child.kill(); return; } + output = Buffer.concat([output, chunk]); + for (;;) { + const end = output.indexOf(10); + if (end < 0) break; + const line = output.subarray(0, end); output = output.subarray(end + 1); + try { + const message = JSON.parse(line.toString()) as Reply; + const request = pending.get(message.id); + if (!request) fail("invalid_authority", "unexpected storage response"); + pending.delete(message.id); + if (message.error) request.reject(new DurableAuthorityStoreError(message.error.code, message.error.message)); + else request.resolve(message.value); + } catch (error) { fence(error as Error); child.kill(); } + } + }); + child?.stderr.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-8192); }); + child?.once("error", error => fence(new DurableAuthorityStoreError("storage_unavailable", error.message))); + child?.once("exit", code => { if (!closing || code !== 0 || pending.size) fence(new DurableAuthorityStoreError("storage_unavailable", stderr.trim() || `storage process exited (${code})`)); }); + child?.stdin.on("error", error => fence(new DurableAuthorityStoreError("storage_unavailable", error.message))); + parentPort!.once("close", () => { child?.stdin.end(); }); + function rpc(operation: string, input: unknown): Promise { + if (failed) return Promise.reject(failed); + if (!child) return Promise.reject(new DurableAuthorityStoreError("storage_unavailable", "storage process is absent")); + do { requestId = requestId >= 0xffff_ffff ? 1 : requestId + 1; } + while (pending.has(requestId)); + const id = requestId; + return new Promise((resolve, reject) => { + pending.set(id, { resolve: value => resolve(value as T), reject }); + child.stdin.write(`${JSON.stringify({ id, operation, input })}\n`); + }); + } + const namespace = (epoch: string, kind: AuthorityRecord["kind"]) => `authority/${kind}/${epoch}`; + const sequenceNamespace = (epoch: string, kind: AuthorityRecord["kind"]) => `sequence/${kind}/${epoch}`; + const encoded = (value: unknown) => Buffer.from(authorityJson(value)).toString("base64"); + const decode = (bytes: string): T => JSON.parse(Buffer.from(bytes, "base64").toString()) as T; + async function get(namespace: string, key: string): Promise { + const bytes = await rpc("get", { namespace, key }); + return bytes === null ? null : decode(bytes); + } + type NativeWork = { key: string; bytes: string; digest: string }; + const decodeWork = (collection: AuthorityWorkRecord["collection"], row: NativeWork): AuthorityWorkRecord => ({ collection, id: row.key, body: decode(row.bytes), sha256: Buffer.from(row.digest, "base64").toString("hex") }); + async function dispatch(operation: string, input: Record): Promise { + if (startupError) throw startupError; + if (readOnly) { + if (operation === "ready") return { unorderedEffectReceipts: true, commandEpochs: true, eventEpochs: true }; + if (operation === "close") { database!.close(); return null; } + if (operation === "commit") fail("invalid_authority", "proof readers cannot commit authority"); + if (!["load", "getWork", "readWorkPage", "getRecord"].includes(operation)) fail("invalid_authority", "proof readers expose current authority and exact references only"); + database!.exec("BEGIN"); + try { + if (database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_manifest'").get() + && database!.prepare("SELECT complete FROM backup_manifest WHERE singleton=1").get()?.complete !== 1) fail("invalid_authority", "indexed backup is incomplete"); + const statement = database!.prepare("SELECT generation,CASE WHEN length(bytes)<=16777216 THEN bytes END AS bytes,CASE WHEN length(digest)=32 THEN digest END AS digest FROM current_state WHERE key='authority'"); + statement.setReadBigInts(true); + const row = statement.get(); + if (!row && operation === "load") return null; + if (!row) fail("stale_authority", "outstanding work has no current authority"); + if (!(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array) || !createHash("sha256").update(row.bytes).digest().equals(Buffer.from(row.digest))) fail("invalid_authority", "current authority digest mismatch"); + const preparedStatement = database!.prepare("SELECT generation,CASE WHEN length(body)<=16777216 THEN body END AS body,digest FROM receipt_prepare WHERE state_key='authority'"); + preparedStatement.setReadBigInts(true); + const prepared = preparedStatement.get(); + if (prepared && (prepared.generation !== row.generation || !(prepared.body instanceof Uint8Array) || !(prepared.digest instanceof Uint8Array) + || !Buffer.from(prepared.body).equals(Buffer.from(row.bytes)) || !Buffer.from(prepared.digest).equals(Buffer.from(row.digest)))) fail("invalid_authority", "current authority has conflicting prepared receipts"); + if (operation === "getRecord") { + const { stdout } = await promisify(execFile)(runnerBinary ?? defaultStorageRunnerBinary(), ["storage", "read-receipt", "--path", path, + "--binding", binding, "--generation", String(row.generation), "--namespace", namespace(String(input.epoch), input.kind as AuthorityRecord["kind"]), "--key", String(input.id)], + { env: {}, windowsHide: true, timeout: 30_000, maxBuffer: 2 * 1024 * 1024 }); + const bytes = JSON.parse(stdout) as string | null; + return bytes === null ? null : decode(bytes); + } + if (operation !== "load") { + const collection = String(input.collection) as AuthorityWorkRecord["collection"]; + const get = (id: string): AuthorityWorkRecord | null => { + validateAuthorityWorkKey(collection, id); + if (!database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='current_work'").get()) return null; + const row = database!.prepare("SELECT CASE WHEN length(bytes)<=16384 THEN bytes END AS bytes,CASE WHEN length(digest)=32 THEN digest END AS digest FROM current_work WHERE collection=? AND key=?").get(collection, id); + if (!row) return null; + if (!(row.bytes instanceof Uint8Array) || !(row.digest instanceof Uint8Array) || !createHash("sha256").update(row.bytes).digest().equals(Buffer.from(row.digest))) fail("invalid_authority", "outstanding-work digest mismatch"); + return { collection, id, body: JSON.parse(Buffer.from(row.bytes).toString()), sha256: Buffer.from(row.digest).toString("hex") }; + }; + if (operation === "getWork") return get(String(input.id)); + const after = String(input.after), limit = Number(input.limit), generation = String(input.expectedGeneration); + validateAuthorityWorkPage(collection, after, limit, generation); + if (String(row.generation) !== generation) fail("stale_authority", "outstanding-work page generation changed"); + if (!database!.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='current_work'").get()) return { records: [], nextAfter: null }; + const rows = database!.prepare("SELECT key FROM current_work WHERE collection=? AND key>? ORDER BY key LIMIT ?").all(collection, after, limit); + const records = rows.map(row => get(String(row.key))!); + return { records, nextAfter: records.at(-1)?.id ?? null }; + } + return { generation: String(row.generation), state: JSON.parse(Buffer.from(row.bytes).toString()) }; + } finally { database!.exec("ROLLBACK"); } + } + switch (operation) { + case "ready": await rpc("ready", {}); return { unorderedEffectReceipts: true, commandEpochs: true, eventEpochs: true }; + case "load": { + const value = await rpc<{ generation: string; bytes: string } | null>("load", { key: "authority" }); + return value && { generation: value.generation, state: decode(value.bytes) }; + } + case "getRecord": return get(namespace(String(input.epoch), input.kind as AuthorityRecord["kind"]), String(input.id)); + case "getSessionEffect": return get("session-effects", String(input.id)); + case "getWork": { + const collection = String(input.collection) as AuthorityWorkRecord["collection"], id = String(input.id); + validateAuthorityWorkKey(collection, id); + const row = await rpc("getWork", { collection, key: id }); + return row && decodeWork(collection, row); + } + case "readWorkPage": { + const collection = String(input.collection) as AuthorityWorkRecord["collection"], after = String(input.after), limit = Number(input.limit), expectedGeneration = String(input.expectedGeneration); + validateAuthorityWorkPage(collection, after, limit, expectedGeneration); + const assertGeneration = async () => { + const state = await rpc<{ generation: string } | null>("load", { key: "authority" }); + if (!state || state.generation !== expectedGeneration) fail("stale_authority", "outstanding-work page generation changed"); + }; + await assertGeneration(); + const rows = await rpc("workPage", { collection, after, limit: String(limit) }); + await assertGeneration(); + return { records: rows.map(row => decodeWork(collection, row)), nextAfter: rows.at(-1)?.key ?? null }; + } + case "commit": { + const commit = input as unknown as AuthorityCommit; + validateAuthorityCommit(commit); + const receipts = commit.records.flatMap(record => { + const bytes = encoded(record); + const entries = [{ namespace: namespace(record.epoch, record.kind), key: record.id, bytes }]; + if (record.kind === "effect") entries.push({ namespace: "session-effects", key: record.id, bytes }); + else entries.push({ namespace: sequenceNamespace(record.epoch, record.kind) + (record.sequenceEpoch ? `/${record.sequenceEpoch}` : ""), key: record.sequence.padStart(19, "0"), bytes: encoded(record.id) }); + return entries; + }); + const work = (commit.work ?? []).map(change => ({ collection: change.collection, key: change.id, + expectedDigest: change.expectedSha256 === null ? null : Buffer.from(change.expectedSha256, "hex").toString("base64"), bytes: change.body === null ? null : encoded(change.body) })); + return rpc("commit", { key: "authority", expectedGeneration: commit.expectedGeneration, bytes: encoded(commit.state), receipts, work }); + } + case "readEvents": { + const epoch = String(input.epoch), after = String(input.after), limit = Number(input.limit), byteBudget = Number(input.byteBudget); + validateAuthorityPage(after, limit, byteBudget, input.sequenceEpoch as string | undefined); + const rows = await rpc>("page", { namespace: sequenceNamespace(epoch, "event") + (input.sequenceEpoch ? `/${input.sequenceEpoch}` : ""), after: authorityInteger(after).toString().padStart(19, "0"), limit: String(limit), byteBudget: "1048576" }); + const records: AuthorityRecord[] = []; + let bytes = 0; + for (const row of rows) { + const record = await get(namespace(epoch, "event"), decode(row.bytes)); + if (!record || record.sequenceEpoch !== input.sequenceEpoch || record.sequence.padStart(19, "0") !== row.key) fail("invalid_authority", "receipt sequence index changed"); + const size = Buffer.byteLength(authorityJson(record.body)); + if (bytes + size > byteBudget) break; + bytes += size; records.push(record); + } + return { records, nextAfter: records.at(-1)?.sequence ?? null }; + } + case "close": { + closing = true; + const exited = new Promise((resolve, reject) => { child!.once("exit", code => code === 0 ? resolve() : reject(new Error(`storage close failed (${code})`))); }); + await rpc("close", {}); child!.stdin.end(); await exited; return null; + } + default: fail("invalid_authority", "unknown storage operation"); + } + } + // Serialize temporary copies and commit order; the owner admits at most 8. + let chain = Promise.resolve(); + parentPort!.on("message", ({ id, operation, input }: { id: number; operation: string; input: Record }) => { + chain = chain.then(async () => { + try { parentPort!.postMessage({ id, value: await dispatch(operation, input) }); } + catch (error) { parentPort!.postMessage({ id, error: { code: error instanceof DurableAuthorityStoreError ? error.code : "storage_unavailable", message: error instanceof Error ? error.message : "storage worker failed" } }); } + }); + }); +} diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts index 739f0212937..e05e6533276 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts @@ -16,6 +16,25 @@ function nodeTransport( } describe("Codex app-server transport limits", () => { + it("correlates concurrent out-of-order responses by opaque request identity", async () => { + const transport = nodeTransport(` + const readline = require("node:readline"); + const pending = []; + readline.createInterface({input:process.stdin}).on("line", line => { + const request = JSON.parse(line); pending.push(request); + if (pending.length === 3) for (const value of pending.reverse()) { + process.stdout.write(JSON.stringify({id:value.id,result:{id:value.id,method:value.method}})+"\\n"); + } + }); + `); + try { + const values = await Promise.all(["first", "second", "third"].map(method => transport.request(method, {}))); + expect(values.map(value => value.method)).toEqual(["first", "second", "third"]); + expect(new Set(values.map(value => value.id)).size).toBe(3); + for (const value of values) expect(value.id).toMatch(/^paperclip-rpc-[a-f0-9-]{36}$/); + } finally { await transport.close(); } + }); + it("passes only bounded controller-projected GitHub credentials", () => { expect( createSanitizedCodexEnvironment({ diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index 172aec56573..deaa5750536 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -1,10 +1,14 @@ import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomUUID } from "node:crypto"; import type { HarnessRuntimeRequestResolution } from "../../contracts/harness-driver.js"; import { githubCredentialEnvironment } from "../../github-credential-environment.js"; +import type { NormalizedDeliveryPort, RawDeliveryCursor } from "../../control-plane/normalized-delivery.js"; export interface CodexRpcNotification { method: string; params: Record; + /** Durable consumer position, always present on indexed runner deliveries. */ + paperclipDelivery?: RawDeliveryCursor; /** Internal-only correlation retained outside provider params and canonical PRP payloads. */ paperclipTrace?: { sourceEventId: string; @@ -36,6 +40,8 @@ export interface CodexAppServerTransport { ): Promise>; notify(method: string, params?: Record): void; notifications(): AsyncIterable; + normalizedDelivery?(): NormalizedDeliveryPort | null; + setBeforeRunRotation?(settle: () => Promise): void; setServerRequestHandler(handler: CodexServerRequestHandler): void; /** * Optional provider-neutral resolution path used when a transport has @@ -367,8 +373,7 @@ const MAX_DIAGNOSTIC_LINE_BYTES = 16 * 1024; export class ProcessCodexAppServerTransport implements CodexAppServerTransport { #process: ChildProcessWithoutNullStreams; #notifications: BoundedAsyncQueue; - #pending = new Map(); - #nextId = 1; + #pending = new Map(); #serverRequestHandler: CodexServerRequestHandler = async () => ({ success: false, contentItems: [{ type: "input_text", text: "Unsupported client request." }], @@ -502,7 +507,8 @@ export class ProcessCodexAppServerTransport implements CodexAppServerTransport { ), ); } - const id = this.#nextId++; + const id = `paperclip-rpc-${randomUUID()}`; + if (this.#pending.has(id)) return Promise.reject(new Error("codex app-server fresh request identity unavailable")); return new Promise((resolve, reject) => { this.#pending.set(id, { resolve, reject }); try { @@ -634,10 +640,10 @@ export class ProcessCodexAppServerTransport implements CodexAppServerTransport { const message = value; const hasResult = Object.hasOwn(message, "result"); const hasError = Object.hasOwn(message, "error"); - if (typeof message.id === "number" && (hasResult || hasError)) { + if ((typeof message.id === "number" || typeof message.id === "string") && (hasResult || hasError)) { if ( - !Number.isSafeInteger(message.id) || - message.id <= 0 || + (typeof message.id === "number" && (!Number.isSafeInteger(message.id) || message.id <= 0)) || + (typeof message.id === "string" && (message.id.length === 0 || message.id.length > 160)) || hasResult === hasError ) { this.#fatal( diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 114ec5387c6..844eefdbbdb 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -379,7 +379,10 @@ export class CodexAppServerDriver implements HarnessDriver { ); await cancellation.wait(this.#persistProcessOwnership(transport)); const existingThread = record(existing.thread); - existingThread.turns = await cancellation.wait(readCodexTurnMetadata(transport, snapshot.driverSessionId)); + const normalizedRecovery = transport.normalizedDelivery?.()?.load() ?? null; + // The indexed reducer plus its unconsumed raw suffix owns recovery. + // Enumerating provider turns here would reintroduce lifetime-sized I/O. + existingThread.turns = normalizedRecovery ? [] : await cancellation.wait(readCodexTurnMetadata(transport, snapshot.driverSessionId)); if (text(existingThread.id) !== snapshot.driverSessionId) { await cancellation.wait(cancellation.close()); return { @@ -473,7 +476,7 @@ export class CodexAppServerDriver implements HarnessDriver { const goal = await cancellation.wait( this.#discoverGoal(transport, opened.threadId), ); - const recoveringAutonomousGoal = snapshot.goal?.status === "active" + const recoveringAutonomousGoal = normalizedRecovery === null && snapshot.goal?.status === "active" && goal != null && goal.createdAt === snapshot.goal.createdAt; if (recoveringAutonomousGoal) { @@ -496,7 +499,7 @@ export class CodexAppServerDriver implements HarnessDriver { recoveredActiveTurnId = active.length === 1 ? text(active[0]?.id) : recoveredActiveTurnId; } if ( - !recoveringAutonomousGoal && + normalizedRecovery === null && !recoveringAutonomousGoal && !this.#direct() && snapshot.semanticResult == null && recoveredActiveTurnId === null && @@ -608,6 +611,7 @@ export class CodexAppServerDriver implements HarnessDriver { stalePendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], lineage: snapshot.lineage, sourceSequence: snapshot.lastSourceSequence ?? 0, + sourceEpoch: snapshot.lastSourceEpoch, }); // A provider may settle the checkpointed turn while this controller is // disconnected (including during timeout cleanup). Reopening a thread @@ -616,8 +620,8 @@ export class CodexAppServerDriver implements HarnessDriver { // nor submit the original work again. Missing/conflicting history still // fails closed in reconcile(). if ( - recoveredActiveTurnId !== null || - reconcileUncheckpointedDispositionTurn + !session.deliveryRestored && (recoveredActiveTurnId !== null || + reconcileUncheckpointedDispositionTurn) ) { await cancellation.wait(session.reconcile?.() ?? Promise.resolve({})); } @@ -904,6 +908,7 @@ export class CodexAppServerDriver implements HarnessDriver { stalePendingRuntimeRequests?: HarnessRuntimeRequest[]; lineage?: HarnessThreadLineageEntry[]; sourceSequence: number; + sourceEpoch?: string; }): CodexHarnessSession { return new CodexHarnessSession({ ...input, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts index dd5f3e26610..3635e62d191 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts @@ -1,5 +1,8 @@ -import { realpathSync } from "node:fs"; +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { fileURLToPath } from "node:url"; +import { afterAll } from "vitest"; import { CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, @@ -45,7 +48,10 @@ import { type CodexTraceInterpretation, } from "./app-server-transport.js"; -export const WORKSPACE = realpathSync.native(process.cwd()); +// A managed checkout can live under ~/.codex, which production correctly +// rejects as provider working space. Fake sessions own a real private fixture. +export const WORKSPACE = realpathSync.native(mkdtempSync(join(tmpdir(), "codex-driver-test-"))); +afterAll(() => rmSync(WORKSPACE, { recursive: true, force: true })); export class TestQueue implements AsyncIterable { values: T[] = []; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index 87abaa1f400..ce8a316d423 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -59,14 +59,17 @@ export class CodexHarnessSession extends CodexSessionState implements HarnessSession { + #eventCommitter: ((event: PrpEvent) => Promise) | null = null; + constructor(input: CodexSessionStateInput) { super(input); this.transport.setServerRequestHandler((request) => handleServerRequest(this, request), ); - initializeCodexSessionEvents(this, input); - if (this.terminal) { - this.eventQueue.close(); + this.transport.setBeforeRunRotation?.(() => this.flushEventDelivery()); + if (!this.deliveryRestored) initializeCodexSessionEvents(this, input); + if (this.terminal && !this.deliveryWriter) { + this.closeEvents(); } else { void pumpNotifications(this); } @@ -116,6 +119,7 @@ export class CodexHarnessSession this.resultFingerprint = null; this.resultCallId = null; this.resultTurnId = null; + this.durableTerminal = null; this.dispositionOnlyRecoveryConsumed = false; this.dispositionOnlyRecoveryTurnId = null; this.terminal = false; @@ -124,6 +128,10 @@ export class CodexHarnessSession this.protocolFailed = false; this.protocolFailureCode = null; this.protocolFailureMessage = null; + this.deliveryWriter = null; + this.deliveryRestored = false; + if (this.transport.normalizedDelivery?.()) { this.sourceSequence = 0; this.sourceSequenceEpoch = null; } + this.initializeDelivery(); this.emit("run.attached", { runId: input.runId, sameSession: true }); } @@ -135,6 +143,28 @@ export class CodexHarnessSession return this.eventQueue; } + async acknowledgeEvent(event: PrpEvent): Promise { + await this.deliveryWriter?.acknowledge(event); + } + + setEventCommitter(commit: (event: PrpEvent) => Promise): void { + this.#eventCommitter = commit; + } + + async flushEventDelivery(): Promise { + const writer = this.deliveryWriter; + if (!writer) return; + await writer.flush(); + for (;;) { + const event = writer.port.load()?.pending[0]; + if (!event) return; + if (!this.#eventCommitter) throw new Error("normalized event sink is unavailable before run rotation"); + await this.#eventCommitter(event); + await writer.acknowledge(event); + this.eventQueue.discard((queued) => queued.sourceEventId === event.sourceEventId); + } + } + async startTurn(input: { message: NativeUserMessage; /** Set by orchestration only after successful provider-session recovery. */ @@ -308,6 +338,7 @@ export class CodexHarnessSession this.requireCapability("steering"); this.requireActiveTurn(input.turnId, "steering"); if (input.correlationId) { + await this.prefetchHistory("steering", [input.correlationId]); const acknowledgedTurnId = this.acknowledgedSteeringCorrelations.get( input.correlationId, ); @@ -344,7 +375,7 @@ export class CodexHarnessSession turnId: input.turnId, itemId: input.correlationId ? `${input.turnId}:steer:${input.correlationId}` - : `${input.turnId}:steer:${++this.steerSequence}`, + : `${input.turnId}:steer:${randomUUID()}`, }, ); } catch (error) { @@ -372,7 +403,7 @@ export class CodexHarnessSession text: "Interrupt queued until the provider assigns the turn identity.", status: "queued", }, - { itemId: `interrupt:queued:${++this.interruptSequence}` }, + { itemId: `interrupt:queued:${randomUUID()}` }, ); return; } @@ -403,7 +434,7 @@ export class CodexHarnessSession }, { turnId, - itemId: `${turnId}:interrupt:${++this.interruptSequence}`, + itemId: `${turnId}:interrupt:${randomUUID()}`, }, ); } catch (error) { @@ -599,7 +630,7 @@ export class CodexHarnessSession action: input.action, goal, }, - { itemId: `${this.opened.threadId}:goal:${this.sourceSequence + 1}` }, + { itemId: `${this.opened.threadId}:goal:${randomUUID()}` }, ); this.emitGoalEvent( input.action === "clear" @@ -649,6 +680,15 @@ export class CodexHarnessSession async reconcile(): Promise> { this.assertProtocolIntegrity(); this.requireCapability("reconciliation"); + if (this.deliveryWriter) { + await this.deliveryWriter.flush(); + const snapshot = await this.read(); + const thread = record(snapshot.thread); + if (text(thread.id) !== this.opened.threadId || (this.opened.providerSessionId !== null && text(thread.sessionId) !== this.opened.providerSessionId)) { + throw new HarnessReconciliationError("thread/read returned a different indexed provider session"); + } + return snapshot; + } const snapshot = await this.read(); const thread = record(snapshot.thread); if (text(thread.id) !== this.opened.threadId) { @@ -768,6 +808,7 @@ export class CodexHarnessSession async snapshot(): Promise { this.assertProtocolIntegrity(); + await this.deliveryWriter?.flush(); return { driverKind: this.driverKind, workingDirectory: this.opened.context.workingDirectory, @@ -795,6 +836,7 @@ export class CodexHarnessSession turnId, fingerprint, })), + ...(this.deliveryWriter && this.durableTerminal ? { durableTerminal: structuredClone(this.durableTerminal) } : {}), dispositionOnlyRecoveryConsumed: this.dispositionOnlyRecoveryConsumed, dispositionOnlyRecoveryTurnId: this.dispositionOnlyRecoveryTurnId, pendingRuntimeRequests: this.pendingRuntimeRequests(), @@ -802,11 +844,14 @@ export class CodexHarnessSession this.currentGoal === null ? null : structuredClone(this.currentGoal), lineage: this.lineage(), lastSourceSequence: this.sourceSequence, + ...(this.sourceSequenceEpoch ? { lastSourceEpoch: this.sourceSequenceEpoch } : {}), }; } async close(input?: { reason: string }): Promise { this.cancelPendingRequests("session_closed"); + await this.deliveryWriter?.flush(); + if (this.#eventCommitter) await this.flushEventDelivery(); this.eventQueue.close(); await this.transport.close(input?.reason); } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-history-cache.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-history-cache.test.ts new file mode 100644 index 00000000000..1f5e1cdbd37 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-history-cache.test.ts @@ -0,0 +1,87 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { DurablePrpControlPlane } from "../../control-plane/durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "../../control-plane/sqlite-authority-store.js"; +import { CodexHistoryMap } from "./codex-history-cache.js"; + +it("evicts only cache entries: ancient identities remain exact after 2,000 receipts and reopen", async () => { + const root = await mkdtemp(join(tmpdir(), "driver-history-")); + const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", runId: "run-1", turnId: "turn", itemId: "item" }; + const config = { path: join(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }; + let store = await SqliteAuthorityStore.open(config); + const options = { stateDirectory: root, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }; + let core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + try { + let port = core.normalizedDelivery()!; + const cache = new CodexHistoryMap("steering", port.history!, [], false); + for (let page = 0; page < 20; page++) { + for (let i = 0; i < 100; i++) cache.set(`call-${page * 100 + i}`, `turn-${page * 100 + i}`); + await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: { epoch: identity.runId, sourceSeq: 0, ordinal: 0 }, driver: { cache: [...cache] }, events: [], receipts: cache.takeReceipts() }); + expect(cache.size).toBeLessThanOrEqual(128); + expect(JSON.stringify((await store.load())!.state).length).toBeLessThan(20_000); + } + expect(() => cache.get("call-0")).toThrow("exact receipt read"); + await cache.prefetch(["call-0", "never-accepted"]); + expect(cache.get("call-0")).toBe("turn-0"); + expect(cache.has("never-accepted")).toBe(false); + await core.stop(); await store.close(); + store = await SqliteAuthorityStore.open({ ...config, create: false }); + core = await DurablePrpControlPlane.open({ ...options, authorityStore: store }); + port = core.normalizedDelivery()!; + expect(await port.history!.get("steering", "call-0")).toBe("turn-0"); + await core.rotateRunIdentity({ ...identity, runId: "run-2", turnId: "turn-2", itemId: "item-2" }); + await expect(port.history!.get("steering", "call-0")).rejects.toThrow("retired run"); + port = core.normalizedDelivery()!; + expect(await port.history!.get("steering", "call-0")).toBe("turn-0"); + const before = await store.load(); + await expect(port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: { epoch: "run-2", sourceSeq: 0, ordinal: 0 }, driver: {}, events: [], receipts: [ + { collection: "steering", key: "new-before-conflict", value: "must-not-commit" }, + { collection: "steering", key: "call-0", value: "different-turn" }, + ] })).rejects.toThrow("receipt_conflict"); + expect(await port.history!.get("steering", "new-before-conflict")).toBeNull(); + expect(await store.load()).toEqual(before); + } finally { await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); } +}, 120_000); + +it("normalizes children of evicted ancestors through exact indexed lineage without growing current state", async () => { + const { FakeCodexTransport, makeDriver, WORKSPACE } = await import("./codex-app-server-driver.test-support.js"); + const { vi } = await import("vitest"); + const root = await mkdtemp(join(tmpdir(), "driver-lineage-history-")); + const identity = { runnerInstanceId: "runner-codex", environmentLeaseId: "lease", normalizedSessionId: "session-1", runId: "run-1", turnId: "turn-1", itemId: "item-1" }; + const store = await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, identity, authorityStore: store, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const port = core.normalizedDelivery()!; + class Transport extends FakeCodexTransport { normalizedDelivery() { return port; } } + const transport = new Transport("thread-root"); + const session = await makeDriver([transport]).openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: WORKSPACE }); + let lastThread: string | null = null; + let failure: unknown; + session.setEventCommitter!(async event => { + if (event.eventType === "session.failed") failure = event; + if (event.eventType === "item.started" && event.payload.kind === "thread_lineage") lastThread = (event.payload.lineage as { threadId: string }).threadId; + }); + const started = (id: string, parent: string, depth: number) => transport.queue.push({ method: "thread/started", params: { thread: { id, sessionId: "codex-account-session", source: { subAgent: { thread_spawn: { parent_thread_id: parent, depth } } }, status: { type: "idle" } } } }); + try { + await session.startTurn({ message: { text: "Work" } }); + await session.flushEventDelivery!(); + for (let index = 0; index < 300; index++) { + started(`thread-child-${index}`, "thread-root", 1); + await vi.waitFor(async () => { expect(await port.history!.get("lineage", `thread-child-${index}`)).not.toBeNull(); }, { interval: 1, timeout: 5_000 }); + await session.flushEventDelivery!(); + expect(failure).toBeUndefined(); + expect(lastThread).toBe(`thread-child-${index}`); + transport.queue.push({ method: "thread/closed", params: { threadId: `thread-child-${index}` } }); + } + await session.flushEventDelivery!(); + expect(session.lineage!().length).toBeLessThanOrEqual(128); + expect(session.lineage!().some(thread => thread.threadId === "thread-child-0")).toBe(false); + started("thread-grandchild", "thread-child-0", 2); + await vi.waitFor(async () => { expect(await port.history!.get("lineage", "thread-grandchild")).not.toBeNull(); }, { interval: 1, timeout: 5_000 }); + await session.flushEventDelivery!(); + expect(await port.history!.get("lineage", "thread-grandchild")).toMatchObject({ threadId: "thread-grandchild", parentThreadId: "thread-child-0", depth: 2 }); + expect(Buffer.byteLength(JSON.stringify((await store.load())!.state))).toBeLessThan(100_000); + expect(port.load()!.pending).toEqual([]); + } finally { await session.close({ reason: "test complete" }); transport.queue.close(); await core.stop(); await store.close(); await rm(root, { recursive: true, force: true }); } +}, 120_000); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-history-cache.ts b/packages/paperclip-runner/src/drivers/codex/codex-history-cache.ts new file mode 100644 index 00000000000..e4009810bda --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-history-cache.ts @@ -0,0 +1,77 @@ +import type { DriverHistoryCollection, DriverHistoryReceipt, NormalizedDeliveryPort } from "../../control-plane/normalized-delivery.js"; + +const CACHE_ENTRIES = 128; + +/** A cache miss is never permission to forget a completed action. Callers must + * prefetch the exact keys before entering their synchronous reducer step. */ +export class CodexHistoryMap extends Map { + #absent = new Set(); + #dirty = new Map(); + + constructor(readonly collection: DriverHistoryCollection, private readonly history: NonNullable, entries: Iterable<[string, T]>, restored: boolean, private readonly encode: (value: T) => unknown = (value) => value, private readonly decode: (value: unknown) => T = (value) => value as T) { + super(); + for (const [key, value] of entries) { + if (!restored) this.#dirty.set(key, structuredClone(value)); + super.set(key, structuredClone(value)); + } + if (this.size > CACHE_ENTRIES) throw new Error("indexed driver cache requires a fenced history import"); + } + + override get(key: string): T | undefined { + if (super.has(key)) return super.get(key); + if (key === "" || this.#absent.has(key)) return undefined; + throw new Error(`indexed driver ${this.collection} lookup requires an exact receipt read`); + } + + override has(key: string): boolean { return this.get(key) !== undefined; } + + override set(key: string, value: T): this { + this.#absent.delete(key); + this.#dirty.set(key, structuredClone(value)); + super.delete(key); + super.set(key, value); + this.#trim(); + return this; + } + + override clear(): void { + super.clear(); + this.#absent.clear(); + this.#dirty.clear(); + } + + #trim(): void { + while (this.size > CACHE_ENTRIES) super.delete(super.keys().next().value!); + while (this.#absent.size > CACHE_ENTRIES) this.#absent.delete(this.#absent.values().next().value!); + } + + async prefetch(keys: Iterable): Promise { + const requested = [...new Set(keys)].filter(Boolean); + if (requested.length > CACHE_ENTRIES) throw new Error("storage_pressure: driver receipt lookup batch exceeds capacity"); + for (const key of requested) { + if (super.has(key)) { const value = super.get(key)!; super.delete(key); super.set(key, value); continue; } + if (this.#absent.has(key)) { this.#absent.delete(key); this.#absent.add(key); continue; } + const value = await this.history.get(this.collection, key); + if (value === null) this.#absent.add(key); + else super.set(key, this.decode(structuredClone(value))); + this.#trim(); + } + } + + takeReceipts(): DriverHistoryReceipt[] { + const receipts = [...this.#dirty].map(([key, value]) => ({ collection: this.collection, key, value: this.encode(value) })); + this.#dirty.clear(); + return receipts; + } +} + +export class CodexHistorySet implements Iterable { + readonly map: CodexHistoryMap; + constructor(history: NonNullable, entries: Iterable, restored: boolean) { + this.map = new CodexHistoryMap("file", history, [...entries].map((key) => [key, true]), restored); + } + has(key: string): boolean { return this.map.has(key); } + add(key: string): this { this.map.set(key, true); return this; } + clear(): void { this.map.clear(); } + [Symbol.iterator](): IterableIterator { return this.map.keys(); } +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-normalized-delivery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-normalized-delivery.test.ts new file mode 100644 index 00000000000..f4e856311ad --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-normalized-delivery.test.ts @@ -0,0 +1,83 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { SqliteAuthorityStore } from "../../control-plane/sqlite-authority-store.js"; +import { applyNormalizedBatch, acknowledgeNormalizedEvent, type NormalizedDeliveryState, type NormalizedDeliveryPort } from "../../control-plane/normalized-delivery.js"; +import { FakeCodexTransport, makeDriver, WORKSPACE, type PrpEvent } from "./codex-app-server-driver.test-support.js"; + +it.each([undefined, 4])("recovers a pending question from SQLite with normalized epoch limit %s without enumerating provider history", async limit => { + const root = await mkdtemp(join(tmpdir(), "codex-normalizer-")); + const config = { path: join(root, "consumer.sqlite"), binding: "test-normalized-consumer", create: true }; + let store = await SqliteAuthorityStore.open(config); + let current: NormalizedDeliveryState | null = null; + let generation = "0"; + let tail: Promise = Promise.resolve(); + const serialize = (action: () => Promise): Promise => { + const result = tail.then(action); tail = result.catch(() => undefined); return result; + }; + const port: NormalizedDeliveryPort = { + epoch: "run-1", + ...(limit ? { eventEpochs: { limit } } : {}), + load: () => structuredClone(current), + commit: (batch) => serialize(async () => { + const candidate = applyNormalizedBatch(current, batch); + generation = await store.commit({ expectedGeneration: generation, state: { consumer: candidate }, records: [] }); + current = candidate; + return structuredClone(candidate); + }), + acknowledge: (event) => serialize(async () => { + const candidate = acknowledgeNormalizedEvent(current!, event); + generation = await store.commit({ expectedGeneration: generation, state: { consumer: candidate }, records: [] }); + current = candidate; + }), + }; + class Transport extends FakeCodexTransport { normalizedDelivery() { return port; } } + const first = new Transport(), second = new Transport(); + const driver = makeDriver([first, second]); + try { + const session = await driver.openSession({ runId: "run-1", normalizedSessionId: "session-1", workingDirectory: WORKSPACE }); + const written: PrpEvent[] = []; + session.setEventCommitter!(async (event) => { written.push(event); }); + await session.startTurn({ message: { text: "Work" } }); + await session.flushEventDelivery!(); + const checkpointBeforeRawDelivery = await session.snapshot(); + first.queue.push({ method: "paperclip/runtimeRequest", params: { request: { + id: "question-1", method: "item/tool/requestUserInput", params: { + threadId: "thread-1", turnId: "turn-1", questions: [{ id: "color", header: "Color", question: "Which color?", options: [{ label: "Blue", description: "Blue" }, { label: "Red", description: "Red" }] }], + }, + } }, paperclipDelivery: { epoch: "run-1", sourceSeq: 1, ordinal: 1 } }); + await vi.waitFor(() => expect(port.load()?.raw).toEqual({ epoch: "run-1", sourceSeq: 1, ordinal: 1 })); + const pending = port.load()!.pending; + if (limit) expect(port.load()!.producedEpoch).toBeTruthy(); + expect(pending.some((event) => event.eventType === "runtime_request.created")).toBe(true); + // Simulate the controller disappearing after mapping but before append. + // The fake provider remains; no close/cancellation is sent to it. + first.queue.close(); + await tail; await store.close(); + store = await SqliteAuthorityStore.open({ ...config, create: false }); + const restored = (await store.load())!; + generation = restored.generation; + current = restored.state.consumer as unknown as NormalizedDeliveryState; + const recovery = await driver.recoverSession(checkpointBeforeRawDelivery, { signal: new AbortController().signal }); + expect(recovery.recovered).toBe(true); + const recovered = recovery.session!; + expect(second.calls.some((call) => call.method === "thread/turns/list" || call.method === "thread/items/list")).toBe(false); + expect(recovered.pendingRuntimeRequests!().map((request) => request.requestId)).toEqual(["question-1"]); + const reader = recovered.events()[Symbol.asyncIterator](); + for (const expected of pending) { + const actual = (await reader.next()).value!; + expect(actual).toEqual(expected); + await recovered.acknowledgeEvent!(actual); + } + expect(port.load()!.pending).toEqual([]); + recovered.setEventCommitter!(async (event) => { written.push(event); }); + await recovered.resolveRuntimeRequest!({ requestId: "question-1", turnId: "turn-1", resolution: { action: "submit", answers: { color: { answers: ["Blue"] } } } }); + await recovered.flushEventDelivery!(); + expect(recovered.pendingRuntimeRequests!()).toEqual([]); + expect(written.filter((event) => event.eventType === "runtime_request.resolved")).toHaveLength(1); + await recovered.close({ reason: "test complete" }); + } finally { + first.queue.close(); second.queue.close(); await tail; await store.close(); await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts index b3932975dfd..374bda3dea6 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts @@ -54,7 +54,7 @@ async function authenticatedRunner( }; const digest = (domain: string, parts: Buffer[]) => createHash("sha256").update(framed(domain, parts)).digest(); - const credential = Buffer.from(core.issueBootstrapTicket()); + const credential = Buffer.from((await core.issueBootstrapTicket())); const authKey = digest("paperclip-runner-auth-key-v1", [credential]); const mac = (domain: string, parts: Buffer[]) => createHmac("sha256", authKey).update(framed(domain, parts)).digest(); @@ -607,8 +607,8 @@ describe("Codex protocol integrity propagation", () => { result, }, }); - await vi.waitFor(() => - expect(core.getCommand(command.commandId)?.status).toBe("completed"), + await vi.waitFor(async () => + expect((await core.getCommand(command.commandId))?.status).toBe("completed"), ); }; const event = ( diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts index 59370b07385..54f387a9ad6 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts @@ -1,8 +1,10 @@ +import { randomUUID } from "node:crypto"; import { observeCodexUsage, codexRunUsage } from "./codex-usage-baseline.js"; import { classifyCodexNotification } from "./codex-notification-identity.js"; import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js"; import { canonicalProviderEventsFromCodex, isCanonicalProviderEventType } from "../../provider-events.js"; import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js"; +import { handleServerRequest } from "./codex-session-server-requests.js"; import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js"; @@ -39,9 +41,31 @@ import { export async function pumpNotifications(state: CodexSessionState): Promise { try { for await (const notification of state.transport.notifications()) { + await state.deliveryWriter?.waitForCapacity(); + if (notification.method === "paperclip/runtimeRequest") { + const request = notification.params.request as unknown as import("./app-server-transport.js").CodexRpcServerRequest; + void handleServerRequest(state, { ...request, paperclipDelivery: notification.paperclipDelivery }).catch((error) => state.eventQueue.fail(error)); + await state.deliveryWriter?.flush(); + continue; + } + if (notification.method === "turn/completed") await state.turnStartSettled; + const params = notification.params; + const lineage = lineageFromThread(params.thread); + await state.prefetchHistory("lineage", [state.opened.threadId, text(params.threadId), text(record(params.turn).threadId), lineage.threadId, lineage.parentThreadId ?? ""]); + const turnId = text(params.turnId, text(record(params.turn).id, state.activeTurnId ?? "")); + await state.prefetchHistory("terminal", [turnId]); + const item = itemFromParams(params); + if (notification.method === "item/completed" && text(item.type) === "agentMessage") { + await state.prefetchHistory("file", paperclipWorkspaceFileReferencesFromText(state.opened.context.workingDirectory, text(item.text), turnId).map((reference) => reference.referenceId)); + } await mapNotification(state, notification); + await state.deliveryWriter?.flush(); } } catch (error) { + if (state.deliveryWriter?.failure) { + state.eventQueue.fail(state.deliveryWriter.failure); + return; + } if (error instanceof NativeSessionProtocolIntegrityError) { state.failProtocolIntegrity(error); return; @@ -59,26 +83,17 @@ export async function pumpNotifications(state: CodexSessionState): Promise } async function mapNotification(state: CodexSessionState, notification: CodexRpcNotification): Promise { - const sourceSequenceBefore = state.sourceSequence; + const emittedEventIds: string[] = []; let rejected = false; try { - await mapNotificationBody(state, notification); + state.capturedEventIds.run(emittedEventIds, () => mapNotificationBody(state, notification)); + state.deliveryWriter?.changed(notification.paperclipDelivery); } catch (error) { rejected = true; throw error; } finally { const correlation = notification.paperclipTrace; if (correlation !== undefined) { - const emittedEventIds: string[] = []; - for ( - let sourceSeq = sourceSequenceBefore + 1; - sourceSeq <= state.sourceSequence; - sourceSeq += 1 - ) { - emittedEventIds.push( - `${state.runnerInstanceId}:${state.runId}:${sourceSeq}`, - ); - } const disposition: CodexTraceInterpretation["disposition"] = rejected ? "rejected" : emittedEventIds.length > 0 @@ -104,7 +119,7 @@ async function mapNotification(state: CodexSessionState, notification: CodexRpcN } } -async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise { +function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): void { if (!isSupportedCodexNotificationMethod(notification.method)) return; if (notification.method === "item/completed" && notification.params.kind === "steering_acknowledgement" && !notification.params.threadId && !notification.params.turnId && !notification.params.thread && !notification.params.turn) return; @@ -270,7 +285,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex }, { turnId: turnId || undefined, - itemId: `${threadId}:goal:update:${state.sourceSequence + 1}`, + itemId: `${threadId}:goal:update:${randomUUID()}`, }, ); state.emitGoalEvent("session.goal.updated", goal, { @@ -289,7 +304,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex action: "notification", goal: null, }, - { itemId: `${threadId}:goal:clear:${state.sourceSequence + 1}` }, + { itemId: `${threadId}:goal:clear:${randomUUID()}` }, ); state.emitGoalEvent("session.goal.cleared", null, { workingNow: state.activeTurnId !== null, @@ -429,7 +444,6 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex // channel before turn/start's own response settles on the request // channel. Wait for the pending turn/start to settle first, so // turn.accepted always precedes the terminal event for the same turn. - await state.turnStartSettled; if (state.terminalTurns.has(turnId)) { mapTerminalTurn(state, turn, turnId); return; @@ -448,6 +462,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex kind: text(item.type, "unknown"), channel, providerPhase: text(item.phase) || undefined, + ...(item.outputBody ? { outputBody: item.outputBody } : {}), text: itemText(item), item, }), @@ -467,6 +482,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex kind: text(item.type, "unknown"), channel, providerPhase: text(item.phase) || undefined, + ...(item.outputBody ? { outputBody: item.outputBody } : {}), text: itemText(item), item, }), @@ -519,6 +535,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex kind: deltaKind, channel, providerMethod: notification.method, + ...(params.outputBody ? { outputBody: params.outputBody } : {}), text: text(params.delta, text(params.patch, text(params.output))), update: params, }), @@ -546,7 +563,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex { kind: "usage", usage: state.usageSnapshot }, { turnId, - itemId: `${turnId}:usage:${state.sourceSequence + 1}`, + itemId: `${turnId}:usage:${randomUUID()}`, }, ); return; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts index 19c78149582..184b84dff6f 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts @@ -29,28 +29,24 @@ export async function handleServerRequest( state: CodexSessionState, request: CodexRpcServerRequest, ): Promise> { - const sourceSequenceBefore = state.sourceSequence; + const emittedEventIds: string[] = []; let rejected = false; try { - const response = await handleServerRequestBody(state, request); + if (request.method !== "item/tool/call") state.pendingRuntimeRequestSources.set(String(request.id), structuredClone(request)); + const pendingResponse = state.capturedEventIds.run(emittedEventIds, () => handleServerRequestBody(state, request)); + // Runtime-input admission executes synchronously, then waits for a user. + // Persist that admission without holding the notification pump hostage. + if (request.method !== "item/tool/call") state.deliveryWriter?.changed(request.paperclipDelivery); + const response = await pendingResponse; rejected = response.success === false; return response; } catch (error) { rejected = true; throw error; } finally { + state.pendingRuntimeRequestSources.delete(String(request.id)); const correlation = request.paperclipTrace; if (correlation !== undefined) { - const emittedEventIds: string[] = []; - for ( - let sourceSeq = sourceSequenceBefore + 1; - sourceSeq <= state.sourceSequence; - sourceSeq += 1 - ) { - emittedEventIds.push( - `${state.runnerInstanceId}:${state.runId}:${sourceSeq}`, - ); - } try { state.transport.recordTraceInterpretation?.({ sourceEventId: correlation.sourceEventId, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 9301a3a2962..9593afb17d0 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -1,3 +1,6 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; +import { normalizedEventId, type EventEpochTransition } from "../../control-plane/event-epochs.js"; import { type CodexUsageBaseline, codexRunUsage } from "./codex-usage-baseline.js"; import type { HarnessRuntimeRequest, @@ -31,6 +34,13 @@ import type { PendingRuntimeRequest, } from "./codex-driver-types.js"; import { canonicalJson, record } from "./codex-driver-values.js"; +import { NormalizedDeliveryWriter } from "../../control-plane/normalized-delivery-writer.js"; +import type { CodexRpcServerRequest } from "./app-server-transport.js"; +import { createCodexQuestionResponseContext, normalizeCodexQuestionSet } from "./codex-question-adapter.js"; +import { CodexHistoryMap, CodexHistorySet } from "./codex-history-cache.js"; + +const durableScalars = ["sourceSequence", "sourceSequenceEpoch", "activeTurnId", "usageSnapshot", "codexUsageBaseline", "result", "resultFingerprint", "resultCallId", "resultTurnId", "protocolFailed", "protocolFailureCode", "protocolFailureMessage", "terminal", "dispositionOnlyRecoveryAvailable", "dispositionOnlyRecoveryConsumed", "dispositionOnlyRecoveryTurnId", "turnStarted", "turnStartPending", "notificationIdentityDiagnostics", "currentGoal", "interruptQueued", "steerSequence", "interruptSequence", "durableTerminal"] as const; +const durableMaps = ["terminalTurns", "workspaceChangesByTurn", "itemChannels", "lineageByThread", "acknowledgedSteeringCorrelations"] as const; class AsyncQueue implements AsyncIterable { #values: T[] = []; @@ -68,6 +78,10 @@ class AsyncQueue implements AsyncIterable { this.#values = []; } + discard(predicate: (value: T) => boolean): void { + this.#values = this.#values.filter((value) => !predicate(value)); + } + [Symbol.asyncIterator](): AsyncIterator { return { next: async () => { @@ -105,7 +119,13 @@ export class CodexSessionState { readonly completionFeedback: CodexAppServerDriverOptions["completionFeedback"]; readonly dynamicToolHandler: CodexAppServerDriverOptions["dynamicToolHandler"]; readonly eventQueue = new AsyncQueue(); + deliveryWriter: NormalizedDeliveryWriter | null = null; + deliveryRestored = false; + durableTerminal: { event: PrpEvent; semanticFingerprint: string | null } | null = null; + readonly pendingRuntimeRequestSources = new Map(); sourceSequence: number; + sourceSequenceEpoch: string | null = null; + readonly capturedEventIds = new AsyncLocalStorage(); activeTurnId: string | null; usageSnapshot: Record | null = null; codexUsageBaseline: CodexUsageBaseline | null = null; @@ -131,20 +151,20 @@ export class CodexSessionState { dispositionOnlyRecoveryConsumed = false; dispositionOnlyRecoveryTurnId: string | null = null; turnStarted = false; - readonly terminalTurns = new Map(); + terminalTurns: Map = new Map(); readonly workspaceChangesByTurn = new Map>(); - readonly emittedFileReferences = new Set(); + emittedFileReferences: Set | CodexHistorySet = new Set(); readonly itemChannels = new Map< string, "progress" | "final" | "summary" | "detail" | "unknown" >(); readonly pendingRuntimeRequestMap = new Map(); notificationIdentityDiagnostics = 0; - readonly lineageByThread = new Map(); + lineageByThread: Map = new Map(); currentGoal: HarnessThreadGoal | null = null; interruptQueued = false; steerSequence = 0; - readonly acknowledgedSteeringCorrelations = new Map(); + acknowledgedSteeringCorrelations: Map = new Map(); interruptSequence = 0; constructor(input: { @@ -165,6 +185,7 @@ export class CodexSessionState { lineage?: HarnessThreadLineageEntry[]; goal?: HarnessThreadGoal | null; sourceSequence: number; + sourceEpoch?: string; now: () => Date; runnerInstanceId: string; driverKind: string; @@ -189,6 +210,7 @@ export class CodexSessionState { this.conversationMode = input.conversationMode; this.activeTurnId = input.activeTurnId ?? null; this.sourceSequence = input.sourceSequence; + this.sourceSequenceEpoch = input.sourceEpoch ?? null; this.now = input.now; this.runnerInstanceId = input.runnerInstanceId; this.driverKind = input.driverKind; @@ -279,6 +301,96 @@ export class CodexSessionState { this.dispositionOnlyRecoveryConsumed = dispositionOnlyRecoveryPreviouslyConsumed; this.dispositionOnlyRecoveryTurnId = dispositionOnlyRecoveryTurnId; + this.initializeDelivery(); + } + + initializeDelivery(): void { + const port = this.transport.normalizedDelivery?.(); + if (!port) return; + const saved = port.load(); + if (saved) { + const state = saved.driver; + if (state.schema !== "paperclip.codex.reducer.v1" || state.runId !== this.runId || state.sessionId !== this.normalizedSessionId || state.threadId !== this.opened.threadId) throw new HarnessReconciliationError("normalized reducer binding mismatch"); + const scalars = record(state.scalars); + for (const key of durableScalars) { + if (key === "sourceSequenceEpoch" && !(key in scalars)) continue; // Retained pre-epoch reducer. + if (!(key in scalars)) throw new HarnessReconciliationError(`normalized reducer lacks ${key}`); + (this as unknown as Record)[key] = structuredClone(scalars[key]); + } + if (this.sourceSequence !== saved.produced || (this.sourceSequenceEpoch ?? undefined) !== saved.producedEpoch) throw new HarnessReconciliationError("normalized reducer cursor mismatch"); + for (const key of durableMaps) { + const entries = state[key]; + if (!Array.isArray(entries)) throw new HarnessReconciliationError(`normalized reducer lacks ${key}`); + const target = this[key] as Map; + target.clear(); + for (const [id, value] of entries) target.set(id, structuredClone(value)); + } + this.emittedFileReferences.clear(); + if (!Array.isArray(state.emittedFileReferences) || !Array.isArray(state.pendingRequests)) throw new HarnessReconciliationError("invalid normalized reducer collections"); + for (const id of state.emittedFileReferences) this.emittedFileReferences.add(id); + for (const pending of state.pendingRequests) { + const request = pending.request as HarnessRuntimeRequest; + const source = pending.source as CodexRpcServerRequest; + if (!source || String(source.id) !== request.requestId || source.method !== request.method) throw new HarnessReconciliationError("normalized runtime request binding mismatch"); + const responseContext = createCodexQuestionResponseContext(); + normalizeCodexQuestionSet(source.method, source.params, responseContext); + this.pendingRuntimeRequestSources.set(request.requestId, source); + // Runnerd owns resolution delivery; the vanished controller's JS + // promise grants no authority and needs no reconstructed callback. + this.pendingRuntimeRequestMap.set(request.requestId, { request, responseContext, settle: () => {} }); + } + this.deliveryRestored = true; + } + if (port.history) { + this.lineageByThread = new CodexHistoryMap("lineage", port.history, this.lineageByThread, saved !== null || this.lineageByThread instanceof CodexHistoryMap, + ({ threadId, providerSessionId, parentThreadId, depth }) => ({ threadId, providerSessionId, parentThreadId, depth }), + (value) => ({ ...value as Pick, nickname: null, role: null, status: "unknown" })); + this.terminalTurns = new CodexHistoryMap("terminal", port.history, this.terminalTurns, saved !== null); + this.acknowledgedSteeringCorrelations = new CodexHistoryMap("steering", port.history, this.acknowledgedSteeringCorrelations, saved !== null || this.acknowledgedSteeringCorrelations instanceof CodexHistoryMap); + this.emittedFileReferences = new CodexHistorySet(port.history, this.emittedFileReferences, saved !== null || this.emittedFileReferences instanceof CodexHistorySet); + } + this.deliveryWriter = new NormalizedDeliveryWriter(port, () => this.deliveryCheckpoint(), (event) => this.eventQueue.push(event), (error) => { + this.eventQueue.fail(error); + void this.transport.detachControllerForRestart?.().catch(() => undefined); + }, () => [ + ...(this.lineageByThread instanceof CodexHistoryMap ? this.lineageByThread.takeReceipts() : []), + ...(this.terminalTurns instanceof CodexHistoryMap ? this.terminalTurns.takeReceipts() : []), + ...(this.acknowledgedSteeringCorrelations instanceof CodexHistoryMap ? this.acknowledgedSteeringCorrelations.takeReceipts() : []), + ...(this.emittedFileReferences instanceof CodexHistorySet ? this.emittedFileReferences.map.takeReceipts() : []), + ]); + } + + async prefetchHistory(collection: "terminal" | "file" | "steering" | "lineage", keys: string[]): Promise { + const cache = collection === "lineage" ? this.lineageByThread : collection === "terminal" ? this.terminalTurns : collection === "steering" ? this.acknowledgedSteeringCorrelations + : this.emittedFileReferences instanceof CodexHistorySet ? this.emittedFileReferences.map : null; + if (!(cache instanceof CodexHistoryMap)) return; + await this.deliveryWriter?.flush(); + await cache.prefetch(keys); + } + + deliveryCheckpoint(): Record { + for (const id of this.pendingRuntimeRequestSources.keys()) if (!this.pendingRuntimeRequestMap.has(id)) this.pendingRuntimeRequestSources.delete(id); + return JSON.parse(JSON.stringify({ + schema: "paperclip.codex.reducer.v1", runId: this.runId, sessionId: this.normalizedSessionId, threadId: this.opened.threadId, + scalars: Object.fromEntries(durableScalars.map((key) => [key, this[key]])), + ...Object.fromEntries(durableMaps.map((key) => [key, [...this[key]]])), + emittedFileReferences: [...this.emittedFileReferences], + pendingRequests: [...this.pendingRuntimeRequestMap].map(([id, pending]) => ({ request: pending.request, source: this.pendingRuntimeRequestSources.get(id) })), + })) as Record; + } + + publishEvent(event: PrpEvent): void { + this.capturedEventIds.getStore()?.push(event.sourceEventId); + if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) this.durableTerminal = { + event: JSON.parse(JSON.stringify(event)) as PrpEvent, semanticFingerprint: this.resultFingerprint, + }; + if (this.deliveryWriter) this.deliveryWriter.emit(event); + else this.eventQueue.push(event); + } + + closeEvents(): void { + if (!this.deliveryWriter) this.eventQueue.close(); + else void Promise.resolve().then(() => this.deliveryWriter!.flush()).then(() => this.eventQueue.close(), (error) => this.eventQueue.fail(error)); } requireActiveTurn(turnId: string, operation: string): void { @@ -417,7 +529,7 @@ export class CodexSessionState { this.activeTurnId = null; } this.terminal = true; - this.eventQueue.close(); + this.closeEvents(); // Notification failure can initiate cleanup before the owning runtime joins // it. Observe this background rejection immediately so a deleted remote // sandbox cannot crash the controller. The transport retains its original @@ -426,16 +538,30 @@ export class CodexSessionState { void this.transport.close(`protocol_failure:${code}`).catch(() => undefined); } + private allocateSource(): Pick { + const limit = this.deliveryWriter?.port.eventEpochs?.limit; + let transition: EventEpochTransition | undefined; + if (limit !== undefined && this.sourceSequence >= limit) { + transition = { schema: "paperclip.prp.event-epoch.v1", runId: this.runId, transitionId: randomUUID(), + fromEpoch: this.sourceSequenceEpoch, nextEpoch: randomUUID(), finalOrdinal: this.sourceSequence }; + this.sourceSequenceEpoch = transition.nextEpoch; + this.sourceSequence = 0; + } + if (!Number.isSafeInteger(this.sourceSequence) || this.sourceSequence >= Number.MAX_SAFE_INTEGER) throw new HarnessReconciliationError("normalized source sequence requires epoch support"); + const sourceSeq = ++this.sourceSequence; + return { sourceEventId: normalizedEventId(this.runnerInstanceId, this.runId, sourceSeq, this.sourceSequenceEpoch ?? undefined), sourceSeq, + ...(this.sourceSequenceEpoch ? { sourceEpoch: this.sourceSequenceEpoch } : {}), ...(transition ? { sourceEpochTransition: transition } : {}) }; + } + emit( eventType: PrpEvent["eventType"], payload: Record, refs: { turnId?: string; itemId?: string } = {}, ): void { - const sourceSeq = ++this.sourceSequence; - this.eventQueue.push({ + const source = this.allocateSource(); + this.publishEvent({ schema: "paperclip.prp.event.v1", - sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`, - sourceSeq, + ...source, sourceInstanceId: this.runnerInstanceId, sourceKind: "runner", runId: this.runId, @@ -506,11 +632,10 @@ export class CodexSessionState { | "session.goal.cleared", payload: Record, ): void { - const sourceSeq = ++this.sourceSequence; - this.eventQueue.push({ + const source = this.allocateSource(); + this.publishEvent({ schema: "paperclip.prp.event.v2", - sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`, - sourceSeq, + ...source, sourceInstanceId: this.runnerInstanceId, sourceKind: "runner", runId: this.runId, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts index 94e354fbd14..791681b1bd1 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts @@ -158,6 +158,8 @@ export function mapTerminalTurn( : "turn.completed"; state.cancelPendingRequests("turn_terminal"); state.activeTurnId = null; + state.workspaceChangesByTurn.delete(turnId); + state.itemChannels.clear(); state.turnStarted = false; state.emit( eventType, diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts index b552adcb28a..e8c020b75ff 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts @@ -186,14 +186,12 @@ describe("runner semantic MCP bridge", () => { expect(handler).toHaveBeenCalledTimes(2); }); - it("times out calls and honors MCP cancellation", async () => { - const starts: string[] = []; + it("times out calls", async () => { const bridge = await startRunnerToolBridge({ tools: [tool("documents.read")], timeoutMs: 20, - handler: ({ callId, signal }) => + handler: ({ signal }) => new Promise((_resolve, reject) => { - starts.push(callId); signal.addEventListener( "abort", () => reject(new Error("handler aborted")), @@ -217,29 +215,56 @@ describe("runner semantic MCP bridge", () => { content: [{ text: "Paperclip tool call timed out" }], }, }); + }); + + it("honors MCP cancellation", async () => { + const starts: string[] = []; + const bridge = await startRunnerToolBridge({ + tools: [tool("documents.read")], + timeoutMs: 5_000, + handler: ({ callId, signal }) => + new Promise((_resolve, reject) => { + starts.push(callId); + signal.addEventListener( + "abort", + () => reject(new Error("handler aborted")), + { once: true }, + ); + }), + }); + bridges.push(bridge); + const pending = rpc(bridge, { id: "cancel-me", method: "tools/call", params: { name: "documents.read", arguments: {} }, }); - await vi.waitFor(() => expect(starts).toContain("cancel-me"), { - interval: 1, - timeout: 15, - }); - expect( - ( - await rpc(bridge, { - method: "notifications/cancelled", - params: { requestId: "cancel-me" }, - }) - ).status, - ).toBe(202); - expect(await (await pending).json()).toMatchObject({ - result: { - isError: true, - content: [{ text: "Paperclip tool call cancelled" }], - }, - }); + // Attach a rejection handler immediately so a failed start gate cannot + // leave an unhandled loopback fetch while teardown closes the server. + void pending.catch(() => undefined); + try { + await vi.waitFor(() => expect(starts).toContain("cancel-me"), { + interval: 10, + timeout: 2_000, + }); + expect( + ( + await rpc(bridge, { + method: "notifications/cancelled", + params: { requestId: "cancel-me" }, + }) + ).status, + ).toBe(202); + expect(await (await pending).json()).toMatchObject({ + result: { + isError: true, + content: [{ text: "Paperclip tool call cancelled" }], + }, + }); + } finally { + await bridge.close(); + await pending.catch(() => undefined); + } }); it("preserves successful mutation identity when the result is oversized", async () => { diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 76ee579b90f..e2fde99b74e 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -29,6 +29,8 @@ export { DurablePrpControlPlane, DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES, inspectWarmRunTransition, + stageLegacyControlPlaneAuthority, + type LegacyAuthorityImportOptions, type DurablePrpControlPlaneOptions, type PrpWireConnection, type PrpWireAttachment, @@ -78,3 +80,19 @@ export * from "./generated/capability-contract.js"; export * from "./semantic-tools/index.js"; export * as acceptedCapabilitySemanticTools from "./semantic-tools/index.js"; export * from "./compatibility.js"; +export * from "./control-plane/durable-authority-store.js"; +export * from "./control-plane/current-authority-evidence.js"; +export { SqliteAuthorityStore } from "./control-plane/sqlite-authority-store.js"; + +export { readDurableControlPlaneState, parseAuthorityLocator, type AuthorityLocation, type AuthorityLocator, type ExternalAuthorityReader } from "./control-plane/authority-locator.js"; + +export { readIndexedLocalState, suspendIndexedRunnerState } from "./control-plane/indexed-local-state-reader.js"; +export type { IndexedLocalSnapshot } from "./control-plane/indexed-local-state-reader.js"; +export { IndexedInspectionDecoder } from "./control-plane/indexed-inspection-codec.js"; +export { migrateLegacySessionAuthority, type LegacySessionMigrationOptions, type PreparedLocalSession } from "./control-plane/legacy-authority-activation.js"; +export { assertNoPendingLegacyMigration } from "./control-plane/legacy-migration-gate.js"; + +export { advanceSourceCursor, normalizedEventId, isEventEpochTransition } from "./control-plane/event-epochs.js"; +export type { SourceCursor, EventEpochTransition } from "./control-plane/event-epochs.js"; + +export { isProviderProcessGeneration, provesProviderGenerationTransition } from "./control-plane/process-generation.js"; diff --git a/packages/paperclip-runner/src/live/cargo-test-binary.test.ts b/packages/paperclip-runner/src/live/cargo-test-binary.test.ts new file mode 100644 index 00000000000..4b247ad3e12 --- /dev/null +++ b/packages/paperclip-runner/src/live/cargo-test-binary.test.ts @@ -0,0 +1,37 @@ +import { resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; + +afterEach(() => vi.unstubAllEnvs()); + +describe("isolated Cargo test binary resolution", () => { + const workspace = resolve("/tmp/paperclip-runner-workspace"); + const executable = (name: string) => `${name}${process.platform === "win32" ? ".exe" : ""}`; + + it("preserves the normal target path when CARGO_TARGET_DIR is unset", () => { + vi.stubEnv("CARGO_TARGET_DIR", ""); + expect(resolveRunnerCargoTestBinary(workspace, "debug", "fake-codex-app-server")) + .toBe(resolve(workspace, "target", "debug", executable("fake-codex-app-server"))); + }); + + it("resolves absolute and workspace-relative isolated target directories", () => { + vi.stubEnv("CARGO_TARGET_DIR", "/tmp/isolated-cargo-target"); + expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd")) + .toBe(resolve("/tmp/isolated-cargo-target", "release", executable("paperclip-runnerd"))); + vi.stubEnv("CARGO_TARGET_DIR", "target/history-development"); + expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd")) + .toBe(resolve(workspace, "target/history-development", "release", executable("paperclip-runnerd"))); + }); + + it("uses staged/default resolution only when isolation is not configured", () => { + const fallback = vi.fn(() => "/staged/paperclip-runnerd"); + vi.stubEnv("CARGO_TARGET_DIR", ""); + expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback)) + .toBe("/staged/paperclip-runnerd"); + expect(fallback).toHaveBeenCalledOnce(); + vi.stubEnv("CARGO_TARGET_DIR", "target/isolated"); + expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback)) + .toBe(resolve(workspace, "target/isolated", "debug", executable("paperclip-runnerd"))); + expect(fallback).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/paperclip-runner/src/live/indexed-active-history.qualification.test.ts b/packages/paperclip-runner/src/live/indexed-active-history.qualification.test.ts new file mode 100644 index 00000000000..42e076faa59 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-active-history.qualification.test.ts @@ -0,0 +1,109 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { createHash } from "node:crypto"; +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +const seconds = Number(process.env.PAPERCLIP_ACTIVE_HISTORY_SOAK_SECONDS ?? 0); +// The short lane checks the fixture; only 259200 seconds is a 72-hour result. +it.skipIf(![60, 259200].includes(seconds))("keeps generating and durably consuming output during one active turn", async () => { + const root = await mkdtemp(join(tmpdir(), "indexed-active-history-")); + const identity = { runnerInstanceId: "active-history-runner", environmentLeaseId: "active-history-lease", normalizedSessionId: "active-history-session", + runId: "active-history-run", turnId: "active-history-turn", itemId: "active-history-item" }; + let authority: SqliteAuthorityStore | undefined; + const sink = await SqliteAuthorityStore.open({ path: join(root, "normalized.sqlite"), binding: "active-history-sink", create: true }); + const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--soak-output-on-steer", "--require-lightweight-history"], + stateDirectory: root, prpIdentity: identity, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "active-history", create: true }), + }); + const driver = new CodexAppServerDriver({ taskEnvelope: { schema: "paperclip.skillless_task.v1", objective: "Qualify a long active turn with growing output.", + completionContract: { revision: "active-history-v1", criteria: [{ id: "history", requirement: "Durably deliver ordinary output." }] }, constraints: [], expectedResultSchema: "paperclip.run_result.v1" }, + runnerInstanceId: identity.runnerInstanceId, approvalPolicy: "never", includeCollaborationModeInstructions: false, + environment: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: "/isolated/home", CODEX_HOME: "/isolated/codex-home", LANG: "C.UTF-8" }, + transportFactory: () => bundle.transport, requireProviderSessionIdentity: true }); + let session: Awaited> | undefined; + let generation = "0", events = 0, items = 0, bytes = 0, peakCurrentBytes = 0, probes = 0, maxProbeMs = 0; + let body: { id: string; offset: number; digest: ReturnType } | null = null; + let consumer: Promise | undefined, failure: unknown, closing = false; + let started = performance.now(), startedAt = new Date().toISOString(); + const report = async (status: string, error?: unknown) => { + const target = process.env.PAPERCLIP_HISTORY_REPORT; + if (!target) return; + await writeFile(`${target}.tmp`, JSON.stringify({ schema: "paperclip.indexed-active-history.v1", status, + provider: "credential-free fake Codex; actual driver and durable event sink", seconds, startedAt, + updatedAt: new Date().toISOString(), elapsedMs: performance.now() - started, events, items, bytes, probes, + peakCurrentBytes, maxProbeMs, runnerPid: bundle.evidence().runnerPid, providerPid: bundle.evidence().providerPid, + ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}) }, null, 2), { mode: 0o600 }); + await rename(`${target}.tmp`, target); + }; + try { + session = await driver.openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: root }); + const commit = async (event: import("../protocol/replay-contract.js").PrpEvent) => { + generation = await sink.commit({ expectedGeneration: generation, state: { lastSourceSeq: event.sourceSeq }, + records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record }] }); + }; + session.setEventCommitter!(commit); + consumer = (async () => { + for await (const event of session!.events()) { + await commit(event); await session!.acknowledgeEvent!(event); events++; + if (event.eventType === "output.body.chunk") { + const ref = event.payload.body as Record, text = String(event.payload.text), offset = Number(event.payload.offset); + if (offset === 0) { expect(body).toBeNull(); body = { id: String(ref.bodyId), offset: 0, digest: createHash("sha256") }; } + expect(body?.id).toBe(ref.bodyId); expect(body?.offset).toBe(offset); + expect(createHash("sha256").update(text).digest("hex")).toBe(event.payload.sha256); + body!.digest.update(text); body!.offset += Buffer.byteLength(text); bytes += Buffer.byteLength(text); + } + if (event.eventType === "item.completed" && event.itemId?.startsWith("soak-output-")) { + const ref = event.payload.outputBody as Record; + expect(body?.id).toBe(ref.bodyId); expect(String(body!.offset)).toBe(ref.byteLength); + expect(body!.digest.digest("hex")).toBe(ref.sha256); + body = null; items++; + } + if (!closing && ["turn.completed", "turn.failed", "session.failed"].includes(event.eventType)) throw new Error(`active soak ended early: ${event.eventType}`); + } + if (!closing) throw new Error("active soak event stream closed early"); + })().catch(error => { failure = error; }); + const turn = await session.startTurn({ message: { text: "Keep this turn active while generating qualification output." } }); + const owner = bundle.evidence(); + started = performance.now(); startedAt = new Date().toISOString(); + await report("running"); + while (performance.now() - started < seconds * 1000) { + if (failure) throw failure; + const before = performance.now(), previous = items; + await session.steer!({ turnId: turn.turnId, message: { text: `Produce qualification output ${probes + 1}.` }, correlationId: `soak-steer-${probes + 1}` }); + await bundle.transport.request("thread/read", { threadId: session.ids().driverSessionId, includeTurns: false }); + const deadline = performance.now() + 30_000; + while (items === previous && !failure && performance.now() < deadline) await new Promise(resolve => setTimeout(resolve, 25)); + if (failure) throw failure; + expect(items).toBeGreaterThan(previous); + maxProbeMs = Math.max(maxProbeMs, performance.now() - before); probes++; + expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, providerPid: owner.providerPid, runnerExited: false }); + peakCurrentBytes = Math.max(peakCurrentBytes, Buffer.byteLength(JSON.stringify((await authority!.load())!.state))); + expect(peakCurrentBytes).toBeLessThan(8 * 1024 * 1024); + await report("running"); + await new Promise(resolve => setTimeout(resolve, Math.min(seconds === 60 ? 1000 : 30_000, Math.max(0, seconds * 1000 - (performance.now() - started))))); + } + closing = true; await report("settling"); + await session.close({ reason: "active history qualification complete" }); await consumer; session = undefined; + if (failure) throw failure; + await bundle.transport.close(); await authority!.close(); authority = undefined; await sink.close(); + expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); + await rm(root, { recursive: true, force: true }); + await report("passed"); + } catch (error) { await report("failed", error); throw error; } + finally { closing = true; await session?.close({ reason: "active history qualification cleanup" }); await consumer; + await bundle.transport.close(); await authority?.close(); await sink.close(); await rm(root, { recursive: true, force: true }); } +}, (seconds + 3600) * 1000); diff --git a/packages/paperclip-runner/src/live/indexed-authority-archive.test.ts b/packages/paperclip-runner/src/live/indexed-authority-archive.test.ts new file mode 100644 index 00000000000..3422fb9c898 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-authority-archive.test.ts @@ -0,0 +1,147 @@ +import { nativeStorageCommand } from "../control-plane/native-storage-command.js"; +import { authorityJson } from "../control-plane/durable-authority-store.js"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, it } from "vitest"; +import { finishIndexedRunnerArchive, prepareIndexedRunnerArchive, publishIndexedArchiveHead, readIndexedArchiveHead } from "./indexed-authority-archive.js"; +import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { runnerdRecoveryInternals } from "./runnerd-codex-transport.js"; + +it("reopens the exact indexed controller after cold rotation without reading older epochs", async () => { + const root = mkdtempSync(resolve(tmpdir(), "indexed-cold-rotation-")); + const controller = resolve(root, "control-plane"), runner = resolve(root, "runner"); + mkdirSync(controller, { mode: 0o700 }); mkdirSync(runner, { mode: 0o700 }); + const identity = { runnerInstanceId: "runner-test", environmentLeaseId: "environment-test", normalizedSessionId: "session-test", runId: "run-old", turnId: "turn-old", itemId: "item-old" }; + const desired = { ...identity, runId: "run-new", turnId: "turn-new", itemId: "item-new" }; + const store = await SqliteAuthorityStore.open({ path: resolve(controller, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + try { + const core = await DurablePrpControlPlane.open({ stateDirectory: controller, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: store }); + const state = structuredClone(core.store.state) as unknown as Record; + await store.close(); + const path = resolve(runner, "runner-state.sqlite"); + const db = new DatabaseSync(path); + try { + chmodSync(path, 0o600); + writeFileSync(`${path}.lifetime`, "", { mode: 0o600 }); + db.exec("CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB)"); + db.prepare("INSERT INTO store_binding VALUES (1,1,'runner/session')").run(); + const bytes = Buffer.from(JSON.stringify({ ...identity, schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended", nextSourceSeq: 1, ackedSourceSeq: 0 })); + db.prepare("INSERT INTO current_state VALUES ('runner',12,?,?)").run(bytes, createHash("sha256").update(bytes).digest()); + } finally { db.close(); } + writeFileSync(resolve(runner, "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/session" }), { mode: 0o600 }); + expect(await runnerdRecoveryInternals.rotateLocalAuthorityEpoch(root, state, desired)).toEqual(state); + const head = readIndexedArchiveHead(resolve(root, "authority-epochs")); + expect(head).not.toBeNull(); + // The legacy scan would select this newer, invalid archive first. + const unrelated = resolve(root, "authority-epochs", "unrelated", "control-plane"); + mkdirSync(unrelated, { recursive: true, mode: 0o700 }); + writeFileSync(resolve(unrelated, "control-plane-state.json"), "not valid authority", { mode: 0o600 }); + expect(await runnerdRecoveryInternals.latestArchivedControlPlaneState(root, desired)).toEqual(state); + await expect(runnerdRecoveryInternals.latestArchivedControlPlaneState(root, { ...desired, normalizedSessionId: "wrong-session" })).rejects.toThrow("archive_conflict"); + expect(await runnerdRecoveryInternals.rotateLocalAuthorityEpoch(root, state, desired)).toEqual(state); + } finally { await store.close(); rmSync(root, { recursive: true, force: true }); } +}); + +it("routes interrupted cold rotation directly despite unavailable old archives", () => { + const root = mkdtempSync(resolve(tmpdir(), "indexed-archive-head-")); + const archive = resolve(root, `epoch-${"a".repeat(24)}`); + mkdirSync(archive, { mode: 0o700 }); + try { + expect(readIndexedArchiveHead(root)).toBeNull(); + publishIndexedArchiveHead(archive); + // Unrelated retained epochs must never be enumerated or read to resume. + for (let index = 0; index < 128; index++) { + symlinkSync(resolve(root, "unavailable-volume", String(index)), resolve(root, `old-${index}`)); + } + expect(readIndexedArchiveHead(root)).toBe(archive); + const next = resolve(root, `epoch-${"b".repeat(24)}`); + mkdirSync(next, { mode: 0o700 }); + publishIndexedArchiveHead(next); + rmSync(archive, { recursive: true }); + expect(readIndexedArchiveHead(root)).toBe(next); + rmSync(next, { recursive: true }); + expect(() => readIndexedArchiveHead(root)).toThrow(); + // Never treat a broken pointer as legacy state and fall back to old epochs. + rmSync(resolve(root, "current-archive.json")); + symlinkSync(resolve(root, "missing"), resolve(root, "current-archive.json")); + expect(() => readIndexedArchiveHead(root)).toThrow("archive_unsafe"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +it.each([ + [1, "12"], [2, "12"], [2, "9223372036854775807"], + [2, "r:59365b32-c44d-46f2-8550-a7b5bba1bd68"], +] as const)("recovers an interrupted schema %s SQLite archive at revision %s without scanning or copying its history", async (schema, generation) => { + const root = mkdtempSync(resolve(tmpdir(), "indexed-archive-")); + const runner = resolve(root, "runner"), archive = resolve(root, "archive"); + mkdirSync(runner, { mode: 0o700 }); mkdirSync(archive, { mode: 0o700 }); + mkdirSync(resolve(root, "control-plane"), { mode: 0o700 }); + const path = resolve(runner, "runner-state.sqlite"); + const db = new DatabaseSync(path); + chmodSync(path, 0o600); + writeFileSync(`${path}.lifetime`, "", { mode: 0o600 }); + db.exec("CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB); CREATE TABLE receipts(key TEXT PRIMARY KEY,bytes BLOB)"); + db.prepare("INSERT INTO store_binding VALUES (1,?,'runner/session')").run(schema); + if (schema === 2) db.exec("CREATE TABLE receipt_prepare(singleton INTEGER PRIMARY KEY,state_key TEXT,generation INTEGER,body BLOB,digest BLOB)"); + const bytes = Buffer.from(JSON.stringify({ lifecycle: "suspended", nextSourceSeq: 2, ackedSourceSeq: 1 })); + db.prepare("INSERT INTO current_state VALUES ('runner',?,?,?)").run(generation.startsWith("r:") ? generation : BigInt(generation), bytes, createHash("sha256").update(bytes).digest()); + db.prepare("INSERT INTO receipts VALUES ('ancient',?)").run(Buffer.from("original outcome")); + db.close(); + if (schema === 2) { + const shards = resolve(runner, "runner-state.sqlite.receipts"); + mkdirSync(shards, { mode: 0o700 }); + writeFileSync(resolve(shards, "partition-fixture"), "immutable receipt evidence", { mode: 0o600 }); + } + const routing = resolve(runner, "runner-state.sqlite.routing"); + mkdirSync(routing, { mode: 0o700 }); + writeFileSync(resolve(routing, "routing-fixture"), "immutable routing evidence", { mode: 0o600 }); + writeFileSync(resolve(runner, "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "runner/session" }), { mode: 0o600 }); + try { + await prepareIndexedRunnerArchive(runner, archive); + await expect(finishIndexedRunnerArchive(runner, archive)).rejects.toThrow("controller_missing"); + renameSync(resolve(root, "control-plane"), resolve(archive, "control-plane")); + // Simulate loss of the controller after the DB rename and before locator publication. + renameSync(path, resolve(archive, "runner-state.sqlite")); + expect(await finishIndexedRunnerArchive(runner, archive)).toBe(true); + expect(existsSync(resolve(runner, "runner-state.json"))).toBe(false); + expect(existsSync(resolve(archive, "runner-state.sqlite.routing", "routing-fixture"))).toBe(true); + expect(existsSync(resolve(runner, "runner-state.sqlite.routing"))).toBe(false); + if (schema === 2) { + expect(existsSync(resolve(archive, "runner-state.sqlite.receipts", "partition-fixture"))).toBe(true); + expect(existsSync(resolve(runner, "runner-state.sqlite.receipts"))).toBe(false); + } + expect(await finishIndexedRunnerArchive(runner, archive)).toBe(true); + expect(await readIndexedLocalState(resolve(archive, "runner-state.json"))).toMatchObject({ generation, state: { lifecycle: "suspended", outbox: [] } }); + const restored = new DatabaseSync(resolve(archive, "runner-state.sqlite"), { readOnly: true }); + try { expect(Buffer.from(restored.prepare("SELECT bytes FROM receipts WHERE key='ancient'").get()!.bytes as Uint8Array).toString()).toBe("original outcome"); } + finally { restored.close(); } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +it("fences standalone controller archive against an open storage owner and stale authority", async () => { + const root = mkdtempSync(resolve(tmpdir(), "indexed-controller-archive-")); + const controller = resolve(root, "controller"), target = resolve(root, "archived"); + mkdirSync(controller, { mode: 0o700 }); + const identity = { runnerInstanceId: "runner-test", environmentLeaseId: "environment-test", normalizedSessionId: "session-test", runId: "run-old", turnId: "turn-old", itemId: "item-old" }; + const store = await SqliteAuthorityStore.open({ path: resolve(controller, "authority.sqlite"), binding: JSON.stringify(identity), create: true }); + try { + const core = await DurablePrpControlPlane.open({ stateDirectory: controller, identity, expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: store }); + const storedState = (await store.load())!.state; + const digest = createHash("sha256").update(authorityJson(storedState)).digest("hex"); + const command = ["archive-controller", "--directory", controller, "--destination", target, "--digest", digest]; + await expect(nativeStorageCommand(command)).rejects.toThrow("open connection"); + await store.close(); + await expect(nativeStorageCommand([...command.slice(0, -1), "0".repeat(64)])).rejects.toThrow("authority changed"); + expect(existsSync(controller)).toBe(true); + await nativeStorageCommand(command); + expect(existsSync(controller)).toBe(false); + expect(existsSync(resolve(target, "authority.sqlite"))).toBe(true); + const reopened = await SqliteAuthorityStore.open({ path: resolve(target, "authority.sqlite"), binding: JSON.stringify(identity), create: false }); + try { expect((await reopened.load())?.state).toEqual(storedState); } finally { await reopened.close(); } + } finally { await store.close(); rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/paperclip-runner/src/live/indexed-authority-archive.ts b/packages/paperclip-runner/src/live/indexed-authority-archive.ts new file mode 100644 index 00000000000..330bb5a7874 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-authority-archive.ts @@ -0,0 +1,117 @@ +import { randomUUID } from "node:crypto"; +import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, openSync, readSync, renameSync, writeFileSync } from "node:fs"; +import { basename, dirname, resolve } from "node:path"; +import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js"; +import { nativeStorageCommand } from "../control-plane/native-storage-command.js"; +import { authorityGeneration } from "../control-plane/durable-authority-store.js"; + +const files = ["runner-state.sqlite", "runner-state.sqlite-wal", "runner-state.sqlite-shm", "runner-state.sqlite.receipts", "runner-state.sqlite.routing", "runner-state.json"]; +type Transfer = { schema: "paperclip.runner.indexed-archive.v1"; generation: string; stateSha256: string; files: string[] }; + +function privateDirectory(path: string): void { + const stat = lstatSync(path); + if (!stat.isDirectory() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_indexed_archive_unsafe"); +} + +export function syncArchiveDirectory(path: string): void { + if (process.platform === "win32") return; + const fd = openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fsyncSync(fd); } finally { closeSync(fd); } +} + +function privateRegular(path: string): void { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("native_indexed_archive_unsafe"); +} + +/** Publish before moving the active controller directory. This is a bounded + * routing hint; the caller must still validate the archived authority binding. */ +export function publishIndexedArchiveHead(archive: string): void { + const root = dirname(archive); + privateDirectory(root); + privateDirectory(archive); + const name = basename(archive); + if (!/^epoch-[a-f0-9]{24}$/.test(name)) throw new Error("native_indexed_archive_invalid"); + const target = resolve(root, "current-archive.json"); + if (existsSync(target)) privateRegular(target); + const temporary = `${target}.${randomUUID()}.tmp`; + writeFileSync(temporary, JSON.stringify({ schema: "paperclip.runner.indexed-archive-head.v1", archive: name }), { mode: 0o600, flag: "wx", flush: true }); + renameSync(temporary, target); + syncArchiveDirectory(root); +} + +/** Once a head exists, missing/corrupt current routing fails closed. It must + * never fall back to choosing an older archive by timestamp or scanning. */ +export function readIndexedArchiveHead(archivesRoot: string): string | null { + const path = resolve(archivesRoot, "current-archive.json"); + // lstat distinguishes an absent pointer from a dangling symlink. + try { lstatSync(path); } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + privateDirectory(archivesRoot); + privateRegular(path); + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > 1024) throw new Error("native_indexed_archive_invalid"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error("native_indexed_archive_changed"); + offset += count; + } + const value = JSON.parse(bytes.toString()) as { schema?: unknown; archive?: unknown }; + if (value.schema !== "paperclip.runner.indexed-archive-head.v1" || typeof value.archive !== "string" || !/^epoch-[a-f0-9]{24}$/.test(value.archive)) throw new Error("native_indexed_archive_invalid"); + const archive = resolve(archivesRoot, value.archive); + privateDirectory(archive); + return archive; + } finally { closeSync(fd); } +} + +function readTransfer(archive: string): Transfer | null { + const path = resolve(archive, "runner-transfer.json"); + if (!existsSync(path)) return null; + privateDirectory(archive); + privateRegular(path); + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.size > 4096) throw new Error("native_indexed_archive_unsafe"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error("native_indexed_archive_changed"); + offset += count; + } + const value = JSON.parse(bytes.toString()) as Transfer; + if (typeof value.generation !== "string") throw new Error("native_indexed_archive_invalid"); + try { authorityGeneration(value.generation); } catch { throw new Error("native_indexed_archive_invalid"); } + if (value.schema !== "paperclip.runner.indexed-archive.v1" || value.generation === "0" || + !/^[a-f0-9]{64}$/.test(value.stateSha256) || !Array.isArray(value.files) || + value.files.length < 2 || value.files.length > files.length || new Set(value.files).size !== value.files.length || + !value.files.includes("runner-state.sqlite") || !value.files.includes("runner-state.json") || value.files.some(file => !files.includes(file))) throw new Error("native_indexed_archive_invalid"); + return value; + } finally { closeSync(fd); } +} + +/** Only called after the existing caller proves a suspended, exclusively owned + * runner. The intent survives a crash between renames of SQLite companion files. */ +export async function prepareIndexedRunnerArchive(runner: string, archive: string, runnerBinary?: string): Promise { + privateDirectory(runner); + privateDirectory(archive); + const snapshot = await readIndexedLocalState(resolve(runner, "runner-state.json"), { runnerBinary }); + if (snapshot.state.lifecycle !== "suspended") throw new Error("native_indexed_archive_unsettled"); + await nativeStorageCommand(["archive-prepare", "--directory", runner, "--destination", archive, + "--generation", snapshot.generation, "--digest", snapshot.stateDigest], runnerBinary); +} + +/** The controller directory must already have moved durably. The native owner + * completes the fixed renames while holding both store lifetime fences. */ +export async function finishIndexedRunnerArchive(runner: string, archive: string, runnerBinary?: string): Promise { + if (!readTransfer(archive)) return false; + await nativeStorageCommand(["archive-finish", "--directory", runner, "--destination", archive], runnerBinary); + return true; +} diff --git a/packages/paperclip-runner/src/live/indexed-channel-rekey.test.ts b/packages/paperclip-runner/src/live/indexed-channel-rekey.test.ts new file mode 100644 index 00000000000..a66f7daace4 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-channel-rekey.test.ts @@ -0,0 +1,76 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +// Real native transport/process/storage; the provider is credential-free. +// The small connection limit exercises the normal production re-auth path. +it.each([false, true])("rekeys repeatedly during one active provider turn (command epochs: %s)", async (commandEpochs) => { + const root = await mkdtemp(join(tmpdir(), "indexed-channel-rekey-")); + const callsPath = join(root, "calls.log"); + let authority: SqliteAuthorityStore | undefined; + const cores: DurablePrpControlPlane[] = []; + const open = DurablePrpControlPlane.open.bind(DurablePrpControlPlane); + const spy = vi.spyOn(DurablePrpControlPlane, "open").mockImplementation(async options => { + const core = await open({ ...options, secureChannelFrameLimit: 16, ...(commandEpochs ? { commandEpochLimit: 8 } : {}) }); + cores.push(core); return core; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--record-process-start", "--call-log", callsPath, "--require-lightweight-history"], + stateDirectory: root, runnerReconnectGraceMs: 10_000, + authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "channel-rekey", create: true }), + }); + let consumer: Promise | undefined, failure: unknown; + try { + const opened = await bundle.transport.request("thread/start", { cwd: root, dynamicTools: [] }); + consumer = (async () => { + for await (const notification of bundle.transport.notifications()) { + const port = bundle.transport.normalizedDelivery!()!; + if (notification.paperclipDelivery) await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: notification.paperclipDelivery, + driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [] }); + } + })().catch(error => { failure = error; }); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Keep the same turn active across encrypted connections." }] }); + const owner = bundle.evidence(), core = cores[0]!, identity = structuredClone(core.store.state.identity); + const firstCommand = structuredClone(core.store.state.commands[0]!); + const epochs = new Set(); + const threadId = (opened.thread as { id: string }).id; + for (let index = 0; index < 64; index++) { + await bundle.transport.request("thread/read", { threadId, includeTurns: false }); + if (failure) throw failure; + epochs.add(core.store.state.indexedState?.controllerEpoch); + expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false }); + } + if (commandEpochs) { + expect(epochs.size).toBeGreaterThanOrEqual(7); + expect(await core.queueCommand(firstCommand.type, firstCommand.payload, firstCommand.commandId)).toMatchObject({ commandId: firstCommand.commandId, status: firstCommand.status }); + } + expect(core.store.state.connectionCount).toBeGreaterThanOrEqual(4); + expect(core.store.state.identity).toEqual(identity); + expect(core.store.state.malformedFrames).toBe(0); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter(call => call === "process-start")).toHaveLength(1); + expect(calls.filter(call => call === "thread/start")).toHaveLength(1); + expect(calls.filter(call => call === "turn/start")).toHaveLength(1); + expect(calls).not.toContain("turn/interrupt"); + expect(calls).not.toContain("thread/resume"); + const first = (await authority!.readEvents(identity.runId, "0", 1, 1024 * 1024)).records[0]!; + expect(await authority!.getRecord(identity.runId, "event", first.id)).toEqual(first); + } finally { + await bundle.transport.close(); await consumer; await authority?.close(); spy.mockRestore(); + await rm(root, { recursive: true, force: true }); + } +}, 90_000); diff --git a/packages/paperclip-runner/src/live/indexed-continuations.qualification.test.ts b/packages/paperclip-runner/src/live/indexed-continuations.qualification.test.ts new file mode 100644 index 00000000000..08bc5da5f14 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-continuations.qualification.test.ts @@ -0,0 +1,81 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +// Explicit protocol scale qualification, with real runner/fake-provider +// processes and real storage. This does not stand in for Product E2E or a soak. +it.skipIf(process.env.PAPERCLIP_HISTORY_CONTINUATIONS !== "1000")("continues one provider through 1,000 indexed run epochs", async () => { + const directory = await mkdtemp(join(tmpdir(), "indexed-continuations-")); + let authority: SqliteAuthorityStore | undefined; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(directory, "fake-state.json"), "--durable-turn-ids"], + stateDirectory: directory, + prpIdentity: { runnerInstanceId: "history-runner", environmentLeaseId: "history-lease", normalizedSessionId: "history-session", runId: "history-run-0", turnId: "history-turn-0", itemId: "history-item-0" }, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + authorityStoreFactory: async (_identity, root) => authority ??= await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: "history-continuations", create: true }), + }); + bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + let completed: (() => void) | null = null; + let consumerFailure: unknown; + let consumer: Promise | undefined; + const started = Date.now(); + const durations: number[] = []; + try { + await bundle.transport.request("initialize", {}); + const opened = await bundle.transport.request("thread/start", { + cwd: directory, + dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }], + completionContract: { revision: "history-contract", criterionIds: ["objective"] }, + }); + const owner = bundle.evidence(); + consumer = (async () => { + for await (const notification of bundle.transport.notifications()) { + const port = bundle.transport.normalizedDelivery!()!; + if (notification.paperclipDelivery) await port.commit({ + expectedRevision: port.load()?.revision ?? 0, + raw: notification.paperclipDelivery, + driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [], + }); + if (notification.method === "turn/completed") completed?.(); + } + })().catch((error) => { consumerFailure = error; completed?.(); }); + bundle.transport.setBeforeRunRotation!(async () => { if (consumerFailure) throw consumerFailure; }); + for (let index = 0; index < 1000; index++) { + const before = Date.now(); + if (index) await bundle.transport.attachRun!({ runId: `history-run-${index}`, turnId: `history-turn-${index}`, itemId: `history-item-${index}` }); + let timer: ReturnType | undefined; + const terminal = new Promise((resolve, reject) => { completed = resolve; timer = setTimeout(() => reject(new Error(`turn ${index} timed out`)), 20_000); }); + try { + await bundle.transport.request("turn/start", { input: [{ type: "text", text: `Continue the same task: ${index}` }] }); + await terminal; + if (consumerFailure) throw consumerFailure; + } finally { clearTimeout(timer); } + expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false }); + const current = await authority!.load(); + expect(Buffer.byteLength(JSON.stringify(current!.state))).toBeLessThan(256 * 1024); + durations.push(Date.now() - before); + } + const sorted = [...durations].sort((a, b) => a - b); + const report = { schema: "paperclip.indexed-continuations-qualification.v1", turns: durations.length, elapsedMs: Date.now() - started, turnP95Ms: sorted[949], first100MeanMs: durations.slice(0,100).reduce((a,b)=>a+b)/100, last100MeanMs: durations.slice(-100).reduce((a,b)=>a+b)/100, sameRunnerAndProvider: true, provider: "credential-free fake Codex process", opened }; + if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, JSON.stringify(report, null, 2), { mode: 0o600 }); + } finally { + await bundle.transport.close(); + await consumer; + await authority?.close(); + expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); + await rm(directory, { recursive: true, force: true }); + } +}, 30 * 60_000); diff --git a/packages/paperclip-runner/src/live/indexed-event-epochs.test.ts b/packages/paperclip-runner/src/live/indexed-event-epochs.test.ts new file mode 100644 index 00000000000..c1f38cbacbd --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-event-epochs.test.ts @@ -0,0 +1,110 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { advanceSourceCursor, type SourceCursor } from "../control-plane/event-epochs.js"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +it.each([undefined, 4])("crosses raw event epochs with normalized limit %s while preserving output and provider ownership", async (normalizedEventEpochLimit) => { + const root = await mkdtemp(join(tmpdir(), "indexed-event-epochs-")); + const callsPath = join(root, "calls.log"); + const identity = { runnerInstanceId: "epoch-runner", environmentLeaseId: "epoch-lease", normalizedSessionId: "epoch-session", + runId: "epoch-run", turnId: "epoch-turn", itemId: "epoch-item" }; + let authority: SqliteAuthorityStore | undefined, core: DurablePrpControlPlane | undefined; + const open = DurablePrpControlPlane.open.bind(DurablePrpControlPlane); + const spy = vi.spyOn(DurablePrpControlPlane, "open").mockImplementation(async options => core = await open({ ...options, eventEpochLimit: 8, commandEpochLimit: 8, secureChannelFrameLimit: 64, normalizedEventEpochLimit })); + const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--hold-turn", "--soak-output-on-steer", "--record-process-start", "--call-log", callsPath, "--require-lightweight-history"], + stateDirectory: root, prpIdentity: identity, + authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"event-epochs",create:true}), + }); + const driver = new CodexAppServerDriver({ taskEnvelope: {schema:"paperclip.skillless_task.v1",objective:"Keep producing output across event epochs.", + completionContract:{revision:"event-epochs-v1",criteria:[{id:"output",requirement:"Deliver every output body exactly."}]},constraints:[],expectedResultSchema:"paperclip.run_result.v1"}, + runnerInstanceId:identity.runnerInstanceId,approvalPolicy:"never",includeCollaborationModeInstructions:false, + environment:{PATH:process.env.PATH ?? "/usr/bin:/bin",HOME:"/isolated/home",CODEX_HOME:"/isolated/codex-home",LANG:"C.UTF-8"},transportFactory:()=>bundle.transport,requireProviderSessionIdentity:true }); + let session: Awaited> | undefined, consumer: Promise | undefined, failure: unknown; + const epochs = new Set(), outputs = new Map(), ids = new Set(); + let completed = 0; + let cursor: SourceCursor = { sourceSeq: 0 }; + const normalizedEpochs = new Set(); + try { + session = await driver.openSession({runId:identity.runId,normalizedSessionId:identity.normalizedSessionId,workingDirectory:root}); + consumer = (async () => { for await (const event of session!.events()) { + expect(ids.has(event.sourceEventId)).toBe(false); ids.add(event.sourceEventId); + cursor = advanceSourceCursor(cursor, event); + normalizedEpochs.add(event.sourceEpoch); + if (event.eventType === "output.body.chunk") { + const id = String((event.payload.body as Record).bodyId); + outputs.set(id, (outputs.get(id) ?? "") + String(event.payload.text)); + } + if (event.eventType === "item.completed" && event.itemId?.startsWith("soak-output-")) { + const body = event.payload.outputBody as Record; + expect(outputs.get(String(body.bodyId))).toBe(`${event.itemId}: ${"s".repeat(8192)}`); + completed++; + } + await session!.acknowledgeEvent!(event); + } })().catch(error => {failure=error;}); + const turn = await session.startTurn({message:{text:"Hold this turn across namespace changes."}}); + const owner = bundle.evidence(); + for (let index=1;index<=16;index++) { + await session.steer!({turnId:turn.turnId,message:{text:`Output ${index}`},correlationId:`epoch-steer-${index}`}); + await vi.waitFor(() => { if(failure) throw failure; expect(completed).toBe(index); },{timeout:20_000,interval:10}); + epochs.add(core!.store.state.indexedState?.sourceEpoch); + expect(bundle.evidence()).toMatchObject({runnerPid:owner.runnerPid,providerPid:owner.providerPid,runnerExited:false}); + if(index%3===0) core!.disconnectActiveRunner(); + } + expect(epochs.size).toBeGreaterThanOrEqual(5); + if (normalizedEventEpochLimit === undefined) expect(normalizedEpochs.size).toBe(1); + else expect(normalizedEpochs.size).toBeGreaterThan(10); + expect(core!.store.state.identity).toEqual(identity); + const calls=(await readFile(callsPath,"utf8")).trim().split(/\r?\n/); + expect(calls.filter(call=>call==="process-start")).toHaveLength(1); + expect(calls.filter(call=>call==="thread/start")).toHaveLength(1); + expect(calls.filter(call=>call==="turn/start")).toHaveLength(1); + expect(calls).not.toContain("turn/interrupt"); expect(calls).not.toContain("thread/resume"); + await session.close({reason:"event epoch qualification complete"}); await consumer; session=undefined; + if(failure) throw failure; + } finally { await session?.close({reason:"event epoch cleanup"}); await consumer; await bundle.transport.close(); await authority?.close(); spy.mockRestore(); await rm(root,{recursive:true,force:true}); } +},90_000); + +it("keeps indexed replay and renewed event namespaces through warm run attachment", async () => { + const root=await mkdtemp(join(tmpdir(),"indexed-event-warm-")); + let authority:SqliteAuthorityStore|undefined, core:DurablePrpControlPlane|undefined, consumer:Promise|undefined, failure:unknown; + const open=DurablePrpControlPlane.open.bind(DurablePrpControlPlane); + const spy=vi.spyOn(DurablePrpControlPlane,"open").mockImplementation(async options=>core=await open({...options,eventEpochLimit:4,commandEpochLimit:8})); + const bundle=createCapabilityRunnerdCodexTransport({runnerBinary:runnerTestBinary(), + codexCommand:resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"),codexArgs:["--state-file",join(root,"fake.json"),"--durable-turn-ids"], + stateDirectory:root,lifecyclePolicy:{mode:"warm",idleTimeoutMs:60_000}, + authorityStoreFactory:async (_identity,directory)=>authority??=await SqliteAuthorityStore.open({path:join(directory,"authority.sqlite"),binding:"event-epochs-warm",create:true}), + }); + bundle.transport.setServerRequestHandler(async()=>({success:true,contentItems:[]})); + let completed=0; + try { + await bundle.transport.request("thread/start",{cwd:root,dynamicTools:[{name:"get_task_context",description:"Read the active task.",inputSchema:{type:"object",properties:{},additionalProperties:false}}],completionContract:{revision:"epoch-warm-v1",criterionIds:["objective"]}}); + const owner=bundle.evidence(); + consumer=(async()=>{for await(const notification of bundle.transport.notifications()) { + const port=bundle.transport.normalizedDelivery!()!; + if(notification.paperclipDelivery) await port.commit({expectedRevision:port.load()?.revision??0,raw:notification.paperclipDelivery,driver:{},events:[]}); + if(notification.method==="turn/completed") completed++; + }})().catch(error=>{failure=error;}); + for(let round=1;round<=3;round++) { + await bundle.transport.request("turn/start",{input:[{type:"text",text:`Run ${round}`}]}); + await vi.waitFor(()=>{if(failure) throw failure;expect(completed).toBe(round);expect(core!.store.state.indexedState?.sourceEpoch).toBeTruthy();},{timeout:20_000,interval:10}); + expect(bundle.evidence()).toMatchObject({runnerPid:owner.runnerPid,providerPid:owner.providerPid,runnerExited:false}); + if(round<3) await bundle.transport.attachRun!({runId:`warm-run-${round}`,turnId:`warm-turn-${round}`,itemId:`warm-item-${round}`}); + } + if(failure) throw failure; + } finally {await bundle.transport.close();await consumer;await authority?.close();spy.mockRestore();await rm(root,{recursive:true,force:true});} +},90_000); diff --git a/packages/paperclip-runner/src/live/indexed-history-pipeline.qualification.test.ts b/packages/paperclip-runner/src/live/indexed-history-pipeline.qualification.test.ts new file mode 100644 index 00000000000..a35f5fe49f7 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-history-pipeline.qualification.test.ts @@ -0,0 +1,130 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver-impl.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +const mib = Number(process.env.PAPERCLIP_HISTORY_PIPELINE_MIB ?? 0); +it.skipIf(![64, 10240].includes(mib))("ingests large ordinary provider output through runner, controller, normalizer and durable event sink", async () => { + const root = await mkdtemp(join(tmpdir(), "indexed-history-pipeline-")); + let authority: SqliteAuthorityStore | undefined; + const sink = await SqliteAuthorityStore.open({ path: join(root, "normalized.sqlite"), binding: "pipeline-sink", create: true }); + const identity = { runnerInstanceId: "pipeline-runner", environmentLeaseId: "pipeline-lease", normalizedSessionId: "pipeline-session", runId: "pipeline-run", turnId: "pipeline-turn", itemId: "pipeline-item" }; + const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "fake.json"), "--durable-turn-ids", "--history-growth-mib", String(mib)], + stateDirectory: root, prpIdentity: identity, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + authorityStoreFactory: async (_identity, directory) => authority ??= await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "history-pipeline", create: true }), + }); + const driver = new CodexAppServerDriver({ taskEnvelope: { schema: "paperclip.skillless_task.v1", objective: "Generate ordinary output to qualify durable history.", + completionContract: { revision: "pipeline-v1", criteria: [{ id: "history", requirement: "Deliver all output." }] }, constraints: [], expectedResultSchema: "paperclip.run_result.v1" }, + runnerInstanceId: identity.runnerInstanceId, approvalPolicy: "never", includeCollaborationModeInstructions: false, + environment: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: "/isolated/home", CODEX_HOME: "/isolated/codex-home", LANG: "C.UTF-8" }, + transportFactory: () => bundle.transport, requireProviderSessionIdentity: true }); + let session: Awaited> | undefined; + let generation = "0", events = 0, normalizedBytes = 0, peakCurrentBytes = 0, historyItems = 0, bodyBytes = 0; + let body: { id: string; offset: number; hash: ReturnType } | null = null; + const hash = createHash("sha256"), started = Date.now(); + const report = async (status: string, extra = {}) => { + if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, JSON.stringify({ schema: "paperclip.indexed-history-pipeline.v1", status, provider: "credential-free fake Codex", mib, events, historyItems, bodyBytes, normalizedBytes, peakCurrentBytes, elapsedMs: Date.now() - started, ...extra }, null, 2), { mode: 0o600 }); + }; + try { + session = await driver.openSession({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, workingDirectory: root }); + const commit = async (event: import("../protocol/replay-contract.js").PrpEvent) => { + generation = await sink.commit({ expectedGeneration: generation, state: { lastSourceSeq: event.sourceSeq }, records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record }] }); + }; + session.setEventCommitter!(commit); + const consumeUntilTerminal = async () => { + for await (const event of session!.events()) { + const bytes = JSON.stringify(event); hash.update(bytes); normalizedBytes += Buffer.byteLength(bytes); events++; + await commit(event); + await session!.acknowledgeEvent!(event); + if (event.eventType === "output.body.chunk") { + const reference = event.payload.body as Record; + const chunk = String(event.payload.text), offset = Number(event.payload.offset); + if (offset === 0) { expect(body).toBeNull(); body = { id: String(reference.bodyId), offset: 0, hash: createHash("sha256") }; } + expect(body?.id).toBe(reference.bodyId); + expect(body?.offset).toBe(offset); + expect(createHash("sha256").update(chunk).digest("hex")).toBe(event.payload.sha256); + body!.hash.update(chunk); body!.offset += Buffer.byteLength(chunk); bodyBytes += Buffer.byteLength(chunk); + } + const outputId = event.eventType === "tool.execution.completed" ? event.payload.executionId : event.itemId; + if (["item.completed", "tool.execution.completed"].includes(event.eventType) && typeof outputId === "string" && outputId.startsWith("history-")) { + const reference = event.payload.outputBody as Record; + expect(body?.offset).toBe(256 * 1024); + expect(body!.hash.digest("hex")).toBe(reference.sha256); + expect(body?.id).toBe(reference.bodyId); + body = null; historyItems++; + } + if (events % 128 === 0) { + peakCurrentBytes = Math.max(peakCurrentBytes, Buffer.byteLength(JSON.stringify((await authority!.load())!.state))); + expect(peakCurrentBytes).toBeLessThan(8 * 1024 * 1024); + await report("running"); + } + if (["session.failed", "turn.failed"].includes(event.eventType)) throw new Error(`pipeline failed: ${JSON.stringify(event.payload)}`); + if (event.eventType === "turn.completed") return; + } + throw new Error("history stream ended before completion"); + }; + const consume = consumeUntilTerminal(); + // Keep any asynchronous consumer failure observed while turn admission runs. + void consume.catch(() => {}); + await session.startTurn({ message: { text: "Generate the qualification output." } }); + await consume; + expect(historyItems).toBe(mib * 4); + expect(bodyBytes).toBe(mib * 1024 * 1024); + await session.flushEventDelivery!(); + const beforeContinuation = bundle.evidence(); + const continuationStarted = performance.now(); + await session.attachRun!({ runId: "pipeline-continuation" }); + const resumed = consumeUntilTerminal(); void resumed.catch(() => {}); + await session.startTurn({ message: { text: "Continue after the retained output history." } }); + await resumed; + const continuationMs = performance.now() - continuationStarted; + expect(bundle.evidence().providerPid).toBe(beforeContinuation.providerPid); + expect(bundle.evidence().runnerPid).toBe(beforeContinuation.runnerPid); + await session.flushEventDelivery!(); + const before = performance.now(); + const snapshots = await Promise.all([authority!.load(), readIndexedLocalState(join(root, "runner/runner-state.json")), readIndexedLocalState(join(root, "runner/codex-provider-state.json"))]); + const resumeReadMs = performance.now() - before; + const currentBytes = snapshots.map(snapshot => Buffer.byteLength(JSON.stringify(snapshot!.state))); + expect(currentBytes.every(bytes => bytes < 8 * 1024 * 1024)).toBe(true); + let physicalBytes = 0, files = 0; + async function inventory(directory: string): Promise { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await inventory(path); + else if (entry.isFile()) { + // SQLite may unlink a transient WAL/SHM after readdir during a + // checkpoint. Persistent files disappearing are still an error. + const metadata = await stat(path).catch(error => { + if ((error as NodeJS.ErrnoException).code === "ENOENT" && /\.sqlite-(wal|shm)$/.test(path)) return null; + throw error; + }); + if (metadata) { physicalBytes += metadata.size; files++; } + } + } + } + await inventory(root); + expect(physicalBytes).toBeGreaterThan(mib * 1024 * 1024); + const evidence = bundle.evidence(); + await session.close({ reason: "qualification complete" }); session = undefined; + await bundle.transport.close(); await authority!.close(); authority = undefined; + await sink.close(); await rm(root, { recursive: true, force: true }); + await report("passed", { physicalBytes, files, currentBytes, resumeReadMs, continuationMs, normalizedSha256: hash.digest("hex"), runner: evidence }); + } catch (error) { await report("failed", { error: error instanceof Error ? error.message : String(error) }); throw error; } + finally { await session?.close({ reason: "history qualification complete" }); await bundle.transport.close(); await authority?.close(); await sink.close(); await rm(root, { recursive: true, force: true }); } +}, 12 * 60 * 60_000); diff --git a/packages/paperclip-runner/src/live/indexed-history-soak.qualification.test.ts b/packages/paperclip-runner/src/live/indexed-history-soak.qualification.test.ts new file mode 100644 index 00000000000..bc0662fd6d9 --- /dev/null +++ b/packages/paperclip-runner/src/live/indexed-history-soak.qualification.test.ts @@ -0,0 +1,86 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { expect, it } from "vitest"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; +const runnerTestBinary = () => resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); + + +const hours = 72; +// Real elapsed-time qualification. A held fake-provider turn exercises runner +// leases and wall-clock limits without paid inference. This does not qualify +// a hosted provider/sandbox TTL or substitute for crash/resource-failure lanes. +it.skipIf(process.env.PAPERCLIP_HISTORY_SOAK_HOURS !== String(hours))("keeps one indexed active turn alive for 72 real hours", async () => { + const directory = await mkdtemp(join(tmpdir(), "indexed-history-soak-")); + let authority: SqliteAuthorityStore | undefined; + const identity = { runnerInstanceId: "soak-runner", environmentLeaseId: "soak-lease", normalizedSessionId: "soak-session", runId: "soak-run", turnId: "soak-turn", itemId: "soak-item" }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: runnerTestBinary(), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), + codexArgs: ["--state-file", join(directory, "fake-state.json"), "--durable-turn-ids", "--hold-turn"], + stateDirectory: directory, prpIdentity: identity, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 600_000 }, + authorityStoreFactory: async (_identity, root) => authority ??= await SqliteAuthorityStore.open({ path: join(root, "authority.sqlite"), binding: "history-soak", create: true }), + }); + bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + let consumer: Promise | undefined; + let consumerFailure: unknown; + let terminal = false; + let startedAt = new Date().toISOString(), started = performance.now(); + let qualified = false; + let probes = 0, peakCurrentBytes = 0, maxProbeMs = 0; + const report = async (status: string, error?: unknown) => { + if (!process.env.PAPERCLIP_HISTORY_REPORT) return; + const path = process.env.PAPERCLIP_HISTORY_REPORT; + const value = { schema: "paperclip.indexed-active-turn-soak.v1", status, startedAt, updatedAt: new Date().toISOString(), elapsedMs: performance.now()-started, requiredHours: hours, probes, peakCurrentBytes, maxProbeMs, provider: "credential-free fake Codex process", runnerPid: bundle.evidence().runnerPid, providerPid: bundle.evidence().codexPid, ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}) }; + await writeFile(`${path}.tmp`, JSON.stringify(value,null,2), { mode: 0o600 }); + await rename(`${path}.tmp`, path); + }; + try { + await bundle.transport.request("initialize", {}); + const opened = await bundle.transport.request("thread/start", { cwd: directory, dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }], completionContract: { revision: "soak-contract", criterionIds: ["objective"] } }) as { thread: { id: string } }; + const owner = bundle.evidence(); + consumer = (async () => { + for await (const notification of bundle.transport.notifications()) { + const port = bundle.transport.normalizedDelivery!()!; + if (notification.paperclipDelivery) await port.commit({ expectedRevision: port.load()?.revision ?? 0, raw: notification.paperclipDelivery, driver: { schema: "paperclip.qualification.raw-consumer.v1" }, events: [] }); + if (notification.method === "turn/completed") terminal = true; + } + })().catch(error => { consumerFailure = error; }); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Keep this turn active until explicitly interrupted." }] }); + startedAt = new Date().toISOString(); started = performance.now(); + await report("running"); + while (performance.now()-started < hours*60*60_000) { + const before = performance.now(); + await bundle.transport.request("thread/read", { threadId: opened.thread.id, includeTurns: false }); + probes++; maxProbeMs = Math.max(maxProbeMs,performance.now()-before); + if (consumerFailure) throw consumerFailure; + expect(terminal).toBe(false); + expect(bundle.evidence()).toMatchObject({ runnerPid: owner.runnerPid, codexPid: owner.codexPid, runnerExited: false }); + const current = await authority!.load(); + const bytes = Buffer.byteLength(JSON.stringify(current!.state)); + peakCurrentBytes = Math.max(peakCurrentBytes,bytes); + expect(bytes).toBeLessThan(256*1024); + await report("running"); + await new Promise(resolve => setTimeout(resolve, Math.min(30_000, Math.max(0,hours*60*60_000-(performance.now()-started))))); + } + qualified = true; + await report("settling"); + } catch (error) { await report("failed",error); throw error; } + finally { + try { + await bundle.transport.close(); await consumer; await authority?.close(); + expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); + if (qualified) await report("passed"); + } catch (error) { await report("cleanup_failed",error); throw error; } + finally { await rm(directory,{recursive:true,force:true}); } + } +}, (hours+1)*60*60_000); diff --git a/packages/paperclip-runner/src/live/legacy-indexed-migration.test.ts b/packages/paperclip-runner/src/live/legacy-indexed-migration.test.ts new file mode 100644 index 00000000000..a8c0facacbb --- /dev/null +++ b/packages/paperclip-runner/src/live/legacy-indexed-migration.test.ts @@ -0,0 +1,145 @@ +import { resolveRunnerCargoTestBinary } from "../../test/cargo-test-binary.js"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { appendFile, cp, mkdtemp, open, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { migrateLegacySessionAuthority, type PreparedLocalSession } from "../control-plane/legacy-authority-activation.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { readDurableControlPlaneState } from "../control-plane/authority-locator.js"; +import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js"; +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; + +const execute = promisify(execFile); +const runnerBinary = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "paperclip-runnerd"); +const providerBinary = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"); +const identity = { runnerInstanceId: "migration-runner", environmentLeaseId: "migration-lease", normalizedSessionId: "migration-session", runId: "migration-run", turnId: "migration-turn", itemId: "migration-item" }; +async function fileDigest(path: string) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path, { highWaterMark: 64 * 1024 })) hash.update(chunk); + return hash.digest("hex"); +} + +it("migrates a real legacy runner/provider through every activation crash boundary, then continues the same provider session", async () => { + const root = await mkdtemp(join(tmpdir(), "legacy-session-migration-")); + const original = join(root, "original"); + const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary, codexCommand: providerBinary, + codexArgs: ["--state-file", join(root, "fake-state.json"), "--durable-turn-ids"], stateDirectory: original, prpIdentity: identity }); + bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + let successor: ReturnType | undefined; + let authority: SqliteAuthorityStore | undefined; + try { + await bundle.transport.request("initialize", {}); + const opened = await bundle.transport.request("thread/start", { cwd: root, + dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }], + completionContract: { revision: "migration-contract", criterionIds: ["objective"] } }); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Complete before conversion." }] }); + for await (const notification of bundle.transport.notifications()) if (notification.method === "turn/completed") break; + await bundle.transport.close(); + expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); + const owner = bundle.evidence(); + const assertExclusiveFence = async () => { + for (const pid of [owner.runnerPid, owner.codexPid]) { + if (!pid) throw new Error("missing exact fixture process owner"); + try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; throw error; } + throw new Error("fixture owner remains alive"); + } + }; + await assertExclusiveFence(); + const controllerBytes = await readFile(join(original, "control-plane/control-plane-state.json")); + const runnerBytes = await readFile(join(original, "runner/runner-state.json")); + const providerBytes = await readFile(join(original, "runner/codex-provider-state.json")); + const originalController = JSON.parse(controllerBytes.toString()); + const runner = JSON.parse(runnerBytes.toString()); + const boundaries = ["source-change", "intent", "prepared", "controller-commit", "publish:runner-state.sqlite", "publish:runner-state.sqlite.receipts", "publish:runner-state.sqlite.routing", "publish:runner-state.sqlite.lifetime", "publish:runner-state.sqlite.lock", "publish:codex-provider-state.sqlite", "publish:codex-provider-state.sqlite.receipts", "publish:codex-provider-state.sqlite.routing", "publish:codex-provider-state.sqlite.lifetime", "publish:codex-provider-state.sqlite.lock", "publish:runner-state.json", "publish:codex-provider-state.json", "controller-locator", "active"]; + for (const [index, boundary] of boundaries.entries()) { + const directory = join(root, `case-${index}`); + await cp(original, directory, { recursive: true }); + const controllerPath = join(directory, "control-plane/control-plane-state.json"); + const runnerPath = join(directory, "runner/runner-state.json"); + let expectedAck = originalController.ackedSourceSeq; + if (boundary === "active") { + // Storage qualification: inject valid historical diagnostic frames + // after the real fixture settles. This is synthesized history, not + // 2,000 model calls. Then migrate >192 MiB and resume real processes. + const file = await open(controllerPath, "w", 0o600); + const { committedEvents, ...metadata } = originalController; + expectedAck += 2_000; + await file.write(`${JSON.stringify({ ...metadata, ackedSourceSeq: expectedAck }).slice(0, -1)},"committedEvents":[`); + let separator = ""; + for (const event of committedEvents) { await file.write(`${separator}${JSON.stringify(event)}`); separator = ","; } + const padding = "x".repeat(100 * 1024); + for (let seq = originalController.ackedSourceSeq + 1; seq <= expectedAck; seq++) { + const sourceEventId = `migration-growth-${seq}`; + const payload = { schema: "paperclip.prp.event.v1", schemaVersion: 1, ...identity, sourceSeq: seq, sourceEventId, + sourceInstanceId: identity.runnerInstanceId, sourceKind: "runner", eventType: "harness.diagnostic", priority: 1, + emittedAt: "2026-09-28T00:00:00.000Z", payload: { code: "qualification-history", text: padding } }; + await file.write(`,${JSON.stringify({ sourceSeq: seq, sourceEventId, eventType: "harness.diagnostic", priority: 1, deliveryCount: 1, logicalEffectCount: 1, + envelope: { protocol: "paperclip.runner", version: 1, kind: "event", ...identity, payload } })}`); + } + await file.write("]}"); await file.close(); + expect((await stat(controllerPath)).size).toBeGreaterThan(192 * 1024 * 1024); + await writeFile(runnerPath, JSON.stringify({ ...runner, ackedSourceSeq: expectedAck, nextSourceSeq: expectedAck + 1 }), { mode: 0o600 }); + } + const originalControllerDigest = await fileDigest(controllerPath), originalRunnerDigest = await fileDigest(runnerPath); + authority = await SqliteAuthorityStore.open({ path: join(directory, "control-plane/authority.sqlite"), binding: `migration-${index}`, create: true }); + const prepareLocal = async (destination: string): Promise => { + const args = ["storage", "stage-legacy", "--state-dir", join(directory, "runner"), "--destination", destination, "--fence-id", "migration-fence", + "--connect-url", "ws://127.0.0.1:3000/api/runner/v1/connect/migration-run", "--runner-version", "qualification", "--runner-digest", `sha256:${"a".repeat(64)}`, + "--max-outbox-bytes", String(runner.maxOutboxBytes), "--p0-reserve-bytes", String(runner.p0ReserveBytes), + "--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId, "--session-id", identity.normalizedSessionId, + "--run-id", identity.runId, "--turn-id", identity.turnId, "--item-id", identity.itemId]; + const { stdout } = await execute(runnerBinary, args, { maxBuffer: 128 * 1024, timeout: 30_000 }); + return JSON.parse(stdout) as PreparedLocalSession; + }; + const options = { root: directory, identity, authority, fenceId: "migration-fence", assertExclusiveFence, prepareLocal }; + await expect(migrateLegacySessionAuthority({ ...options, onBoundary: async (name) => { if (name === boundary || (boundary === "source-change" && name === "prepared")) throw new Error(`crash:${boundary}`); } })).rejects.toThrow(`crash:${boundary}`); + if (boundary === "source-change") { + await appendFile(join(directory, "control-plane/control-plane-state.json"), "\n"); + await expect(migrateLegacySessionAuthority(options)).rejects.toThrow("source_changed"); + expect((await authority.load())!.state.phase).toBe("prepared"); + await authority.close(); authority = undefined; + continue; + } + if (boundary !== "active") await expect(DurablePrpControlPlane.open({ stateDirectory: join(directory, "control-plane"), identity, expectedRunnerVersion: "test", expectedRunnerDigest: `sha256:${"a".repeat(64)}`, authorityStore: authority })).rejects.toThrow("migration_pending"); + await authority.close(); + authority = await SqliteAuthorityStore.open({ path: join(directory, "control-plane/authority.sqlite"), binding: `migration-${index}`, create: false }); + await migrateLegacySessionAuthority({ ...options, authority }); + const migrated = await readDurableControlPlaneState(join(directory, "control-plane")); + expect(migrated.identity).toEqual(identity); + expect(migrated.ackedSourceSeq).toBe(expectedAck); + expect(await fileDigest(join(directory, "indexed-migration/legacy/control-plane-state.json"))).toBe(originalControllerDigest); + expect(await fileDigest(join(directory, "indexed-migration/legacy/runner-state.json"))).toBe(originalRunnerDigest); + expect(await readFile(join(directory, "indexed-migration/legacy/codex-provider-state.json"))).toEqual(providerBytes); + for (const name of ["runner-state", "codex-provider-state"]) { + expect((await stat(join(directory, "runner", `${name}.sqlite.routing`))).isDirectory()).toBe(true); + await expect(stat(join(directory, "indexed-migration/runner", `${name}.sqlite.routing`))).rejects.toMatchObject({ code: "ENOENT" }); + } + expect((await readIndexedLocalState(join(directory, "runner/runner-state.json"))).state).toMatchObject({ ...identity, lifecycle: "suspended" }); + const firstCommand = originalController.commands.find((command: {status:string}) => command.status === "completed"); + expect((await authority.getRecord(identity.runId, "command", firstCommand.commandId))!.body).toEqual(firstCommand); + if (index === boundaries.length - 1) { + await authority.close(); authority = undefined; + successor = createCapabilityRunnerdCodexTransport({ runnerBinary, codexCommand: providerBinary, + codexArgs: ["--state-file", join(root, "fake-state.json"), "--durable-turn-ids"], + stateDirectory: directory, prpIdentity: { ...identity, runId: "migration-next-run", turnId: "migration-next-turn", itemId: "migration-next-item" }, + authorityStoreFactory: async (_identity, currentDirectory) => authority = await SqliteAuthorityStore.open({ path: join(currentDirectory, "authority.sqlite"), binding: `migration-${index}`, create: true }) }); + successor.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + const resumed = await successor.transport.request("thread/read", {}); + expect((resumed.thread as Record).id).toBe((opened.thread as Record).id); + await successor.transport.request("turn/start", { input: [{ type: "text", text: "Continue after indexed migration." }] }); + for await (const notification of successor.transport.notifications()) if (notification.method === "turn/completed") break; + await successor.transport.close(); + expect(successor.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); + } + await authority.close(); authority = undefined; + } + } finally { + await successor?.transport.close(); await bundle.transport.close(); await authority?.close(); + await rm(root, { recursive: true, force: true }); + } +}, 120_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts index b760a59e099..37331ed7bd0 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts @@ -1,3 +1,4 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; // The retained-settlement maintenance suite below moved out of // runnerd-codex-transport.test.ts. Vitest schedules whole files onto // workers, so that single 8.9k-line file serialized ~400s of tests and was @@ -51,7 +52,12 @@ import { // Explicit private-artifact test lane; production/default dist is never changed. const defaultCapabilityRunnerdBinary = () => process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - qualifiedCapabilityRunnerdBinary(); + resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, + ); function maintenanceFixtureBackendName(fixtureId: string): string { return `maintenance-test-${fixtureId}`; @@ -138,10 +144,7 @@ it.each([ const activated = join(directory, "activated"); const home = join(directory, "source-home"); await mkdir(home); - const fakeCodex = resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", - ); + const fakeCodex = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"); const bin = join(directory, "provider-bin"); if (bareCodex) { await mkdir(bin); @@ -270,10 +273,10 @@ it.each([ .update(await readFile(fixtureRunner)) .digest("hex")}`, }); - builder.queueCommand("turn.stop", { + (await builder.queueCommand("turn.stop", { reason: "interrupted original close", - }); - if (!missingHome) builder.queueCommand("runner.suspend", {}); + })); + if (!missingHome) (await builder.queueCommand("runner.suspend", {})); // Match the retained production split: runner-owned unacknowledged // output plus another full provider-owned prefix behind the old suspend. const runnerFile = join(original, "runner/runner-state.json"); @@ -815,10 +818,10 @@ it.each([ .update(await readFile(fixtureRunner)).digest("hex")}`, onCommittedEvent: appendEvent, }); - const snapshot = replayCore.queueCommand("session.snapshot", {}); - const stop = replayCore.queueCommand("turn.stop", { + const snapshot = (await replayCore.queueCommand("session.snapshot", {})); + const stop = (await replayCore.queueCommand("turn.stop", { reason: "startup-fence regression only", - }); + })); let replayHandle: ReturnType | null = null; let replayAssertionFailed = false; try { @@ -828,7 +831,7 @@ it.each([ connectUrl: replayCore.connectUrl, stateDirectory: join(copy, "runner"), identity, - ticket: replayCore.issueBootstrapTicket(), + ticket: (await replayCore.issueBootstrapTicket()), maxOutboxBytes: runnerState.maxOutboxBytes, p0ReserveBytes: runnerState.p0ReserveBytes, maxRuntimeMs: 2_000, @@ -846,9 +849,9 @@ it.each([ hasRuntimeContext: false, }), }); - await vi.waitFor(() => { + await vi.waitFor(async () => { for (const queued of [snapshot, stop]) { - const command = replayCore.getCommand(queued.commandId); + const command = (await replayCore.getCommand(queued.commandId)); expect(command?.status).toBe("failed"); expect(command?.result).toMatchObject({ result: { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 2f5d38c249a..369939fd98c 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1,3 +1,4 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "../../test/cargo-test-binary.js"; import { cp, mkdir, @@ -35,6 +36,9 @@ import { executeNativeSession } from "../native-session-runtime.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +// Preserve the real implementation even if a timed-out fixture is still unwinding. +const openUnmockedControlPlane = DurablePrpControlPlane.open.bind(DurablePrpControlPlane); +import { SqliteAuthorityStore } from "../control-plane/sqlite-authority-store.js"; import { NATIVE_RUNTIME_ASSET_SCHEMA, @@ -90,7 +94,12 @@ import { // Explicit private-artifact test lane; production/default dist is never changed. const defaultCapabilityRunnerdBinary = () => process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - qualifiedCapabilityRunnerdBinary(); + resolveRunnerCargoTestBinaryOrDefault( + resolve(import.meta.dirname, "../../runner"), + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, + ); async function expectTurnStarted( notifications: AsyncIterator<{ method: string }>, @@ -705,7 +714,9 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( codexCommand: fakeCodex, codexArgs: fakeCodexArgs(stateDirectory, "--split-event-burst"), stateDirectory, - closeGraceMs: 2_000, + // Only the expiry case requires a short deadline. The success case must + // leave room for real journal fsyncs under a parallel storage test load. + closeGraceMs: mode === "within_budget" ? 10_000 : 2_000, controlPlaneRegistration: async (authority) => { core = authority; await authority.start(); @@ -832,7 +843,7 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( await rm(stateDirectory, { recursive: true, force: true }); } }, - 15_000, + 30_000, ); it("infers a remote provider turn until its own terminal event is durable", () => { @@ -2106,6 +2117,16 @@ it("rehydrates canonical session goals into Codex goal notifications", () => { ).toEqual({ revision: 7, threadId: "thread-1", workingNow: false }); }); +it("preserves the command output channel and execution binding through the facade", () => { + const payload = { kind: "commandExecution", providerMethod: "item/commandExecution/outputDelta", itemId: "exec-1", text: "stdout" }; + expect(runnerdCanonicalNotificationMethod("item.delta", payload)).toBe("item/commandExecution/outputDelta"); + expect(rehydrateRunnerdDeltaNotification(payload, "thread", "turn")).toMatchObject({ + threadId: "thread", turnId: "turn", itemId: "exec-1", delta: "stdout", kind: "commandExecution", + }); + expect(runnerdCanonicalNotificationMethod("item.delta", { ...payload, providerMethod: "paperclip/runResult" })).toBeUndefined(); + expect(runnerdCanonicalNotificationMethod("item.delta", { kind: "agentMessage", text: "prose" })).toBe("item/agentMessage/delta"); +}); + it("routes canonical session goals back through the Codex notification facade", () => { expect( runnerdCanonicalNotificationMethod("session.goal.updated", { @@ -2201,10 +2222,7 @@ it("recovers provider readiness from an already-committed journal without replay ).toEqual(persistedReady); }); -const fakeCodex = resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", -); +const fakeCodex = resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"); function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { return [ @@ -2325,13 +2343,13 @@ it("keeps a quiet active Codex turn in the same process across connection lease const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-renew-active-")); const callsPath = join(stateDirectory, "calls.log"); const cores: DurablePrpControlPlane[] = []; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function(options: ConstructorParameters[0]) { - const core = new OriginalCore({ ...options, connectionLeaseTtlMs: 60_000 }); + const openCore = openUnmockedControlPlane; + const coreSpy = vi.spyOn(durableControlPlane.DurablePrpControlPlane, "open") + .mockImplementation(async function(options: Parameters[0]) { + const core = await openCore({ ...options, connectionLeaseTtlMs: 60_000 }); cores.push(core); return core; - } as unknown as typeof OriginalCore); + }); const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(stateDirectory, "--hold-turn", "--record-process-start", "--call-log", callsPath), @@ -2556,13 +2574,23 @@ it("controls a Codex session goal end to end through durable PRP v2", async () = }); }, 30_000); -it.each([false, true])("binds goal turns through the full Codex harness (autonomous continuation: %s)", async (autocontinue) => { +it.each([{ autocontinue: false, indexed: false }, { autocontinue: true, indexed: false }, { autocontinue: true, indexed: true }])("binds goal turns through the full Codex harness (autonomous continuation: $autocontinue, indexed: $indexed)", async ({ autocontinue, indexed }) => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-goal-harness-")); + let authority: SqliteAuthorityStore | null = null; + const sink = indexed ? await SqliteAuthorityStore.open({ path: join(stateDirectory, "normalized.sqlite"), binding: "normalized-goal-test", create: true }) : null; + let sinkGeneration = "0"; + const append = async (event: PrpEvent) => { + if (sink) sinkGeneration = await sink.commit({ expectedGeneration: sinkGeneration, state: { lastSourceSeq: event.sourceSeq }, records: [{ epoch: event.runId, kind: "event", id: event.sourceEventId, sequence: String(event.sourceSeq), body: event as unknown as Record }] }); + }; const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(stateDirectory, "--goal-autostart", ...(autocontinue ? ["--goal-autocontinue"] : [])), stateDirectory, + ...(indexed ? { + prpIdentity: { runnerInstanceId: "runner-codex", environmentLeaseId: "environment-goal", runId: "run-goal-harness-autostart", normalizedSessionId: "normalized-goal-harness-autostart", turnId: "turn-goal", itemId: "item-goal" }, + authorityStoreFactory: async (_identity: unknown, directory: string) => authority = await SqliteAuthorityStore.open({ path: join(directory, "authority.sqlite"), binding: "goal-harness-indexed", create: true }), + } : {}), }); const driver = new CodexAppServerDriver({ taskEnvelope: { @@ -2593,11 +2621,13 @@ it.each([false, true])("binds goal turns through the full Codex harness (autonom normalizedSessionId: "normalized-goal-harness-autostart", workingDirectory: tmpdir(), }); + if (indexed) session.setEventCommitter!(append); const observed: Array<{ eventType: string }> = []; const turnStarted = Promise.race([ (async () => { for await (const event of session!.events()) { observed.push(event); + if (indexed) { await append(event); await session!.acknowledgeEvent!(event); } if (event.eventType === "turn.started" && event.turnId === (autocontinue ? "provider-goal-turn-2" : "provider-goal-turn-1")) return event; if (event.eventType === "session.failed") { throw new Error(`goal autostart failed: ${JSON.stringify(event.payload)}`); @@ -2625,6 +2655,8 @@ it.each([false, true])("binds goal turns through the full Codex harness (autonom } finally { await session?.close(); await bundle.transport.close(); + await (authority as SqliteAuthorityStore | null)?.close(); + await sink?.close(); await rm(stateDirectory, { recursive: true, force: true }); } expect(bundle.evidence()).toMatchObject({ @@ -3439,10 +3471,7 @@ it("does not retry a real memoized transport close whose suspension proof is una })); const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", - ), + codexCommand: resolveRunnerCargoTestBinary(resolve(import.meta.dirname, "../../runner"), "debug", "fake-codex-app-server"), codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], stateDirectory, closeGraceMs: 400, @@ -4273,13 +4302,13 @@ it.each(["held-ack", "lost-ack", "rejected-attach"] as const)( const commitEntered = new Promise((resolveEntered) => { enteredCommit = resolveEntered; }); - const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const openCore = openUnmockedControlPlane; const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], + .spyOn(durableControlPlane.DurablePrpControlPlane, "open") + .mockImplementation(async function ( + options: Parameters[0], ) { - const core = new OriginalCore({ + const core = await openCore({ ...options, onCommittedEvent: async (event) => { await options.onCommittedEvent?.(event); @@ -4314,7 +4343,7 @@ it.each(["held-ack", "lost-ack", "rejected-attach"] as const)( }); cores.push(core); return core; - } as unknown as typeof OriginalCore); + }); const launch = durableControlPlane.spawnRunner; const launchSpy = vi .spyOn(durableControlPlane, "spawnRunner") @@ -4570,16 +4599,16 @@ it.each([false, true])( const callsPath = join(stateDirectory, "calls.log"); const cores: DurablePrpControlPlane[] = []; const handles: ReturnType[] = []; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const openCore = openUnmockedControlPlane; const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], + .spyOn(durableControlPlane.DurablePrpControlPlane, "open") + .mockImplementation(async function ( + options: Parameters[0], ) { - const core = new OriginalCore(options); + const core = await openCore(options); cores.push(core); return core; - } as unknown as typeof OriginalCore); + }); const launch = durableControlPlane.spawnRunner; const launchSpy = vi .spyOn(durableControlPlane, "spawnRunner") @@ -4651,8 +4680,8 @@ it.each([false, true])( const getCommand = core.getCommand.bind(core); observerSpy = vi .spyOn(core, "getCommand") - .mockImplementation((commandId) => { - const command = getCommand(commandId); + .mockImplementation(async (commandId) => { + const command = await getCommand(commandId); if ( command?.type === "run.attach" && core.store.state.completedWarmTransition?.command.commandId !== @@ -4929,20 +4958,20 @@ it.each([ }, }; }; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const openCore = openUnmockedControlPlane; const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], + .spyOn(durableControlPlane.DurablePrpControlPlane, "open") + .mockImplementation(async function ( + options: Parameters[0], ) { - const core = new OriginalCore(options); + const core = await openCore(options); cores.push(core); if (recovering && snapshotFault) { const getCommand = core.getCommand.bind(core); snapshotObserverSpy = vi .spyOn(core, "getCommand") - .mockImplementation((id) => { - const command = getCommand(id); + .mockImplementation(async (id) => { + const command = await getCommand(id); if ( command?.type !== "session.snapshot" || command.status !== "completed" @@ -4968,7 +4997,7 @@ it.each([ }); } return core; - } as unknown as typeof OriginalCore); + }); const launch = durableControlPlane.spawnRunner; const launchSpy = vi .spyOn(durableControlPlane, "spawnRunner") diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index bdfb23bdb3c..625295b7a92 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,3 +1,9 @@ +import { isProviderProcessGeneration, provesProviderGenerationTransition } from "../control-plane/process-generation.js"; +import { prepareIndexedRunnerArchive, finishIndexedRunnerArchive, publishIndexedArchiveHead, readIndexedArchiveHead, syncArchiveDirectory } from "./indexed-authority-archive.js"; +import { readDurableControlPlaneState, readDurableControlPlaneSnapshot, type ExternalAuthorityReader } from "../control-plane/authority-locator.js"; +import { readIndexedLocalState } from "../control-plane/indexed-local-state-reader.js"; +import { defaultStorageRunnerBinary, nativeStorageCommand } from "../control-plane/native-storage-command.js"; +import { authorityJson, INDEXED_DURABILITY_CAPABILITY, type DurableAuthorityStore } from "../control-plane/durable-authority-store.js"; import { RunnerdTraceFrameIndex } from "./runnerd-trace-frame-index.js"; import { waitForWarmAttachmentReadiness } from "./warm-attachment-readiness.js"; import { codexExecutableReadOnlyRoots } from "../drivers/codex/codex-security-config.js"; @@ -40,6 +46,7 @@ import type { } from "../contracts/durable-recovery.js"; import type { NativeRunIdentity } from "../contracts/types.js"; import type { PrpEvent } from "../protocol/replay-contract.js"; +import { RAW_DELIVERY_FRAME_COMPLETE } from "../control-plane/normalized-delivery.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import type { HarnessRuntimeRequestResolution, @@ -149,20 +156,9 @@ export function withCodexCollaborationRuntimeInstructions( return `${base}\n\n${CODEX_COLLABORATION_RUNTIME_INSTRUCTIONS}`; } -function readControlPlaneState(directory: string): Record { - const path = resolve(directory, "control-plane-state.json"); - const metadata = lstatSync(path); - if ( - metadata.isSymbolicLink() || - !metadata.isFile() || - metadata.size > 64 * 1024 * 1024 - ) { - throw new Error("native_runner_control_plane_state_unsafe"); - } - return record(JSON.parse(readFileSync(path, "utf8"))); -} +const readControlPlaneState = readDurableControlPlaneState; -function readRunnerState(path: string): Record { +async function readRunnerState(path: string): Promise> { const metadata = lstatSync(path); if ( metadata.isSymbolicLink() || @@ -171,7 +167,9 @@ function readRunnerState(path: string): Record { ) { throw new Error("native_runner_authority_rotation_state_unsafe"); } - return record(JSON.parse(readFileSync(path, "utf8"))); + const value = record(JSON.parse(readFileSync(path, "utf8"))); + if (["paperclip.runner.durable.state.indexed.v1", "paperclip.runner.codex-provider-state.indexed.v1"].includes(String(value.schema))) return (await readIndexedLocalState(path)).state; + return value; } function controlPlaneIdentity( @@ -217,6 +215,9 @@ function assertRealDirectory(path: string): void { } function quarantineLocalRuntimeState(root: string, reason: unknown): never { + // An operator backup or an interrupted archive is recoverable contention, + // not evidence of corruption. Preserve the active routing for a retry. + if (reason instanceof Error && reason.message.includes("storage_pressure:")) throw reason; assertRealDirectory(root); const quarantine = resolve( dirname(root), @@ -233,21 +234,39 @@ function quarantineLocalRuntimeState(root: string, reason: unknown): never { function authorityArchiveDirectory( root: string, identity: DurableRecoveryIdentity, + indexed = false, ): string { const digest = createHash("sha256") - .update(JSON.stringify(identity)) + // Indexed stores canonicalize object keys. Archive routing must survive + // reopening the same identity in a different JSON property order. + .update(indexed ? authorityJson(identity) : JSON.stringify(identity)) .digest("hex") .slice(0, 24); return resolve(root, "authority-epochs", `epoch-${digest}`); } -function latestArchivedControlPlaneState( +async function latestArchivedControlPlaneState( root: string, desired: DurableRecoveryIdentity, -): Record | null { + readAuthority?: ExternalAuthorityReader, +): Promise | null> { const archivesRoot = resolve(root, "authority-epochs"); if (!existsSync(archivesRoot)) return null; assertRealDirectory(archivesRoot); + const indexedArchive = readIndexedArchiveHead(archivesRoot); + if (indexedArchive !== null) { + const directory = resolve(indexedArchive, "control-plane"); + assertRealDirectory(directory); + const state = await readControlPlaneState(directory, readAuthority); + const identity = controlPlaneIdentity(state); + if ( + authorityArchiveDirectory(root, identity, true) !== indexedArchive || + identity.runnerInstanceId !== desired.runnerInstanceId || + identity.environmentLeaseId !== desired.environmentLeaseId || + identity.normalizedSessionId !== desired.normalizedSessionId + ) throw new Error("native_runner_authority_archive_conflict"); + return state; + } const candidates = readdirSync(archivesRoot, { withFileTypes: true }) .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) .map((entry) => resolve(archivesRoot, entry.name, "control-plane")) @@ -262,7 +281,7 @@ function latestArchivedControlPlaneState( .sort((left, right) => right.modifiedAt - left.modifiedAt); for (const candidate of candidates) { assertRealDirectory(candidate.directory); - const state = readControlPlaneState(candidate.directory); + const state = await readControlPlaneState(candidate.directory, readAuthority); const identity = controlPlaneIdentity(state); if ( identity.runnerInstanceId === desired.runnerInstanceId && @@ -275,11 +294,12 @@ function latestArchivedControlPlaneState( return null; } -function rotateLocalAuthorityEpoch( +async function rotateLocalAuthorityEpoch( root: string, controlPlaneState: Record, desired: DurableRecoveryIdentity, -): Record { + runnerBinary?: string, +): Promise> { const priorIdentity = controlPlaneIdentity(controlPlaneState); if ( priorIdentity.runnerInstanceId !== desired.runnerInstanceId || @@ -293,9 +313,16 @@ function rotateLocalAuthorityEpoch( } const runnerDirectory = resolve(root, "runner"); const runnerStatePath = resolve(runnerDirectory, "runner-state.json"); - const archive = authorityArchiveDirectory(root, priorIdentity); + const indexedArchive = existsSync(resolve(runnerDirectory, "runner-state.sqlite")) || + existsSync(resolve(root, "authority-epochs", "current-archive.json")); + const archive = authorityArchiveDirectory(root, priorIdentity, indexedArchive); const archivedControlPlane = resolve(archive, "control-plane"); const archivedRunnerState = resolve(archive, "runner-state.json"); + const activeControlPlane = resolve(root, "control-plane"); + if (existsSync(resolve(archive, "runner-transfer.json")) && existsSync(archivedControlPlane)) { + if (existsSync(activeControlPlane)) throw new Error("native_runner_authority_archive_conflict"); + await finishIndexedRunnerArchive(runnerDirectory, archive, runnerBinary); + } const runnerStateSource = existsSync(runnerStatePath) ? runnerStatePath : archivedRunnerState; @@ -303,7 +330,7 @@ function rotateLocalAuthorityEpoch( throw new Error("native_runner_authority_rotation_state_unavailable"); } assertRealDirectory(runnerDirectory); - const runnerState = readRunnerState(runnerStateSource); + const runnerState = await readRunnerState(runnerStateSource); if ( runnerState.runnerInstanceId !== priorIdentity.runnerInstanceId || runnerState.environmentLeaseId !== priorIdentity.environmentLeaseId || @@ -327,15 +354,33 @@ function rotateLocalAuthorityEpoch( mkdirSync(archive, { mode: 0o700 }); } assertRealDirectory(archive); - const activeControlPlane = resolve(root, "control-plane"); + syncArchiveDirectory(archivesRoot); + const indexed = existsSync(resolve(runnerDirectory, "runner-state.sqlite")); + if (indexed && existsSync(runnerStatePath)) { + await prepareIndexedRunnerArchive(runnerDirectory, archive, runnerBinary); + publishIndexedArchiveHead(archive); + } if (existsSync(activeControlPlane)) { assertRealDirectory(activeControlPlane); if (existsSync(archivedControlPlane)) { throw new Error("native_runner_authority_archive_conflict"); } - renameSync(activeControlPlane, archivedControlPlane); + const locatorPath = resolve(activeControlPlane, "control-plane-state.json"); + const locator = lstatSync(locatorPath).size <= 8192 ? JSON.parse(readFileSync(locatorPath, "utf8")) : null; + if (locator?.schema === "paperclip.runner.authority-locator.v1" && locator.location?.kind === "sqlite") { + const snapshot = await readDurableControlPlaneSnapshot(activeControlPlane); + if (!snapshot.storedSha256 || authorityJson(snapshot.state) !== authorityJson(controlPlaneState)) { + throw new Error("native_runner_authority_archive_conflict"); + } + await nativeStorageCommand(["archive-controller", "--directory", activeControlPlane, "--destination", archivedControlPlane, + "--digest", snapshot.storedSha256], runnerBinary); + } else renameSync(activeControlPlane, archivedControlPlane); + syncArchiveDirectory(root); + syncArchiveDirectory(archive); } - if (existsSync(runnerStatePath)) { + if (existsSync(resolve(archive, "runner-transfer.json"))) { + await finishIndexedRunnerArchive(runnerDirectory, archive, runnerBinary); + } else if (existsSync(runnerStatePath)) { if (existsSync(archivedRunnerState)) { throw new Error("native_runner_authority_archive_conflict"); } @@ -392,7 +437,7 @@ async function rotateExternalAuthorityEpoch( throw new Error("native_runner_authority_archive_conflict"); } const archivedIdentity = controlPlaneIdentity( - readControlPlaneState(archivedControlPlane), + await readControlPlaneState(archivedControlPlane), ); if (!recoveryIdentityMatches(archivedIdentity, priorIdentity)) { throw new Error("native_runner_authority_archive_conflict"); @@ -552,21 +597,21 @@ type ProviderDrainState = ReturnType | "unreadable" | null; async function awaitProviderDrainBarrier(input: { - readProviderState: () => ProviderDrainState; + readProviderState: () => ProviderDrainState | Promise; semanticResultsSettled: () => boolean; commands: () => readonly { commandId: string; status: string; result?: unknown; }[]; - queueDrain: (commandId: string) => void; - pump: () => void; + queueDrain: (commandId: string) => void | Promise; + pump: () => void | Promise; deadline: number; pollIntervalMs?: number; }): Promise { let receiptConfirmed = false; while (Date.now() < input.deadline) { - input.pump(); + await input.pump(); // A callback is not part of the provider FIFO until its result is durably // queued and completed. Never certify a temporarily empty prefix while // that admitted old-authority result is still being produced. @@ -576,7 +621,7 @@ async function awaitProviderDrainBarrier(input: { ); continue; } - const state = input.readProviderState(); + const state = await input.readProviderState(); // Remote roots still require the exact runner-owned receipt. Their // checkpoint separately verifies provider settlement on the remote host. if ( @@ -594,9 +639,9 @@ async function awaitProviderDrainBarrier(input: { continue; } const commandId = `command_close_drain_${randomUUID().replaceAll("-", "")}`; - input.queueDrain(commandId); + await input.queueDrain(commandId); while (Date.now() < input.deadline) { - input.pump(); + await input.pump(); const command = input .commands() .find((candidate) => candidate.commandId === commandId); @@ -713,10 +758,10 @@ async function awaitRunnerSuspensionBarrier(input: { type: string; status: string; }[]; - queueSuspend: (commandId: string) => void; + queueSuspend: (commandId: string) => void | Promise; readRunnerState: () => Promise>; runnerHasExited: () => Promise; - pump: () => void; + pump: () => void | Promise; deadline: number; pollIntervalMs?: number; }): Promise { @@ -742,10 +787,10 @@ async function awaitRunnerSuspensionBarrier(input: { const commandId = existing?.commandId ?? `command_close_suspend_${randomUUID().replaceAll("-", "")}`; - if (!existing) input.queueSuspend(commandId); + if (!existing) await input.queueSuspend(commandId); while (Date.now() < input.deadline) { - input.pump(); + await input.pump(); const command = input .commands() .find((candidate) => candidate.commandId === commandId); @@ -1075,6 +1120,9 @@ export interface CapabilityRunnerdProcessEvidence { } export interface CapabilityRunnerdCodexTransportOptions { + /** Fresh indexed authority or an already activated authority; legacy stays on its compatibility reader. */ + authorityStoreFactory?: (identity: DurableRecoveryIdentity, directory: string) => Promise; + readAuthorityState?: ExternalAuthorityReader; provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx"; opencodePermissionMode?: NativeOpenCodePermissionMode; acpxAgent?: QualifiedAcpxAgent; @@ -1338,6 +1386,12 @@ export function runnerdCanonicalNotificationMethod( if (eventType === "session.goal.snapshot" && payload.goal === null) { return undefined; } + // Tool output is its own provider channel. Treating it as assistant prose + // loses the execution binding and pollutes the agent message history. + if (eventType === "item.delta" && payload.kind === "commandExecution") { + return payload.providerMethod === "item/commandExecution/outputDelta" + ? "item/commandExecution/outputDelta" : undefined; + } return ( { "turn.started": "turn/started", @@ -1792,6 +1846,7 @@ export function rehydrateRunnerdItemNotification( type: rawItem.type ?? rawParams.kind, status: rawItem.status ?? rawParams.status, text: rawItem.text ?? rawParams.text, + ...(rawParams.outputBody ? { outputBody: rawParams.outputBody } : {}), ...(phase === undefined ? {} : { phase }), ...(channel === undefined ? {} : { channel }), }, @@ -2500,9 +2555,9 @@ async function settleRetainedRunnerdSessionOwned( before.provider.activeProviderTurnId !== null && before.provider.activeProviderTurnId !== undefined ) { - core.queueCommand("turn.stop", { + (await core.queueCommand("turn.stop", { reason: "exact retained authority cleanup", - }); + })); } await core.start(); await authorize(); @@ -2515,7 +2570,7 @@ async function settleRetainedRunnerdSessionOwned( connectUrl: core.connectUrl, stateDirectory: resolve(root, "runner"), identity: input.identity, - ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), + ticket: (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)), maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 30_000, @@ -2575,7 +2630,7 @@ async function settleRetainedRunnerdSessionOwned( const provider = providerDrainStateFromSnapshot(state.provider); if (!terminalOnly && !terminalQueued && provider.providerSettled) { if (!drainQueued) { - core.queueCommand("runner.drain", {}, undefined, true); + (await core.queueCommand("runner.drain", {}, undefined, true)); drainQueued = true; } else if ( provider.pendingEventCount === 0 && @@ -2584,7 +2639,7 @@ async function settleRetainedRunnerdSessionOwned( (command) => command.status === "pending", ) ) { - core.queueCommand("runner.suspend", {}, undefined, true); + (await core.queueCommand("runner.suspend", {}, undefined, true)); terminalQueued = true; } } @@ -2759,10 +2814,10 @@ async function settleRetainedRunnerdSessionOwned( pids.add(identity.processId); } if ( - !Number.isSafeInteger(before.provider.providerProcessGeneration) || - settled.provider.providerProcessGeneration !== - Number(before.provider.providerProcessGeneration) + - (epochRestoresProvider ? 1 : 0) + !isProviderProcessGeneration(before.provider.providerProcessGeneration) || + (epochRestoresProvider + ? !provesProviderGenerationTransition(before.provider.providerProcessGeneration, settled.provider.providerProcessGeneration, settled.provider.processGenerationTransition) + : settled.provider.providerProcessGeneration !== before.provider.providerProcessGeneration) ) throw maintenanceDenied(); const provider = providerDrainStateFromSnapshot(settled.provider); @@ -3306,6 +3361,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #adoptedRunnerAuthenticated = false; #pump: NodeJS.Timeout | null = null; #eventSourceSeq = 0; + #eventSourceEpoch?: string; #eventIdentity: DurableRecoveryIdentity | null = null; #pendingWarmRecoveryCompletion: { transitionId: string; @@ -3559,7 +3615,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); while (Date.now() < recoveryDeadline) { this.#throwIfFailed(); - this.#pumpEvents(); + await this.#pumpEvents(); terminal = [...(this.#core?.store.state.committedEvents ?? [])] .reverse() .find((event) => { @@ -3595,7 +3651,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } } this.#recoveryTurnBindingPending = false; - this.#pumpEvents(); + await this.#pumpEvents(); return { thread: { id: this.#threadId, @@ -3673,6 +3729,27 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return this.#queue; } + normalizedDelivery() { + return this.#core?.normalizedDelivery() ?? null; + } + + #beforeRunRotation: (() => Promise) | null = null; + setBeforeRunRotation(settle: () => Promise): void { this.#beforeRunRotation = settle; } + + async #settleNormalizedDelivery(): Promise { + const core = this.#core; + if (!core?.indexedPersistence) return; + const deadline = Date.now() + 30_000; + for (;;) { + await this.#pumpEvents(); + await this.#beforeRunRotation?.(); + const consumer = core.store.state.indexedState?.normalizedDelivery; + if (consumer && consumer.raw.sourceEpoch === core.store.state.indexedState?.sourceEpoch && consumer.raw.sourceSeq === core.store.state.ackedSourceSeq && (consumer.raw.sourceSeq === 0 || consumer.raw.ordinal === RAW_DELIVERY_FRAME_COMPLETE) && consumer.pending.length === 0) return; + if (Date.now() >= deadline) throw new Error("storage_pressure: normalized delivery has not quiesced before run rotation"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + } + setServerRequestHandler(handler: CodexServerRequestHandler): void { this.#handler = handler; } @@ -3750,6 +3827,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { throw new Error("native_runner_prp_run_rotation_unavailable"); } await this.#awaitWarmRunAttachmentReady(); + await this.#settleNormalizedDelivery(); const prior = core.store.state.identity; const desired: DurableRecoveryIdentity = { ...prior, @@ -3786,20 +3864,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#authorizedTools, this.options.resumeCompletionContract, ); - this.#runAttachTemplate = structuredClone(runAttachTemplate); + if (core.indexedPersistence) (runAttachTemplate as Record).durability = INDEXED_DURABILITY_CAPABILITY; + this.#runAttachTemplate = structuredClone(runAttachTemplate); const payload = { ...runAttachTemplate, paperclipNextAuthority: { identity: desired, connection }, }; - core.queueCommand("run.attach", payload, commandId, true); + (await core.queueCommand("run.attach", payload, commandId, true)); await this.#waitCommand("run.attach", commandId); - const attached = core.getCommand(commandId); + const attached = (await core.getCommand(commandId)); if (attached?.status !== "completed") { throw new Error("native_runner_prp_run_rotation_failed"); } activationStarted = true; - core.rotateRunIdentity(desired, runAttachTemplate); + (await core.rotateRunIdentity(desired, runAttachTemplate)); await registration?.activate?.(); if (registration?.failure) { void registration.failure.catch((error: unknown) => { @@ -3826,6 +3905,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } this.#eventIdentity = structuredClone(desired); this.#eventSourceSeq = 0; + this.#eventSourceEpoch = undefined; this.#deferredTurnStartEvents = []; this.#durableTurnId = desired.turnId; await previousRelease?.(); @@ -3935,24 +4015,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #readDurableRunnerState(): Promise> { if (this.options.readRunnerState) return this.options.readRunnerState(); - return record( - JSON.parse( - readFileSync( - resolve(this.#root, "runner", "runner-state.json"), - "utf8", - ), - ), - ); + return readRunnerState(resolve(this.#root, "runner", "runner-state.json")); } - #providerDrainState(): - | { - pendingEventCount: number; - activeProviderTurnId: string | null; - providerSettled: boolean; - } - | "unreadable" - | null { + async #providerDrainState(): Promise { if (this.options.runnerFilesystemRoot !== undefined) return null; const provider = this.options.provider ?? "codex"; const filename = @@ -3973,7 +4039,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }; } try { - const state = record(JSON.parse(readFileSync(statePath, "utf8"))); + const state = await readRunnerState(statePath); return providerDrainStateFromSnapshot(state); } catch { return "unreadable"; @@ -3981,7 +4047,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } async #stopActiveProviderTurnBeforeSuspend(deadline: number): Promise { - const state = this.#providerDrainState(); + const state = await this.#providerDrainState(); const core = this.#core; const inferredActiveProviderTurnId = state === null && @@ -4001,12 +4067,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return; } const commandId = `command_close_stop_${randomUUID().replaceAll("-", "")}`; - core.queueCommand( + (await core.queueCommand( "turn.stop", { reason: "transport closing after durable run terminal" }, commandId, true, - ); + )); while (Date.now() < deadline) { this.#pumpEventsSafely(); const command = core.store.state.commands.find( @@ -4040,8 +4106,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { readProviderState: () => this.#providerDrainState(), semanticResultsSettled: () => core.semanticToolResultsSettled(), commands: () => core.store.state.commands, - queueDrain: (commandId) => { - core.queueCommand("runner.drain", {}, commandId, true); + queueDrain: async (commandId) => { + (await core.queueCommand("runner.drain", {}, commandId, true)); }, pump: () => this.#pumpEventsSafely(), deadline: Date.now() + timeoutMs, @@ -4057,7 +4123,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#diagnostic( "provider suffix did not prove durable drain before bounded runner suspension: " + JSON.stringify({ - providerState: this.#providerDrainState(), + providerState: await this.#providerDrainState(), semanticResultsSettled: core.semanticToolResultsSettled(), drainStatus: lastDrain?.status ?? null, retainedEventsDrained: @@ -4162,8 +4228,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { suspensionRequired = true; runnerSuspended = await awaitRunnerSuspensionBarrier({ commands: () => this.#core?.store.state.commands ?? [], - queueSuspend: (commandId) => { - this.#core?.queueCommand("runner.suspend", {}, commandId, true); + queueSuspend: async (commandId) => { + (await this.#core?.queueCommand("runner.suspend", {}, commandId, true)); }, readRunnerState: async () => { const state = await this.#readDurableRunnerState(); @@ -4242,7 +4308,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // provider state. Quiesce the authenticated route, then probe its // independently verified durable state before releasing the process owner; // an incomplete or identity-conflicting state remains fail-closed. - const finalProviderState = this.#providerDrainState(); + const finalProviderState = await this.#providerDrainState(); const runnerSettled = runnerSuspended && providerDrained && @@ -4309,7 +4375,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const dynamicTools = Array.isArray(params.dynamicTools) ? params.dynamicTools.map(record) : []; - const core = new DurablePrpControlPlane({ + const core = await DurablePrpControlPlane.open({ + authorityStore: await this.options.authorityStoreFactory?.(identity, resolve(this.#root, "control-plane")), stateDirectory: resolve(this.#root, "control-plane"), identity, expectedRunnerVersion: runnerArtifact.version, @@ -4637,10 +4704,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // Preserve the first generation's provider attachment seed independently // of bounded command history. The in-memory copy serves a live warm // continuation; the control-plane copy serves a controller/runner resume. + if (core.indexedPersistence) (runAttachTemplate as Record).durability = INDEXED_DURABILITY_CAPABILITY; this.#runAttachTemplate = structuredClone(runAttachTemplate); - core.persistRunAttachTemplate(runAttachTemplate); - core.queueCommand("run.prepare", runAttachTemplate); - core.queueCommand("session.open", { reuse: "same_session" }); + (await core.persistRunAttachTemplate(runAttachTemplate)); + (await core.queueCommand("run.prepare", runAttachTemplate)); + (await core.queueCommand("session.open", { reuse: "same_session" })); const registration = this.options.controlPlaneRegistration ? await this.options.controlPlaneRegistration(core) : null; @@ -4652,6 +4720,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#controlPlaneRelease = registration.release; } const handle = spawnRunner({ + indexedDurability: core.indexedPersistence, connection: registration?.connection ?? { mode: "connect", connectUrl: registration?.connectUrl ?? core.connectUrl, @@ -4659,7 +4728,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { stateDirectory: this.options.runnerStateDirectory ?? resolve(this.#root, "runner"), identity, - ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), + ticket: (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)), maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 0, @@ -4768,7 +4837,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); let candidateRunnerState = localStateOwner && existsSync(localRunnerStatePath) - ? readRunnerState(localRunnerStatePath) + ? await readRunnerState(localRunnerStatePath) : null; const hasRunnerWarmBoundary = candidateRunnerState?.warmTransition !== undefined || @@ -4777,10 +4846,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { let controlPlaneState: Record | null; try { controlPlaneState = existsSync(controlPlaneStatePath) - ? readControlPlaneState(controlPlaneDirectory) + ? await readControlPlaneState(controlPlaneDirectory, this.options.readAuthorityState) : null; } catch (error) { - if (localProvider && localStateOwner && !hasRunnerWarmBoundary) { + if (localProvider && localStateOwner && !hasRunnerWarmBoundary && !this.options.authorityStoreFactory) { quarantineLocalRuntimeState(this.#root, error); } throw error; @@ -4901,19 +4970,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (!localProvider) { throw new Error("PRP provider resume state is unavailable"); } - const archivedState = latestArchivedControlPlaneState( + const archivedState = await latestArchivedControlPlaneState( this.#root, desiredIdentity, + this.options.readAuthorityState, ); if (!archivedState) { throw new Error("PRP provider resume state is unavailable"); } if (localStateOwner) { try { - controlPlaneState = rotateLocalAuthorityEpoch( + controlPlaneState = await rotateLocalAuthorityEpoch( this.#root, archivedState, desiredIdentity, + this.options.runnerBinary, ); } catch (error) { quarantineLocalRuntimeState(this.#root, error); @@ -4941,10 +5012,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (localStateOwner) { try { - controlPlaneState = rotateLocalAuthorityEpoch( + controlPlaneState = await rotateLocalAuthorityEpoch( this.#root, controlPlaneState, desiredIdentity, + this.options.runnerBinary, ); } catch (error) { quarantineLocalRuntimeState(this.#root, error); @@ -5032,7 +5104,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { opencodeExecutable!, ) : undefined; - const core = new DurablePrpControlPlane({ + const core = await DurablePrpControlPlane.open({ + authorityStore: await this.options.authorityStoreFactory?.(identity, controlPlaneDirectory), stateDirectory: controlPlaneDirectory, identity, expectedRunnerVersion: runnerArtifact.version, @@ -5091,8 +5164,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ], }); } - this.#runAttachTemplate = structuredClone(runAttachTemplate); - core.queueCommand("run.attach", runAttachTemplate); + if (core.indexedPersistence) (runAttachTemplate as Record).durability = INDEXED_DURABILITY_CAPABILITY; + this.#runAttachTemplate = structuredClone(runAttachTemplate); + (await core.queueCommand("run.attach", runAttachTemplate)); } const committedEvents = core.store.state.committedEvents; const runAttachment = recoveredRunAttachment(core.store.state); @@ -5104,7 +5178,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? `command_resume_probe_${randomUUID().replaceAll("-", "")}` : null; if (recoveryProbeCommandId !== null) { - core.queueCommand("runner.drain", {}, recoveryProbeCommandId); + (await core.queueCommand("runner.drain", {}, recoveryProbeCommandId)); } // A controller retry can open the exact authority after run.attach has // already reached a durable outcome. Re-observe that command instead of @@ -5117,6 +5191,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? committedEvents[runAttachment.providerIdentityEventIndex]!.sourceSeq - 1 : core.store.state.ackedSourceSeq; + if (core.indexedPersistence) { + const consumer = core.normalizedDelivery()!.load(); + this.#eventSourceSeq = Math.max(0, (consumer?.raw.sourceSeq ?? 0) - 1); + this.#eventSourceEpoch = consumer?.raw.sourceEpoch; + for (const pending of (consumer?.driver.pendingRequests ?? []) as Array<{ request: { requestId: string } }>) { + this.#bridgedRuntimeInputs.set(pending.request.requestId, { durableTurnId: this.#durableTurnId }); + } + } const adoptedProviderIdentityIndex = latestProviderIdentityEventIndex(committedEvents); if ( @@ -5268,21 +5350,22 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const bootstrapTicket = adoptedRunner ? null : warmRecovery - ? core.issueWarmTransitionBootstrapTicket( + ? (await core.issueWarmTransitionBootstrapTicket( { transitionId: warmRecovery.transitionId, runnerState: warmRecovery.runnerState, }, RUNNER_BOOTSTRAP_TICKET_TTL_MS, - ) - : core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS); + )) + : (await core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)); if (warmRecovery) { await this.options.authorizeWarmTransitionRecovery?.("before_spawn"); } const handle = adoptedRunner ? null : spawnRunner({ - connection: registration?.connection ?? { + indexedDurability: core.indexedPersistence, + connection: registration?.connection ?? { mode: "connect", connectUrl: registration?.connectUrl ?? core.connectUrl, }, @@ -5356,12 +5439,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } - if (core.getCommand(warmRecovery.commandId)?.status !== "completed") { + if ((await core.getCommand(warmRecovery.commandId))?.status !== "completed") { throw new Error("native_runner_warm_transition_result_unproven"); } this.#durableTurnId = desiredIdentity.turnId; this.#eventIdentity = structuredClone(desiredIdentity); this.#eventSourceSeq = 0; + this.#eventSourceEpoch = undefined; this.#deferredTurnStartEvents = []; if (oldTransitionRegistration && newTransitionRegistration) { await oldTransitionRegistration.release(); @@ -5592,7 +5676,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); while (!providerTurnStarted() && Date.now() < deadline) { this.#throwIfFailed(); - this.#pumpEvents(); + await this.#pumpEvents(); if (providerTurnStarted()) break; if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider turn startup"); @@ -5655,7 +5739,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } } else { - core.queueCommand(type, payload, commandId, true); + (await core.queueCommand(type, payload, commandId, true)); } await this.#waitCommand(type, commandId); } @@ -5668,9 +5752,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const core = this.#core; if (core === null) throw new Error("PRP provider thread is not started"); const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`; - core.queueCommand(type, payload, commandId, true); + (await core.queueCommand(type, payload, commandId, true)); await this.#waitCommand(type, commandId, deadline); - const command = core.getCommand(commandId); + const command = (await core.getCommand(commandId)); if (command?.status !== "completed" || command.type !== type) { throw new Error(`PRP command ${type} omitted its durable result`); } @@ -5730,7 +5814,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { this.#throwIfFailed(); - this.#pumpEvents(); + await this.#pumpEvents(); if ( this.#threadId.length > 0 && (this.#checkpointProviderIdentityExpectation !== null || @@ -5759,7 +5843,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? this.#core?.store.state.commands.find( (candidate) => candidate.type === type, ) - : this.#core?.getCommand(commandId); + : (await this.#core?.getCommand(commandId)); if (command?.status === "completed") return; if (command !== undefined && command.status !== "pending") { throw new Error( @@ -5776,7 +5860,29 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } - #pumpEvents(): void { + #eventPumpPending: Promise | null = null; + + #queueDelivery(event: DurableRecoveryCommittedEvent, ordinal: number, notification: CodexRpcNotification): void { + const port = this.normalizedDelivery(); + if (!port) { this.#queue.push(notification); return; } + const raw = this.#core?.store.state.indexedState?.normalizedDelivery?.raw; + if (raw && raw.sourceEpoch === event.sourceEpoch && (event.sourceSeq < raw.sourceSeq || (event.sourceSeq === raw.sourceSeq && ordinal <= raw.ordinal))) return; + this.#queue.push({ ...notification, paperclipDelivery: { epoch: port.epoch, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}), sourceSeq: event.sourceSeq, ordinal } }); + } + + #queueRawBoundary(event: DurableRecoveryCommittedEvent): void { + if (this.#core?.indexedPersistence) this.#queueDelivery(event, RAW_DELIVERY_FRAME_COMPLETE, { method: "paperclip/rawConsumed", params: {} }); + } + + #pumpEvents(): Promise { + // Timer, drain, and command waiters share one bounded history read. + // A slow database must not accumulate one query every five milliseconds. + return this.#eventPumpPending ??= this.#readAndPumpEvents().finally(() => { + this.#eventPumpPending = null; + }); + } + + async #readAndPumpEvents(): Promise { const core = this.#core; // The controller may activate its new epoch before attachRun observes the // confirmed handoff. Never consume either epoch with the other's cursor. @@ -5787,7 +5893,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ) return; this.#flushPendingTraceRehydrations(); - const events = core.store.state.committedEvents; + // The database inbox absorbs downstream backpressure. Only a bounded + // delivery prefix may be materialized into the driver notification queue. + const consumed = core.store.state.indexedState?.normalizedDelivery?.raw; + if (core.indexedPersistence && (this.#eventSourceEpoch !== consumed?.sourceEpoch || this.#eventSourceSeq - (consumed?.sourceSeq ?? 0) >= 128)) return; + const readIdentity = structuredClone(this.#eventIdentity); + const events = await core.readCommittedEvents(this.#eventSourceSeq, 128, this.#eventSourceEpoch); + if (!this.#eventIdentity || !recoveryIdentityMatches(readIdentity, this.#eventIdentity) || !recoveryIdentityMatches(readIdentity, core.store.state.identity)) return; for (;;) { const deferredEvent = !this.#recoveryTurnBindingPending && @@ -5797,7 +5909,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const event = deferredEvent ?? events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq); - if (event === undefined) return; + if (event === undefined) { + if (core.indexedPersistence && this.#deferredTurnStartEvents.length === 0) { + const transition = await core.readEventEpochTransition(this.#eventSourceEpoch); + const raw = core.store.state.indexedState?.normalizedDelivery?.raw; + if (transition && raw && this.#eventSourceSeq === transition.finalOrdinal && raw.sourceEpoch === this.#eventSourceEpoch && raw.sourceSeq === transition.finalOrdinal && raw.ordinal === RAW_DELIVERY_FRAME_COMPLETE) { + // Publish one explicit boundary only after every preceding frame is + // durably reduced. The next pump waits for this cursor to commit. + this.#eventSourceEpoch = transition.nextEpoch; + this.#eventSourceSeq = 0; + this.#queue.push({ method: "paperclip/rawEpoch", params: {}, paperclipDelivery: { epoch: core.store.state.identity.runId, sourceEpoch: transition.nextEpoch, sourceSeq: 0, ordinal: 0 } }); + } + } + return; + } const fromDeferredQueue = deferredEvent !== undefined; if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) { throw new Error( @@ -5878,6 +6003,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { event.eventType === "session.resumed" ) { this.#applyProviderIdentityEvent(event); + if (core.indexedPersistence && !fromDeferredQueue && this.#deferredTurnStartEvents.some((earlier) => earlier.sourceSeq < event.sourceSeq)) { + // Identity can unblock thread/read while older frames are waiting + // for its turn binding. Its consumer cursor must still follow those + // frames, even though the transport needs the identity immediately. + this.#deferredTurnStartEvents.push(structuredClone(event)); + continue; + } + this.#queueRawBoundary(event); continue; } if (event.eventType === "harness.diagnostic") { @@ -5924,7 +6057,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? event.envelope.turnId : this.#durableTurnId, }); - void this.#handler({ + const bridgedRequest = { id: requestId, method, params, @@ -5932,12 +6065,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { sourceEventId: event.sourceEventId, sourceEventType: event.eventType, }, - }).catch((error) => { + }; + if (core.indexedPersistence) this.#queueDelivery(event, 1, { method: "paperclip/runtimeRequest", params: { request: bridgedRequest } }); + else void this.#handler(bridgedRequest).catch((error) => { this.#failTransport( error instanceof Error ? error : new Error(String(error)), ); }); } + this.#queueRawBoundary(event); continue; } if ( @@ -5950,6 +6086,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ).requestId; if (typeof requestId === "string") this.#bridgedRuntimeInputs.delete(requestId); + this.#queueRawBoundary(event); continue; } const sessionUpdatePayload = record(eventPayload); @@ -5970,7 +6107,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : canonicalMethod ? expandRunnerdCanonicalNotifications(canonicalMethod, eventPayload) : []; + let deliveryOrdinal = 0; for (const payload of notifications) { + deliveryOrdinal++; const method = payload.method; if (typeof method !== "string") continue; const rawParams = record(payload.params); @@ -6080,7 +6219,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } } } - this.#queue.push({ + this.#queueDelivery(event, deliveryOrdinal, { method, params, ...(this.options.environment?.PAPERCLIP_PROVIDER_TRACE_PATH @@ -6093,6 +6232,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : {}), }); } + this.#queueRawBoundary(event); if (this.options.environment?.PAPERCLIP_PROVIDER_TRACE_PATH) { const pending = { sourceEventId: event.sourceEventId, @@ -6277,9 +6417,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#pendingDriverTraceInterpretations = driverRetry; } - #pumpEventsSafely(): void { + async #pumpEventsSafely(): Promise { try { - this.#pumpEvents(); + await this.#pumpEvents(); } catch (error) { this.#failTransport( new Error( @@ -6477,7 +6617,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { let recoveredHandle: RunnerProcessHandle; try { recoveredHandle = restart( - this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), + (await this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS)), ); } catch (error) { lastDetail = error instanceof Error ? error.message : String(error); @@ -6544,6 +6684,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } #failTransport(error: Error): void { + if (this.#failure === null && error.message.includes("Durable authority commit is indeterminate")) this.#diagnostic(error.message); if (this.#failure !== null || this.#closed) return; this.#failure = error; this.#rejectFailureSignal(error); @@ -6587,15 +6728,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } export function defaultCapabilityRunnerdBinary(): string { - const staged = resolve( - packageRoot, - `dist/bin/paperclip-runnerd${executableSuffix}`, - ); - if (existsSync(staged)) return staged; - return resolve( - packageRoot, - `runner/target/debug/paperclip-runnerd${executableSuffix}`, - ); + return defaultStorageRunnerBinary(); } /** Starts an authenticated durable PRP authority, runnerd, and Codex provider transport. */ @@ -6633,6 +6766,8 @@ export const runnerdRecoveryInternals = Object.freeze({ releaseRunnerProcessOwnership, runnerCloseDeadlines, rotatedRunAttachPayload, + latestArchivedControlPlaneState, + rotateLocalAuthorityEpoch, rotateExternalAuthorityEpoch, turnStartCommandResultValid, turnStartNotificationDisposition, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index a71be1998df..308f54198b6 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -1,3 +1,4 @@ +import { advanceSourceCursor, decodeSourceCursor, encodeSourceCursor, type SourceCursor } from "./control-plane/event-epochs.js"; import { randomUUID } from "node:crypto"; import type { @@ -904,6 +905,7 @@ async function consumeTurn( const consumer = (async () => { let eventCount = 0; let highestContiguousSourceSeq = 0; + let highestContiguousSourceEpoch: string | undefined; let governedResult: PrpStructuredRunResult | null = null; let semanticResultProposal: PrpStructuredRunResult | null = null; let sessionGoalObserved = sessionGoal !== undefined; @@ -934,6 +936,7 @@ async function consumeTurn( event, eventCount, highestContiguousSourceSeq, + highestContiguousSourceEpoch, governedResult: result, }; }; @@ -966,12 +969,11 @@ async function consumeTurn( const receipt = await controlPlane.appendEvent(event, { signal: appendAbort.signal, }); + await session.acknowledgeEvent?.(event); if (stopConsumer) throw new Error("native event consumer stopped"); eventCount += receipt.disposition === "committed" ? 1 : 0; - highestContiguousSourceSeq = Math.max( - highestContiguousSourceSeq, - receipt.highestContiguousSourceSeq, - ); + highestContiguousSourceSeq = receipt.highestContiguousSourceSeq; + highestContiguousSourceEpoch = receipt.highestContiguousSourceEpoch; const eventGoal = goalFromEvent(event); const goalChanged = eventGoal !== undefined && goalLifecycleFingerprint(eventGoal) !== goalLifecycleFingerprint(previousGoal); @@ -1127,6 +1129,7 @@ async function consumeTurn( event, eventCount, highestContiguousSourceSeq, + highestContiguousSourceEpoch, governedResult: governedResult ?? (goalStatus(eventGoal) === "complete" ? semanticResultProposal : null) ?? @@ -1146,6 +1149,7 @@ async function consumeTurn( event, eventCount, highestContiguousSourceSeq, + highestContiguousSourceEpoch, governedResult: governedResult ?? (goalStatus(latestSessionGoal) === "complete" ? semanticResultProposal : null) ?? @@ -1173,6 +1177,7 @@ async function consumeTurn( event, eventCount, highestContiguousSourceSeq, + highestContiguousSourceEpoch, governedResult, }; } @@ -1481,6 +1486,15 @@ function checkpointCursor(cursor: string | null | undefined): number { return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0; } +/** This is an allocation high-water mark, not a contiguous delivery ACK. + * Retained ledgers can have holes, so never reuse a larger persisted ordinal. + * Crossing namespaces still requires the exact predecessor transition. */ +function advanceRecoveryCursor(cursor: SourceCursor, event: PrpEvent): SourceCursor { + if (cursor.sourceEpoch !== event.sourceEpoch) return advanceSourceCursor(cursor, event); + if (event.sourceEpochTransition || !Number.isSafeInteger(event.sourceSeq) || event.sourceSeq <= cursor.sourceSeq) throw new Error("native_recovery_replay_did_not_advance"); + return { sourceSeq: event.sourceSeq, ...(event.sourceEpoch ? { sourceEpoch: event.sourceEpoch } : {}) }; +} + async function reconcileRecoveryCursor(input: { controlPlane: ControlPlanePort; checkpoint: PersistedNativeSession; @@ -1488,38 +1502,16 @@ async function reconcileRecoveryCursor(input: { sourceInstanceId: string; signal: AbortSignal; }): Promise { - const checkpointHighWater = checkpointCursor(input.checkpoint.cursor); - let afterSourceSeq = checkpointHighWater; - let persistedHighWater = checkpointHighWater; - while (true) { - const replay = await input.controlPlane.replayEvents( - { - runId: input.runId, - sourceInstanceId: input.sourceInstanceId, - afterSourceSeq, - limit: 1_000, - }, - { signal: input.signal }, - ); + let cursor = decodeSourceCursor(input.checkpoint.cursor); + for (;;) { + const replay = await input.controlPlane.replayEvents({ runId: input.runId, sourceInstanceId: input.sourceInstanceId, + afterSourceSeq: cursor.sourceSeq, sourceEpoch: cursor.sourceEpoch, limit: 1_000 }, { signal: input.signal }); input.signal.throwIfAborted(); - if (replay.events.length === 0) break; - const pageHighWater = replay.events.reduce( - (highest, event) => Math.max(highest, event.sourceSeq), - afterSourceSeq, - ); - if (pageHighWater <= afterSourceSeq) { - throw new Error("native_recovery_replay_did_not_advance"); - } - persistedHighWater = Math.max(persistedHighWater, pageHighWater); - afterSourceSeq = pageHighWater; + if (!replay.events.length) break; + for (const event of replay.events) cursor = advanceRecoveryCursor(cursor, event); } - if ( - persistedHighWater === checkpointHighWater && - input.checkpoint.cursor === String(checkpointHighWater) - ) { - return input.checkpoint; - } - return { ...input.checkpoint, cursor: String(persistedHighWater) }; + const encoded = encodeSourceCursor(cursor); + return input.checkpoint.cursor === encoded ? input.checkpoint : { ...input.checkpoint, cursor: encoded }; } async function replayCheckpointedTurnTerminal(input: { @@ -1529,63 +1521,22 @@ async function replayCheckpointedTurnTerminal(input: { priorTerminalTurnIds: readonly string[]; expectedTurnId?: string | null; }): Promise<{ terminal: PrpEvent; hasPriorResultProposal: boolean } | null> { - let afterSourceSeq = 0; - const terminals: PrpEvent[] = []; - const latestResultProposalByTurn = new Map(); + let cursor: SourceCursor = { sourceSeq: 0 }; + let latest: { terminal: PrpEvent; hasPriorResultProposal: boolean } | null = null; const priorTerminalTurnIds = new Set(input.priorTerminalTurnIds); - while (true) { - const replay = await input.controlPlane.replayEvents({ - runId: input.runId, - sourceInstanceId: input.sourceInstanceId, - afterSourceSeq, - limit: 1_000, - }); - if (replay.events.length === 0) { - // Disposition recovery owns the newest durable provider terminal. An - // older task turn may also have a valid proposal, but selecting it would - // finalize stale work and strand the actual recovery terminal. - const terminal = - [...terminals].sort( - (left, right) => right.sourceSeq - left.sourceSeq, - )[0] ?? null; - const proposalSequence = - latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0; - return terminal === null - ? null - : { - terminal, - hasPriorResultProposal: - proposalSequence > 0 && proposalSequence < terminal.sourceSeq, - }; - } + // A result proposal is only relevant to the currently inspected turn. + let proposedTurn: string | undefined; + for (;;) { + const replay = await input.controlPlane.replayEvents({ runId: input.runId, sourceInstanceId: input.sourceInstanceId, + afterSourceSeq: cursor.sourceSeq, sourceEpoch: cursor.sourceEpoch, limit: 1_000 }); + if (!replay.events.length) return latest; for (const event of replay.events) { - if (event.turnId && event.eventType === "run.result.proposed") { - latestResultProposalByTurn.set( - event.turnId, - Math.max( - latestResultProposalByTurn.get(event.turnId) ?? 0, - event.sourceSeq, - ), - ); - } - if ( - event.turnId && - isTurnTerminal(event) && - (input.expectedTurnId !== undefined && input.expectedTurnId !== null - ? event.turnId === input.expectedTurnId - : !priorTerminalTurnIds.has(event.turnId)) - ) { - terminals.push(structuredClone(event)); + cursor = advanceRecoveryCursor(cursor, event); + if (event.turnId && event.eventType === "run.result.proposed") proposedTurn = event.turnId; + if (event.turnId && isTurnTerminal(event) && (input.expectedTurnId != null ? event.turnId === input.expectedTurnId : !priorTerminalTurnIds.has(event.turnId))) { + latest = { terminal: structuredClone(event), hasPriorResultProposal: proposedTurn === event.turnId }; } } - const pageHighWater = replay.events.reduce( - (highest, event) => Math.max(highest, event.sourceSeq), - afterSourceSeq, - ); - if (pageHighWater <= afterSourceSeq) { - throw new Error("native_recovery_replay_did_not_advance"); - } - afterSourceSeq = pageHighWater; } } @@ -1652,21 +1603,22 @@ async function replayProvesEffectFreeInitialAcpxTurn(input: { } const targetTurnId = terminalTurns[0]!.turnId; const events: PrpEvent[] = []; - let afterSourceSeq = 0; + let cursor: SourceCursor = { sourceSeq: 0 }; try { while (true) { const replay = await input.controlPlane.replayEvents({ runId: input.runId, sourceInstanceId: input.sourceInstanceId, - afterSourceSeq, + afterSourceSeq: cursor.sourceSeq, + sourceEpoch: cursor.sourceEpoch, limit: 1_000, }); if (replay.events.length === 0) break; for (const event of replay.events) { // An incomplete or reordered replay cannot prove absence of work. - if (event.sourceSeq !== afterSourceSeq + 1) return false; + cursor = advanceSourceCursor(cursor, event); events.push(structuredClone(event)); - afterSourceSeq = event.sourceSeq; + if (events.length > 10_000) return false; } } @@ -2035,6 +1987,9 @@ export async function executeNativeSession( throw error; } sessionOriginRunnerInstances.set(session, options.runnerInstanceId); + session.setEventCommitter?.(async (event) => { + await options.controlPlane.appendEvent(event, { signal: AbortSignal.timeout(30_000) }); + }); let sessionClosePromise: Promise | null = null; let sessionQuarantined = false; const quarantineSession = (reason: string) => { @@ -2189,6 +2144,7 @@ export async function executeNativeSession( event: null as PrpEvent | null, eventCount: 0, highestContiguousSourceSeq: 0, + highestContiguousSourceEpoch: undefined as string | undefined, governedResult: null as PrpStructuredRunResult | null, }; const completionSnapshot = @@ -2205,6 +2161,7 @@ export async function executeNativeSession( completedSemanticResultTurnId(completionSnapshot), } : null; + if (completed) await session.flushEventDelivery?.(); if (!completed) { // A recovered driver is authoritative about whether a provider turn is // still active. In particular, drivers normalize the checkpoint race @@ -2295,6 +2252,7 @@ export async function executeNativeSession( eventCount: 0, highestContiguousSourceSeq: replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq, + highestContiguousSourceEpoch: replayedDisposition === null ? undefined : recoveryTerminal.sourceEpoch, governedResult: null, }); // Event consumption must begin before startTurn so an eager provider cannot @@ -2582,7 +2540,7 @@ export async function executeNativeSession( } } } - consumed.highestContiguousSourceSeq = Math.max( + if (!consumed.highestContiguousSourceEpoch) consumed.highestContiguousSourceSeq = Math.max( consumed.highestContiguousSourceSeq, controlReplay.highestContiguousSourceSeq, ); @@ -2599,7 +2557,7 @@ export async function executeNativeSession( accountedControlEventSequences.add(event.sourceSeq); consumed.eventCount += 1; } - consumed.highestContiguousSourceSeq = Math.max( + if (!consumed.highestContiguousSourceEpoch) consumed.highestContiguousSourceSeq = Math.max( consumed.highestContiguousSourceSeq, receipt.highestContiguousSourceSeq, ); @@ -2639,6 +2597,7 @@ export async function executeNativeSession( driverKind: descriptor.name, nativeEventCount: consumed.eventCount, highestContiguousSourceSeq: consumed.highestContiguousSourceSeq, + ...(consumed.highestContiguousSourceEpoch ? { highestContiguousSourceEpoch: consumed.highestContiguousSourceEpoch } : {}), }; }, }); diff --git a/packages/paperclip-runner/src/provider-events.test.ts b/packages/paperclip-runner/src/provider-events.test.ts index f7485a4593e..5b91e3c6190 100644 --- a/packages/paperclip-runner/src/provider-events.test.ts +++ b/packages/paperclip-runner/src/provider-events.test.ts @@ -33,6 +33,16 @@ function envelope( } describe("provider-neutral events", () => { + it.each([1, 2])("accepts lossless tool output references in PRP v%s without allowing arbitrary fields", version => { + const [event] = canonicalProviderEventsFromCodex("item/completed", { item: { + id: "command-output", type: "commandExecution", command: "qualification", status: "completed", aggregatedOutput: "x".repeat(5000), exitCode: 0, + } }); + const reference = { schema: "paperclip.output.body.v1", bodyId: "a".repeat(64), sha256: "a".repeat(64), byteLength: "5000", mediaType: "text/plain; charset=utf-8" }; + const input = { ...envelope(event), schema: `paperclip.prp.event.v${version}`, schemaVersion: version, payload: { ...event.payload, outputBody: reference } }; + expect(validatePrpEvent(input).ok).toBe(true); + expect(validatePrpEvent({ ...input, payload: { ...input.payload, outputBody: { ...reference, byteLength: 5000 } } }).ok).toBe(false); + expect(validatePrpEvent({ ...input, payload: { ...input.payload, unexpected: true } }).ok).toBe(false); + }); it("preserves Codex notice summaries with legacy and empty-message fallbacks", () => { for (const method of ["configWarning", "deprecationNotice", "warning"]) { for (const [params, expected] of [ diff --git a/packages/paperclip-runner/src/provider-events.ts b/packages/paperclip-runner/src/provider-events.ts index 9efd699aed0..99082177f04 100644 --- a/packages/paperclip-runner/src/provider-events.ts +++ b/packages/paperclip-runner/src/provider-events.ts @@ -78,6 +78,7 @@ export interface TypedEventFamilyCapability { } export const CANONICAL_PROVIDER_EVENT_TYPES = [ + "output.body.chunk", "harness.diagnostic", "plan.updated", "tool.execution.started", diff --git a/packages/paperclip-runner/test/cargo-test-binary.ts b/packages/paperclip-runner/test/cargo-test-binary.ts new file mode 100644 index 00000000000..dc233247111 --- /dev/null +++ b/packages/paperclip-runner/test/cargo-test-binary.ts @@ -0,0 +1,34 @@ +import { isAbsolute, resolve } from "node:path"; + +/** Resolve a Cargo-built test fixture without depending on the shared default + * target directory. Relative CARGO_TARGET_DIR values are relative to Cargo's + * workspace (the runner crate workspace), matching Cargo's build invocation. + */ +export function resolveRunnerCargoTestBinary( + runnerWorkspace: string, + profile: "debug" | "release", + binaryName: string, +): string { + const configuredTarget = process.env.CARGO_TARGET_DIR; + const targetDirectory = configuredTarget + ? isAbsolute(configuredTarget) + ? configuredTarget + : resolve(runnerWorkspace, configuredTarget) + : resolve(runnerWorkspace, "target"); + const executable = `${binaryName}${process.platform === "win32" ? ".exe" : ""}`; + return resolve(targetDirectory, profile, executable); +} + +/** Preserve existing staged/default resolution when no isolated Cargo target + * is configured, while honoring CARGO_TARGET_DIR for fixture runs. + */ +export function resolveRunnerCargoTestBinaryOrDefault( + runnerWorkspace: string, + profile: "debug" | "release", + binaryName: string, + defaultBinary: () => string, +): string { + return process.env.CARGO_TARGET_DIR + ? resolveRunnerCargoTestBinary(runnerWorkspace, profile, binaryName) + : defaultBinary(); +} diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.test.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.test.ts new file mode 100644 index 00000000000..6a2a1c87924 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.test.ts @@ -0,0 +1,27 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { visitCommandLines, writeTopLevelNames } from "./file-sync-stream.js"; + +it("streams a listing beyond 32 MiB without retaining its lines", async () => { + let count = 0; + await visitCommandLines(process.execPath, ["-e", `const fs=require('fs'); for(let i=0;i<10000;i++) fs.writeSync(1,'x'.repeat(4096)+'\\n');`], line => { expect(line.length).toBe(4096); count++; }); + expect(count).toBe(10_000); +}, 30_000); + +it("bounds an individual entry and propagates child failure", async () => { + await expect(visitCommandLines(process.execPath, ["-e", "process.stdout.write('x'.repeat(70000))"], () => undefined)).rejects.toThrow("too long"); + await expect(visitCommandLines(process.execPath, ["-e", "process.exit(7)"], () => undefined)).rejects.toThrow("listing failed"); +}); + +it("writes literal NUL-delimited names without shell expansion or a directory argv", async () => { + const root = await mkdtemp(join(tmpdir(), "daytona-catalog-")); + try { + const source = join(root, "source"), names = join(root, "names"); await mkdir(source); + const entries = ["-C", "a name", "line\nbreak", "$(not-a-command)"]; + for (const name of entries) await writeFile(join(source, name), "x"); + await writeTopLevelNames(source, names); + expect((await readFile(names, "utf8")).split("\0").filter(Boolean).sort()).toEqual(entries.map(name => `./${name}`).sort()); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.ts new file mode 100644 index 00000000000..b68ef34cd7a --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync-stream.ts @@ -0,0 +1,37 @@ +import { spawn } from "node:child_process"; +import { open, opendir } from "node:fs/promises"; + +/** NUL-delimited disk catalog; filenames never become a growing argv array. */ +export async function writeTopLevelNames(directory: string, destination: string): Promise { + const file = await open(destination, "wx", 0o600); + try { + for await (const entry of await opendir(directory, { bufferSize: 128 })) await file.writeFile(`./${entry.name}\0`); + } finally { await file.close(); } +} + +/** Stream a tar listing with a per-entry bound instead of a total listing cap. */ +export async function visitCommandLines(command: string, args: string[], visit: (line: string) => void): Promise { + const child = spawn(command, args, { env: { ...process.env, COPYFILE_DISABLE: "1", LC_ALL: "C" }, stdio: ["ignore", "pipe", "pipe"] }); + let pending = "", diagnosticBytes = 0; + const maxLine = 64 * 1024; + const finished = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", code => code === 0 ? resolve() : reject(new Error("Daytona sync archive listing failed"))); + }); + void finished.catch(() => undefined); + child.stderr.on("data", (chunk: Buffer) => { diagnosticBytes += chunk.length; if (diagnosticBytes > maxLine) child.kill(); }); + child.stdout.setEncoding("utf8"); + try { + for await (const chunk of child.stdout) { + pending += String(chunk); let end: number; + while ((end = pending.indexOf("\n")) >= 0) { + if (end > maxLine) throw new Error("Daytona sync archive entry listing too long"); + visit(pending.slice(0, end)); pending = pending.slice(end + 1); + } + if (pending.length > maxLine) throw new Error("Daytona sync archive entry listing too long"); + } + if (pending) visit(pending); + await finished; + if (diagnosticBytes > maxLine) throw new Error("Daytona sync archive listing failed"); + } catch (error) { child.kill(); await finished.catch(() => undefined); throw error; } +} diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts index f2350767539..9649b79576d 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts @@ -1,3 +1,4 @@ +import { visitCommandLines, writeTopLevelNames } from "./file-sync-stream.js"; import path from "node:path"; import os from "node:os"; import { promises as fs, createReadStream, createWriteStream } from "node:fs"; @@ -164,32 +165,13 @@ async function createHostTarball(input: { followSymlinks?: boolean; }): Promise { const excludeArgs = ["._*", ...(input.exclude ?? [])].flatMap((entry) => ["--exclude", entry]); - const entries = (await fs.readdir(input.localDir)).sort((left, right) => left.localeCompare(right)); - if (entries.length === 0) { - // An empty source is valid (blank workspace / empty asset dir). Write a valid - // gzip-compressed empty tar (1024-byte zero EOF marker) so extraction is a - // clean no-op and uses the same transport as non-empty directories. - await fs.writeFile(input.archivePath, await new Promise((resolve, reject) => { - zlib.gzip(Buffer.alloc(1024), (error, compressed) => error ? reject(error) : resolve(compressed)); - })); - return; - } - await execFileAsync( - "tar", - [ - "-cz", - "--no-xattrs", - ...(input.followSymlinks ? ["-h"] : []), - "-f", - input.archivePath, - "-C", - input.localDir, - ...excludeArgs, - "--", - ...entries, - ], - { env: { ...process.env, COPYFILE_DISABLE: "1" }, maxBuffer: 32 * 1024 * 1024 }, - ); + const namesPath = `${input.archivePath}.names`; + try { + await writeTopLevelNames(input.localDir, namesPath); + await execFileAsync("tar", ["-cz", "--no-xattrs", ...(input.followSymlinks ? ["-h"] : []), + "-f", input.archivePath, "-C", input.localDir, ...excludeArgs, "--null", "-T", namesPath], + { env: { ...process.env, COPYFILE_DISABLE: "1" }, maxBuffer: 64 * 1024 }); + } finally { await fs.rm(namesPath, { force: true }); } } /** @@ -251,12 +233,8 @@ export function splitLinkEntryOnce(field: string, delimiter: string): { name: st * targets are inspected; any unparseable line fails closed. */ async function assertTarballEntriesConfined(archivePath: string): Promise { - const { stdout } = await execFileAsync("tar", ["-tvf", archivePath], { - env: { ...process.env, COPYFILE_DISABLE: "1" }, - maxBuffer: 32 * 1024 * 1024, - }); - const lines = stdout.split("\n").filter((line) => line.trim().length > 0); - for (const line of lines) { + await visitCommandLines("tar", ["-tvf", archivePath], (line) => { + if (!line.trim()) return; const parsed = parseTarVerboseListingLine(line); if (!parsed) { throw new Error(`Daytona syncOut refusing tarball with an unparseable entry listing: ${line}`); @@ -287,7 +265,7 @@ async function assertTarballEntriesConfined(archivePath: string): Promise ); } } - } + }); } async function extractHostTarball(input: { archivePath: string; localDir: string }): Promise { @@ -305,8 +283,7 @@ async function countHostFiles(root: string, exclude?: string[]): Promise const excludeSet = new Set(exclude ?? []); let total = 0; const walk = async (dir: string): Promise => { - const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []); - for (const entry of entries) { + for await (const entry of await fs.opendir(dir, { bufferSize: 128 })) { if (excludeSet.has(entry.name)) continue; const full = path.join(dir, entry.name); if (entry.isDirectory()) { @@ -1183,14 +1160,14 @@ async function syncOutDirectoryMapping(input: { // Tar the source in-sandbox (naming top-level entries so no "." self-entry is // embedded), reproducing the `followSymlinks` → `-h` mapping, then stream the // single archive back over the native bulk channel. + const namesPath = `${remoteTar}.names`; const tarScript = [ + "set -e", `cd ${shellQuote(mapping.sourcePath)}`, - "set -- *", - 'if [ "$#" -eq 1 ] && [ "$1" = "*" ] && [ ! -e "$1" ] && [ ! -L "$1" ]; then set --; fi', - 'for entry in .[!.]* ..?*; do [ -e "$entry" ] || [ -L "$entry" ] || continue; set -- "$@" "$entry"; done', - `if [ "$#" -eq 0 ]; then dd if=/dev/zero of=${shellQuote(remoteTar)} bs=1024 count=1; ` + - `else tar -c --no-xattrs ${mapping.followSymlinks ? "-h " : ""}${excludeFlags} -f ${shellQuote(remoteTar)} -- "$@"; fi`, - ].join(" && "); + `trap ${shellQuote(`rm -f -- ${shellQuote(namesPath)}`)} EXIT`, + `find . -mindepth 1 -maxdepth 1 -print0 > ${shellQuote(namesPath)}`, + `tar -c --no-xattrs ${mapping.followSymlinks ? "-h " : ""}${excludeFlags} -f ${shellQuote(remoteTar)} --null -T ${shellQuote(namesPath)}`, + ].join("; "); await assertSandboxCommandOk(sandbox, `sh -c ${shellQuote(tarScript)}`, timeoutSeconds, "syncOut tar"); guardRoundTrips += 1; diff --git a/scripts/tests/native-provider-startup-codex.mjs b/scripts/tests/native-provider-startup-codex.mjs index 2eca77066e5..755a6ea5c71 100644 --- a/scripts/tests/native-provider-startup-codex.mjs +++ b/scripts/tests/native-provider-startup-codex.mjs @@ -157,12 +157,12 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) { await delay(10); } }; - const launch = (core, iteration) => + const launch = async (core, iteration) => spawnRunner({ connectUrl: core.connectUrl, stateDirectory: runnerDirectory, identity, - ticket: core.issueBootstrapTicket(60_000), + ticket: (await core.issueBootstrapTicket(60_000)), maxOutboxBytes: 16 * 1024 * 1024, p0ReserveBytes: 1024 * 1024, maxRuntimeMs: 10_000, @@ -302,7 +302,7 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) { let core = makeCore(); let handle; let firstExit; - const prepare = core.queueCommand( + const prepare = (await core.queueCommand( "run.prepare", { provider: { @@ -329,30 +329,30 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) { }, }, "startup-prepare", - ); - const open = core.queueCommand("session.open", {}, "startup-open"); + )); + const open = (await core.queueCommand("session.open", {}, "startup-open")); try { await core.start(); - handle = launch(core, 1); + handle = await launch(core, 1); // Atomic controller commits publish new snapshots. Observe commands by // stable ID instead of waiting on the original queueCommand object. await waitUntil( - () => - (core.getCommand(open.commandId)?.status ?? "pending") !== "pending", + async () => + ((await core.getCommand(open.commandId))?.status ?? "pending") !== "pending", ); assert.equal( - core.getCommand(prepare.commandId)?.status, + (await core.getCommand(prepare.commandId))?.status, "completed", "prepare_must_succeed", ); assert.equal( - core.getCommand(open.commandId)?.status, + (await core.getCommand(open.commandId))?.status, "failed", "missing_thread_must_fail", ); assert.ok( - core - .getCommand(open.commandId) + (await core + .getCommand(open.commandId)) ?.result?.result?.message?.includes( `no rollout found for thread id ${missingThread}`, ), @@ -451,31 +451,31 @@ if (args.length === 0 || (args.length === 1 && args[0] === "--help")) { ]) await copyFile(source, join(snapshots, name)); const firstFailure = structuredClone( - core.getCommand(open.commandId)?.result, + (await core.getCommand(open.commandId))?.result, ); const ledgerBefore = await readFile(ledger); const identityLedgerBefore = await readFile(identityLedger); core = makeCore(); - const snapshot = core.queueCommand( + const snapshot = (await core.queueCommand( "session.snapshot", {}, "reopened-snapshot", - ); - const reopen = core.queueCommand("session.open", {}, "reopened-open"); + )); + const reopen = (await core.queueCommand("session.open", {}, "reopened-open")); handle = null; let secondExit; try { await core.start(); - handle = launch(core, 2); + handle = await launch(core, 2); await waitUntil( - () => - (core.getCommand(snapshot.commandId)?.status ?? "pending") !== + async () => + ((await core.getCommand(snapshot.commandId))?.status ?? "pending") !== "pending" && - (core.getCommand(reopen.commandId)?.status ?? "pending") !== + ((await core.getCommand(reopen.commandId))?.status ?? "pending") !== "pending", ); for (const queued of [snapshot, reopen]) { - const command = core.getCommand(queued.commandId); + const command = (await core.getCommand(queued.commandId)); assert.ok(command, "reopened_command_must_exist"); assert.equal(command.status, "failed"); assert.ok(