diff --git a/server/src/__tests__/native-process-evidence.test.ts b/server/src/__tests__/native-process-evidence.test.ts new file mode 100644 index 00000000000..b1168e6aaa4 --- /dev/null +++ b/server/src/__tests__/native-process-evidence.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, heartbeatRunEventEpochs, heartbeatRunEvents, heartbeatRuns, nativeRunProcessEvidence } from "@paperclipai/db"; +import { appendHeartbeatRunEvent, appendHeartbeatRunEventInTransaction } from "../services/heartbeat-run-events.js"; +import { readNativeProcessEvidence } from "../services/native-process-evidence.js"; +import { hasNativeLocalProcessStop } from "../services/native-local-process-stop.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("current process evidence", () => { + it("commits launch/stop authority atomically, ignores provider claims, and reads without historical events", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-process-evidence-"); + const db = createDb(temporary.connectionString); + const companyId = "20000000-0000-4000-8000-000000000001"; + const agentId = "20000000-0000-4000-8000-000000000002"; + const runId = "20000000-0000-4000-8000-000000000003"; + const emptyRunId = "20000000-0000-4000-8000-000000000004"; + const event = { companyId, agentId, runId, eventType: "native.local_process_stopped", payload: { processPid: 4321, processGroupId: 4321 } }; + try { + await db.insert(companies).values({ id: companyId, name: "Process fixture", issuePrefix: "PEV" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Process agent" }); + await db.insert(heartbeatRuns).values([runId, emptyRunId].map(id => ({ id, companyId, agentId, status: "running" }))); + expect(await readNativeProcessEvidence(db, companyId, emptyRunId)).toMatchObject({ seq: 0, eventType: null }); + expect(await readNativeProcessEvidence(db, "20000000-0000-4000-8000-000000000099", runId)).toBeNull(); + await appendHeartbeatRunEvent(db, event); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(true); + await expect(db.transaction(async tx => { + await appendHeartbeatRunEventInTransaction(tx, { ...event, eventType: "native.process_start_requested" }); + throw new Error("injected failure before commit"); + })).rejects.toThrow("injected failure"); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(true); + await appendHeartbeatRunEvent(db, { ...event, eventType: "native.process_start_requested" }); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(false); + await appendHeartbeatRunEvent(db, { ...event, nativeSource: { sourceInstanceId: "untrusted-provider", sourceEventId: "forged-stop", sourceSeq: 1, + protocolSchemaVersion: 1, canonicalPayload: { claimed: "stopped" } } }); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(false); + await appendHeartbeatRunEvent(db, event); + await db.execute(sql`alter table heartbeat_run_events rename to unavailable_history`); + // These reads still work: neither a positive nor a negative receipt + // consults the transcript, regardless of its size or availability. + expect(await readNativeProcessEvidence(db, companyId, runId)).toMatchObject({ seq: 4, eventType: event.eventType, processPid: 4321 }); + expect(await readNativeProcessEvidence(db, companyId, emptyRunId)).toMatchObject({ seq: 0, eventType: null }); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(true); + } finally { await temporary.cleanup(); } + }, 60_000); + + it("backfills one latest server-authored receipt once for retained runs", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-process-backfill-"); + const db = createDb(temporary.connectionString); + const companyId = "20000000-0000-4000-8000-000000000011"; + const agentId = "20000000-0000-4000-8000-000000000012"; + const runId = "20000000-0000-4000-8000-000000000013"; + try { + await db.insert(companies).values({ id: companyId, name: "Retained process fixture", issuePrefix: "OLD" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Retained process agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "failed", nextEventSeq: 4 }); + await db.insert(heartbeatRunEvents).values([ + { companyId, runId, agentId, seq: 1, eventType: "native.process_start_requested" }, + { companyId, runId, agentId, seq: 2, eventType: "native.local_process_stopped", payload: { processPid: 5432, processGroupId: 5432 } }, + { companyId, runId, agentId, seq: 3, eventType: "native.process_start_requested", sourceEventId: "provider-claim" }, + ]); + const results = await Promise.all(Array.from({ length: 4 }, () => readNativeProcessEvidence(db, companyId, runId))); + expect(results.every(r => r?.seq === 2 && r.processPid === 5432)).toBe(true); + expect(await db.select().from(nativeRunProcessEvidence)).toHaveLength(1); + // Event retention cannot erase the independently committed stop fact. + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(true); + await appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: "native.process_start_requested" }); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(false); + } finally { await temporary.cleanup(); } + }, 60_000); + + it("selects latest process authority across epochs when local sequence numbers repeat", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-process-epochs-"); + const db = createDb(temporary.connectionString); + const companyId = "20000000-0000-4000-8000-000000000021"; + const agentId = "20000000-0000-4000-8000-000000000022"; + const runId = "20000000-0000-4000-8000-000000000023"; + const nextEpoch = "20000000-0000-4000-8000-000000000024"; + try { + await db.insert(companies).values({ id: companyId, name: "Epoch process fixture", issuePrefix: "EPO" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Epoch process agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "failed", nextEventSeq: 3 }); + await db.insert(heartbeatRunEvents).values([ + { companyId, runId, agentId, seq: 1, eventEpoch: "", eventType: "native.process_start_requested" }, + { companyId, runId, agentId, seq: 2, eventEpoch: "", eventType: "native.local_process_stopped", payload: { processPid: 5432, processGroupId: 5432 } }, + ]); + await db.insert(heartbeatRunEventEpochs).values({ companyId, runId, epoch: "", nextEpoch, finalSeq: 2 }); + await db.update(heartbeatRuns).set({ eventEpoch: nextEpoch, nextEventSeq: 2 }).where(eq(heartbeatRuns.id, runId)); + await db.insert(heartbeatRunEvents).values({ + companyId, runId, agentId, seq: 1, eventEpoch: nextEpoch, eventType: "native.process_start_requested", + }); + + expect(await readNativeProcessEvidence(db, companyId, runId)).toMatchObject({ seq: 1, eventType: "native.process_start_requested", processPid: null }); + expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(false); + } finally { await temporary.cleanup(); } + }, 60_000); +}); diff --git a/server/src/realtime/runner-prp-outbound.test.ts b/server/src/realtime/runner-prp-outbound.test.ts index 4c814c59c50..9f5b80c2a20 100644 --- a/server/src/realtime/runner-prp-outbound.test.ts +++ b/server/src/realtime/runner-prp-outbound.test.ts @@ -1,6 +1,6 @@ -import { EventEmitter } from "node:events"; +import { EventEmitter, once } from "node:events"; import { describe, expect, it, vi } from "vitest"; -import WebSocket from "ws"; +import WebSocket, { WebSocketServer } from "ws"; import type { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js"; import { __runnerPrpOutboundTesting, @@ -11,6 +11,9 @@ import { class FakeSocket extends EventEmitter { readyState: number = WebSocket.OPEN; send = vi.fn(); + pause = vi.fn(); + resume = vi.fn(); + bufferedAmount = 0; close = vi.fn((code?: number) => { this.readyState = WebSocket.CLOSED; this.emit("close", code ?? 1000, Buffer.alloc(0)); @@ -18,6 +21,60 @@ class FakeSocket extends EventEmitter { } describe("runner provider-ingress WebSocket wire", () => { + it("drains a real WebSocket burst in order after a paused durable consumer resumes", async () => { + const server = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await once(server, "listening"); + const address = server.address(); + if (typeof address === "string" || !address) throw new Error("missing test port"); + const accepted = once(server, "connection"); + const client = new WebSocket(`ws://127.0.0.1:${address.port}`); + const wire = new WsJsonWireConnection(client); + const [peer] = await accepted as [WebSocket]; + const delivered: number[] = []; + wire.onJson(value => { delivered.push((value as { ordinal: number }).ordinal); if (delivered.length === 1) wire.pauseRead(); }); + try { + for (let ordinal = 0; ordinal < 80; ordinal++) peer.send(JSON.stringify({ ordinal, text: "x".repeat(512) })); + await vi.waitFor(() => expect(delivered).toHaveLength(1)); + expect(client.readyState).toBe(WebSocket.OPEN); + wire.resumeRead(); + await vi.waitFor(() => expect(delivered).toHaveLength(80)); + expect(delivered).toEqual(Array.from({ length: 80 }, (_, index) => index)); + expect(client.readyState).toBe(WebSocket.OPEN); + } finally { + const closed = once(client, "close"); client.terminate(); peer.terminate(); await closed; + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + } + }); + + it("bounds frames before listener attachment and honors pause during buffered delivery", () => { + const socket = new FakeSocket(), wire = new WsJsonWireConnection(socket as unknown as WebSocket); + socket.emit("message", Buffer.from('{"ordinal":1}'), false); + socket.emit("message", Buffer.from('{"ordinal":2}'), false); + const delivered: unknown[] = []; + wire.onJson(value => { delivered.push(value); if (delivered.length === 1) wire.pauseRead(); }); + expect(delivered).toEqual([{ ordinal: 1 }]); + wire.resumeRead(); + expect(delivered).toEqual([{ ordinal: 1 }, { ordinal: 2 }]); + expect(socket.pause).toHaveBeenCalled(); expect(socket.resume).toHaveBeenCalled(); + wire.pauseRead(); + for (let index = 0; index < 1000; index++) socket.emit("message", Buffer.from('{"small":true}'), false); + expect(socket.close).toHaveBeenCalledOnce(); + expect(socket.close).toHaveBeenCalledWith(1013); + wire.resumeRead(); + expect(delivered).toHaveLength(2); + }); + + it("bounds buffered bytes and unconsumed outgoing acknowledgements", () => { + const socket = new FakeSocket(), wire = new WsJsonWireConnection(socket as unknown as WebSocket); + const large = Buffer.from(JSON.stringify({ text: "x".repeat(3 * 1024 * 1024) })); + socket.emit("message", large, false); socket.emit("message", large, false); + expect(socket.close).toHaveBeenCalledWith(1013); + const stalled = new FakeSocket(), output = new WsJsonWireConnection(stalled as unknown as WebSocket); + stalled.bufferedAmount = 4 * 1024 * 1024; + output.sendJson({ kind: "ack" }); + expect(stalled.close).toHaveBeenCalledWith(1013); expect(stalled.send).not.toHaveBeenCalled(); + }); + it("bounds credential refresh by the fixed deadline and cancellation", async () => { const signal = new AbortController().signal; await expect( diff --git a/server/src/realtime/runner-prp-outbound.ts b/server/src/realtime/runner-prp-outbound.ts index 6146b8b7fce..26e9bf9d76a 100644 --- a/server/src/realtime/runner-prp-outbound.ts +++ b/server/src/realtime/runner-prp-outbound.ts @@ -30,18 +30,29 @@ export class WsJsonWireConnection implements PrpWireConnection { #onJson: ((value: unknown) => void) | null = null; #onClose = new Set<(reason: TransportCloseReason) => void>(); #pendingJson: unknown[] = []; + #pendingBytes = 0; + #readPaused = false; #closeReason: TransportCloseReason | null = null; constructor(readonly socket: WebSocket) { socket.on("message", (data, isBinary) => { + if (this.#closeReason) return; if (isBinary) { this.close(1003); return; } try { - const value = JSON.parse(data.toString()) as unknown; - if (this.#onJson === null) { + const text = data.toString(); + const size = Buffer.byteLength(text); + if (size > MAX_FRAME_BYTES) { this.close(1009); return; } + const value = JSON.parse(text) as unknown; + if (this.#onJson === null || this.#readPaused) { + // ws can still emit the rest of its current socket chunk after + // pause(). Hold a bounded suffix until authority admission resumes. + if (this.#pendingJson.length >= 256 || this.#pendingBytes + size > MAX_FRAME_BYTES) { this.close(1013); return; } this.#pendingJson.push(value); + this.#pendingBytes += size; + this.socket.pause(); } else { this.#onJson(value); } @@ -59,7 +70,9 @@ export class WsJsonWireConnection implements PrpWireConnection { sendJson(value: unknown): void { if (this.socket.readyState === WebSocket.OPEN) { - this.socket.send(JSON.stringify(value)); + const text = JSON.stringify(value); + if (this.socket.bufferedAmount + Buffer.byteLength(text) > MAX_FRAME_BYTES) { this.close(1013); return; } + this.socket.send(text); } } @@ -74,7 +87,18 @@ export class WsJsonWireConnection implements PrpWireConnection { onJson(listener: (value: unknown) => void): void { this.#onJson = listener; - for (const value of this.#pendingJson.splice(0)) listener(value); + this.#drainPending(); + } + + pauseRead(): void { this.#readPaused = true; this.socket.pause(); } + resumeRead(): void { this.#readPaused = false; this.#drainPending(); } + #drainPending(): void { + while (!this.#closeReason && !this.#readPaused && this.#onJson && this.#pendingJson.length) { + const value = this.#pendingJson.shift(); + this.#pendingBytes -= Buffer.byteLength(JSON.stringify(value)); + this.#onJson(value); + } + if (!this.#closeReason && !this.#readPaused && this.#onJson) this.socket.resume(); } onClose(listener: (reason: TransportCloseReason) => void): void { @@ -85,6 +109,7 @@ export class WsJsonWireConnection implements PrpWireConnection { #notifyClose(reason: TransportCloseReason): void { if (this.#closeReason) return; this.#closeReason = reason; + this.#pendingJson = []; this.#pendingBytes = 0; for (const listener of this.#onClose) listener(reason); this.#onClose.clear(); } diff --git a/server/src/realtime/runner-prp-ws.test.ts b/server/src/realtime/runner-prp-ws.test.ts index 8263701b3f7..54568c31eb7 100644 --- a/server/src/realtime/runner-prp-ws.test.ts +++ b/server/src/realtime/runner-prp-ws.test.ts @@ -132,35 +132,35 @@ describe("runner PRP websocket route", () => { }); expect( - queueLiveRunnerPrpCommand({ + (await queueLiveRunnerPrpCommand({ companyId: "company-1", issueId: "issue-1", agentId: "agent-1", type: "session.goal.get", - }), + })), ).toMatchObject({ runId: newRunId, commandId: "new-command" }); expect(oldQueueCommand).not.toHaveBeenCalled(); expect(newQueueCommand).toHaveBeenCalledOnce(); await oldRegistration.release(); expect( - queueLiveRunnerPrpCommand({ + (await queueLiveRunnerPrpCommand({ companyId: "company-1", issueId: "issue-1", agentId: "agent-1", type: "session.goal.get", - }), + })), ).toMatchObject({ runId: newRunId, commandId: "new-command" }); expect(newQueueCommand).toHaveBeenCalledTimes(2); await newRegistration.release(); expect( - queueLiveRunnerPrpCommand({ + (await queueLiveRunnerPrpCommand({ companyId: "company-1", issueId: "issue-1", agentId: "agent-1", type: "session.goal.get", - }), + })), ).toBeNull(); server.close(); }); @@ -168,7 +168,15 @@ describe("runner PRP websocket route", () => { it("queues one company-bound, idempotent runtime request resolution", async () => { const server = createServer(); setupRunnerPrpWebSocketServer(server, { apiUrl: "http://127.0.0.1:3213" }); - const queueCommand = vi.fn(() => ({ commandId: "command-resolution-1" })); + const receipts = new Map(); + const queueCommand = vi.fn(async (_type, payload, id) => { + const encoded = JSON.stringify(payload); + if (receipts.has(id) && receipts.get(id) !== encoded) { + throw new Error("Durable PRP command replay conflicts with persisted state."); + } + receipts.set(id, encoded); + return { commandId: "command-resolution-1" }; + }); const runId = "00000000-0000-4000-8000-000000000780"; const registration = await registerRunnerPrpAuthority({ companyId: "company-1", @@ -197,12 +205,10 @@ describe("runner PRP websocket route", () => { resolution: { action: "accept" as const }, }; - expect(queueRunnerPrpRuntimeRequestResolution(input)).toEqual({ - commandId: "command-resolution-1", - }); - expect(queueRunnerPrpRuntimeRequestResolution(input)).toEqual({ - commandId: "command-resolution-1", - }); + expect(await Promise.all([ + queueRunnerPrpRuntimeRequestResolution(input), + queueRunnerPrpRuntimeRequestResolution(input), + ])).toEqual([{ commandId: "command-resolution-1" }, { commandId: "command-resolution-1" }]); expect(queueCommand).toHaveBeenCalledTimes(1); expect(queueCommand).toHaveBeenCalledWith( "request.resolve", @@ -217,46 +223,48 @@ describe("runner PRP websocket route", () => { isInstanceAdmin: true, }, }, - undefined, + expect.stringMatching(/^runtime_resolution_[a-f0-9]{64}$/), true, ); - expect(() => - queueRunnerPrpRuntimeRequestResolution({ + await expect(async () => + (await queueRunnerPrpRuntimeRequestResolution({ ...input, companyId: "company-2", - }), - ).toThrowError("runner_prp_authority_not_active"); - expect(() => - queueRunnerPrpRuntimeRequestResolution({ + })), + ).rejects.toThrowError("runner_prp_authority_not_active"); + await expect(async () => + (await queueRunnerPrpRuntimeRequestResolution({ ...input, resolution: { action: "decline" }, - }), - ).toThrowError(RunnerPrpRuntimeRequestResolutionError); + })), + ).rejects.toThrowError(RunnerPrpRuntimeRequestResolutionError); - expect(() => - queueRunnerPrpRuntimeRequestResolution({ + await expect(async () => + (await queueRunnerPrpRuntimeRequestResolution({ ...input, actor: { type: "user", userId: "ordinary-member", isInstanceAdmin: false, }, - }), - ).toThrowError("native_runtime_request_resolver_denied"); + })), + ).rejects.toThrowError("native_runtime_request_resolver_denied"); for (const pendingRequest of [ { ...input.pendingRequest, companyId: "company-2" }, { ...input.pendingRequest, runId: "00000000-0000-4000-8000-000000000783" }, ]) { - expect(() => queueRunnerPrpRuntimeRequestResolution({ + await expect(async () => (await queueRunnerPrpRuntimeRequestResolution({ ...input, pendingRequest, - })).toThrowError("runner_prp_authority_not_active"); + }))).rejects.toThrowError("runner_prp_authority_not_active"); } - expect(queueCommand).toHaveBeenCalledTimes(1); + expect(queueCommand).toHaveBeenCalledTimes(2); + expect(await queueRunnerPrpRuntimeRequestResolution(input)).toEqual({ commandId: "command-resolution-1" }); + expect(receipts.size).toBe(1); await registration.release(); - expect(() => queueRunnerPrpRuntimeRequestResolution(input)).toThrowError( + await expect(async () => (await queueRunnerPrpRuntimeRequestResolution(input))).rejects.toThrowError( "runner_prp_authority_not_active", ); server.close(); @@ -329,14 +337,14 @@ describe("runner PRP websocket route", () => { }, }; // Registering the future URL cannot dispatch into the old authority. - expect(() => - queueRunnerPrpRuntimeRequestResolution(nextInput), - ).toThrowError( + await expect(async () => + (await queueRunnerPrpRuntimeRequestResolution(nextInput)), + ).rejects.toThrowError( "runner_prp_authority_not_active", ); expect(authority.store.state.commands).toEqual([]); if (cached) { - const queued = queueRunnerPrpRuntimeRequestResolution(input); + const queued = (await queueRunnerPrpRuntimeRequestResolution(input)); const command = authority.store.state.commands.find( (candidate) => candidate.commandId === queued.commandId, )!; @@ -344,20 +352,20 @@ describe("runner PRP websocket route", () => { command.status = "completed"; command.result = { status: "completed" }; } - authority.rotateRunIdentity(nextIdentity); + (await authority.rotateRunIdentity(nextIdentity)); expect(authority.store.state.identity).toEqual(nextIdentity); expect(authority.store.state.commands).toEqual([]); const statePath = join(directory, "control-plane-state.json"); const before = readFileSync(statePath, "utf8"); - expect(() => queueRunnerPrpRuntimeRequestResolution(input)).toThrowError( + await expect(async () => (await queueRunnerPrpRuntimeRequestResolution(input))).rejects.toThrowError( "runner_prp_authority_not_active", ); expect(authority.store.state.commands).toEqual([]); expect(readFileSync(statePath, "utf8")).toBe(before); - const next = queueRunnerPrpRuntimeRequestResolution(nextInput); - expect(queueRunnerPrpRuntimeRequestResolution(nextInput)).toEqual(next); + const next = (await queueRunnerPrpRuntimeRequestResolution(nextInput)); + expect((await queueRunnerPrpRuntimeRequestResolution(nextInput))).toEqual(next); expect(authority.store.state.commands).toHaveLength(1); expect(authority.store.state.commands[0]).toMatchObject({ commandId: next.commandId, diff --git a/server/src/realtime/runner-prp-ws.ts b/server/src/realtime/runner-prp-ws.ts index 5ac629e89a1..e66656ab2e7 100644 --- a/server/src/realtime/runner-prp-ws.ts +++ b/server/src/realtime/runner-prp-ws.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; +import { authorityJson } from "../vendor/paperclip-runner/index.js"; import type { IncomingMessage, Server } from "node:http"; import type { Duplex } from "node:stream"; @@ -25,7 +27,7 @@ interface RegisteredAuthority { readonly generation: symbol; readonly runtimeRequestResolutions: Map< string, - { readonly fingerprint: string; readonly commandId: string } + { readonly fingerprint: string; readonly result: Promise<{ readonly commandId: string }> } >; } @@ -172,30 +174,30 @@ export async function registerRunnerPrpAuthority(input: { }; } -export function queueLiveRunnerPrpCommand(input: { +export async function queueLiveRunnerPrpCommand(input: { companyId: string; issueId: string; agentId: string; type: string; payload?: Record; commandId?: string; -}): { +}): Promise<{ runId: string; commandId: string; controllerSeq: number; completion: Promise | null>; -} | null { +} | null> { const current = currentLiveAuthorities.get(liveAuthorityKey(input)); if (!current) return null; const binding = registrations.get(current.runId); if (!binding || binding.generation !== current.generation) return null; const runId = current.runId; - const command = binding.authority.queueCommand( + const command = (await binding.authority.queueCommand( input.type, input.payload ?? {}, input.commandId, true, - ); + )); return { runId, commandId: command.commandId, @@ -203,7 +205,7 @@ export function queueLiveRunnerPrpCommand(input: { completion: (async () => { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { - const outcome = binding.authority.commandOutcome(command.commandId); + const outcome = (await binding.authority.commandOutcome(command.commandId)); if (!outcome) { throw new Error(`runner_prp_command_missing:${command.commandId}`); } @@ -241,13 +243,13 @@ export class RunnerPrpRuntimeRequestResolutionError extends Error { * Identical browser retries reuse the original command; a different answer for * the same request fails closed instead of answering the provider twice. */ -export function queueRunnerPrpRuntimeRequestResolution(input: { +export async function queueRunnerPrpRuntimeRequestResolution(input: { readonly companyId: string; readonly runId: string; readonly pendingRequest: PendingNativeRuntimeRequest; readonly actor: NativeRuntimeRequestResolver; readonly resolution: HarnessRuntimeRequestResolution; -}): { readonly commandId: string } { +}): Promise<{ readonly commandId: string }> { const registration = registrations.get(input.runId); if ( !registration @@ -274,39 +276,45 @@ export function queueRunnerPrpRuntimeRequestResolution(input: { // but only this edge owns the durable command mutation. assertNativeRuntimeRequestResolverAuthorized(pending, input.actor); - const fingerprint = JSON.stringify({ + const payload = { + requestId: pending.requestId, requestKind: pending.requestKind, turnId: pending.turnId, - actor: input.actor, resolution: input.resolution, - }); + resolutionActor: input.actor, + }; + const fingerprint = authorityJson(payload); const previous = registration.runtimeRequestResolutions.get(pending.requestId); if (previous) { if (previous.fingerprint !== fingerprint) { - throw new RunnerPrpRuntimeRequestResolutionError( - "runtime_request_resolution_conflict", - ); + throw new RunnerPrpRuntimeRequestResolutionError("runtime_request_resolution_conflict"); } - return { commandId: previous.commandId }; + return previous.result; + } + // A stable identity lets the durable receipt own retries across restarts. + // This map only coalesces concurrent requests; settled history stays on disk. + const commandId = `runtime_resolution_${createHash("sha256") + .update(authorityJson([input.companyId, input.runId, pending.requestId])) + .digest("hex")}`; + const result = (async () => { + try { + const command = await registration.authority.queueCommand( + "request.resolve", payload, commandId, true, + ); + return { commandId: command.commandId }; + } catch (error) { + if (error instanceof Error && error.message === "Durable PRP command replay conflicts with persisted state.") { + throw new RunnerPrpRuntimeRequestResolutionError("runtime_request_resolution_conflict"); + } + throw error; + } + })(); + registration.runtimeRequestResolutions.set(pending.requestId, { fingerprint, result }); + try { + return await result; + } finally { + registration.runtimeRequestResolutions.delete(pending.requestId); } - - const command = registration.authority.queueCommand( - "request.resolve", - { - requestId: pending.requestId, - requestKind: pending.requestKind, - turnId: pending.turnId, - resolution: input.resolution, - resolutionActor: input.actor, - }, - undefined, - true, - ); - registration.runtimeRequestResolutions.set(pending.requestId, { - fingerprint, - commandId: command.commandId, - }); - return { commandId: command.commandId }; } export const runnerPrpWebSocketInternals = { diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index b0572776052..66b442314eb 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -1,6 +1,7 @@ -import { and, desc, eq, inArray, isNull } from "drizzle-orm"; -import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, type Db } from "@paperclipai/db"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; +import { readNativeProcessEvidence } from "./native-process-evidence.js"; export const PROCESS_START_REQUESTED = "native.process_start_requested"; export const PROCESS_IDENTITY_RECORDED = "native.process_identity_recorded"; @@ -46,16 +47,7 @@ export async function recordNativeLocalProcessStop(db: Db, run: typeof heartbeat /** Only server-authored evidence counts. A later launch invalidates the receipt. */ export async function hasNativeLocalProcessStop(db: Db, companyId: string, runId: string) { - const [event] = await db.select({ eventType: heartbeatRunEvents.eventType }) - .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.companyId, companyId), - eq(heartbeatRunEvents.runId, runId), - isNull(heartbeatRunEvents.sourceEventId), - inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, LOCAL_PROCESS_STOPPED]), - )) - .orderBy(desc(heartbeatRunEvents.seq)) - .limit(1); + const event = await readNativeProcessEvidence(db, companyId, runId); return event?.eventType === LOCAL_PROCESS_STOPPED; } @@ -67,19 +59,16 @@ export async function hasNativeLocalProcessStop(db: Db, companyId: string, runId export async function hasHistoricalSuspendedNativeSession(db: Db, run: typeof heartbeatRuns.$inferSelect) { if (run.runtimeMode !== "native" || run.processPid || run.processGroupId || !run.nativeSessionId || !run.runnerInstanceId || !run.nativeIssueId) return false; - const [modernProcessEvidence] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( - eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - isNull(heartbeatRunEvents.sourceEventId), - inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, LOCAL_PROCESS_STOPPED]), - )).limit(1); + const modernProcessEvidence = await readNativeProcessEvidence(db, run.companyId, run.id); // A newer launch invalidates an old stop receipt. Never bypass that fence // with a suspended file that could belong to the earlier process generation. - if (modernProcessEvidence) return false; + if (modernProcessEvidence?.eventType) return false; const checkpoint = run.runnerProfileJson?.sessionCheckpoint as Record | undefined; if (checkpoint?.providerSessionId != null && (typeof checkpoint.providerSessionId !== "string" || !checkpoint.providerSessionId.trim())) return false; const { nativeFailedRunRetryStateIsSafe } = await import("./native-runtime/native-session-executor.js"); return (await nativeFailedRunRetryStateIsSafe({ + db, execution: run.runnerProfileJson?.nativeExecutionInput, companyId: run.companyId, issueId: run.nativeIssueId, agentId: run.agentId, runId: run.id, nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId, @@ -92,14 +81,9 @@ export async function hasHistoricalSuspendedNativeSession(db: Db, run: typeof he /** Recover the exact stopped identity after the mutable run fields were cleared. */ export async function readNativeLocalProcessStop(db: Db, companyId: string, runId: string) { - const [event] = await db.select({ eventType: heartbeatRunEvents.eventType, payload: heartbeatRunEvents.payload }) - .from(heartbeatRunEvents) - .where(and(eq(heartbeatRunEvents.companyId, companyId), eq(heartbeatRunEvents.runId, runId), - isNull(heartbeatRunEvents.sourceEventId), - inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, LOCAL_PROCESS_STOPPED]))) - .orderBy(desc(heartbeatRunEvents.seq)).limit(1); - const pid = event?.payload?.processPid; - const group = event?.payload?.processGroupId; + const event = await readNativeProcessEvidence(db, companyId, runId); + const pid = event?.processPid; + const group = event?.processGroupId; if (event?.eventType !== LOCAL_PROCESS_STOPPED || typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 1 || group !== pid || !absent(pid) || !absent(-pid)) return null; return { processPid: pid, processGroupId: pid }; diff --git a/server/src/services/native-process-evidence.ts b/server/src/services/native-process-evidence.ts new file mode 100644 index 00000000000..922d607566d --- /dev/null +++ b/server/src/services/native-process-evidence.ts @@ -0,0 +1,38 @@ +import { and, eq } from "drizzle-orm"; +import { heartbeatRunEvents, heartbeatRuns, nativeRunProcessEvidence, type Db } from "@paperclipai/db"; +import { readRunEventLane } from "./run-event-history.js"; + +export const NATIVE_PROCESS_EVENT_TYPES = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"] as const; +type Transaction = Parameters[0]>[0]; +const pid = (value: unknown) => typeof value === "number" && Number.isSafeInteger(value) && value > 1 && value <= 2_147_483_647 ? value : null; + +/** Only call under the run row lock, in the same transaction as event append. */ +export async function recordNativeProcessEvidence(tx: Transaction, row: typeof heartbeatRunEvents.$inferSelect): Promise { + if (row.sourceEventId !== null || !NATIVE_PROCESS_EVENT_TYPES.includes(row.eventType as typeof NATIVE_PROCESS_EVENT_TYPES[number])) return; + const value = { companyId: row.companyId, runId: row.runId, seq: row.seq, eventType: row.eventType, + processPid: pid(row.payload?.processPid), processGroupId: pid(row.payload?.processGroupId) }; + await tx.insert(nativeRunProcessEvidence).values(value).onConflictDoUpdate({ + target: [nativeRunProcessEvidence.companyId, nativeRunProcessEvidence.runId], set: value, + }); +} + +/** Older runs need one bounded-memory compatibility scan. Persist even the + * absence of process evidence, so old/no-process sessions never rescan output. + * The run lock fences concurrent launch/stop writers during this backfill. */ +export async function readNativeProcessEvidence(db: Db, companyId: string, runId: string) { + const scope = and(eq(nativeRunProcessEvidence.companyId, companyId), eq(nativeRunProcessEvidence.runId, runId)); + const [current] = await db.select().from(nativeRunProcessEvidence).where(scope); + if (current) return current; + return db.transaction(async tx => { + const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).for("update"); + if (!run) return null; + const [existing] = await tx.select().from(nativeRunProcessEvidence).where(scope); + if (existing) return existing; + const [latest] = await readRunEventLane(tx as unknown as Db, runId, "native-process", 1); + if (latest) await recordNativeProcessEvidence(tx, latest); + else await tx.insert(nativeRunProcessEvidence).values({ companyId, runId, seq: 0 }); + const [saved] = await tx.select().from(nativeRunProcessEvidence).where(scope); + return saved!; + }); +} diff --git a/server/src/services/native-runtime/cargo-test-binary.test.ts b/server/src/services/native-runtime/cargo-test-binary.test.ts new file mode 100644 index 00000000000..1a965411f5f --- /dev/null +++ b/server/src/services/native-runtime/cargo-test-binary.test.ts @@ -0,0 +1,37 @@ +import { resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "./cargo-test-binary.js"; + +afterEach(() => vi.unstubAllEnvs()); + +describe("isolated Cargo integration binary resolution", () => { + const workspace = resolve("/tmp/paperclip-runner-workspace"); + const executable = (name: string) => `${name}${process.platform === "win32" ? ".exe" : ""}`; + + it("preserves the normal target path when CARGO_TARGET_DIR is unset", () => { + vi.stubEnv("CARGO_TARGET_DIR", ""); + expect(resolveRunnerCargoTestBinary(workspace, "debug", "fake-codex-app-server")) + .toBe(resolve(workspace, "target", "debug", executable("fake-codex-app-server"))); + }); + + it("resolves absolute and workspace-relative target directories", () => { + vi.stubEnv("CARGO_TARGET_DIR", "/tmp/isolated-cargo-target"); + expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd")) + .toBe(resolve("/tmp/isolated-cargo-target", "release", executable("paperclip-runnerd"))); + vi.stubEnv("CARGO_TARGET_DIR", "target/history-development"); + expect(resolveRunnerCargoTestBinary(workspace, "release", "paperclip-runnerd")) + .toBe(resolve(workspace, "target/history-development", "release", executable("paperclip-runnerd"))); + }); + + it("uses staged/default resolution only without an isolated target", () => { + const fallback = vi.fn(() => "/staged/paperclip-runnerd"); + vi.stubEnv("CARGO_TARGET_DIR", ""); + expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback)) + .toBe("/staged/paperclip-runnerd"); + expect(fallback).toHaveBeenCalledOnce(); + vi.stubEnv("CARGO_TARGET_DIR", "target/isolated"); + expect(resolveRunnerCargoTestBinaryOrDefault(workspace, "debug", "paperclip-runnerd", fallback)) + .toBe(resolve(workspace, "target/isolated", "debug", executable("paperclip-runnerd"))); + expect(fallback).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/services/native-runtime/cargo-test-binary.ts b/server/src/services/native-runtime/cargo-test-binary.ts new file mode 100644 index 00000000000..c9176e8ad98 --- /dev/null +++ b/server/src/services/native-runtime/cargo-test-binary.ts @@ -0,0 +1,29 @@ +import { isAbsolute, resolve } from "node:path"; + +/** Resolve Cargo integration-test binaries under the target directory used by + * the matching runner workspace build. + */ +export function resolveRunnerCargoTestBinary( + runnerWorkspace: string, + profile: "debug" | "release", + binaryName: string, +): string { + const configuredTarget = process.env.CARGO_TARGET_DIR; + const targetDirectory = configuredTarget + ? isAbsolute(configuredTarget) + ? configuredTarget + : resolve(runnerWorkspace, configuredTarget) + : resolve(runnerWorkspace, "target"); + return resolve(targetDirectory, profile, `${binaryName}${process.platform === "win32" ? ".exe" : ""}`); +} + +export function resolveRunnerCargoTestBinaryOrDefault( + runnerWorkspace: string, + profile: "debug" | "release", + binaryName: string, + defaultBinary: () => string, +): string { + return process.env.CARGO_TARGET_DIR + ? resolveRunnerCargoTestBinary(runnerWorkspace, profile, binaryName) + : defaultBinary(); +} diff --git a/server/src/services/native-runtime/chat-attachment-reuse.test.ts b/server/src/services/native-runtime/chat-attachment-reuse.test.ts index 324dab123b3..a6cbcb4f43a 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.test.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { activityLog, @@ -25,6 +25,7 @@ import { issueComments, issues, issueWorkProducts, + nativeToolReceipts, toolApplications, toolConnections, } from "@paperclipai/db"; @@ -503,66 +504,85 @@ describe("native same-conversation chat attachment reuse", () => { reusedFromCommentId: sourceCommentId, }), }); - const [run] = await db - .select({ resultJson: heartbeatRuns.resultJson }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)); - const receipts = ( - run.resultJson as { - semanticToolReceipts: Record< - string, - { - operationId: string; - input: unknown; - result: unknown; - } - >; + const receiptBinding = { companyId, issueId, runId }; + const receiptKeys = ["reuse-earlier-v1", "reuse-earlier-v2"]; + type StoredReceiptRow = { idempotencyKey: string; receipt: unknown }; + const readReceiptRows = () => db.select({ + idempotencyKey: nativeToolReceipts.idempotencyKey, + receipt: nativeToolReceipts.receipt, + }).from(nativeToolReceipts).where(and( + eq(nativeToolReceipts.companyId, companyId), + eq(nativeToolReceipts.runId, runId), + inArray(nativeToolReceipts.idempotencyKey, receiptKeys), + )); + const writeReceiptRows = async (rows: StoredReceiptRow[]) => { + for (const row of rows) { + await db.update(nativeToolReceipts).set({ receipt: row.receipt }).where(and( + eq(nativeToolReceipts.companyId, companyId), + eq(nativeToolReceipts.runId, runId), + eq(nativeToolReceipts.idempotencyKey, row.idempotencyKey), + )); } - ).semanticToolReceipts; + }; + const receiptRows = await readReceiptRows(); + expect(receiptRows.map(({ idempotencyKey }) => idempotencyKey).sort()).toEqual(receiptKeys); + const receipts = Object.fromEntries(receiptRows.map(row => [row.idempotencyKey, row.receipt])) as Record< + string, + { operationId: string; input: unknown; result: unknown } + >; expect(receipts["reuse-earlier-v1"]).toMatchObject({ operationId: "reuse_chat_attachment", input: call.arguments, }); expect(receipts["reuse-earlier-v1"]?.result).toEqual(first); expect(receipts["reuse-earlier-v2"]?.result).toEqual(duplicate); - const verifyReceipt = (semanticToolReceipts: unknown) => validateNativeDeliverableEvidence(db, { - companyId, issueId, runId, objective: "Send me that file again.", semanticToolReceipts, + const verifyReceipt = () => validateNativeDeliverableEvidence(db, { + ...receiptBinding, objective: "Send me that file again.", semanticToolReceipts: null, }, { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Prepared the requested existing file.", completionClaim: { contractRevision: "test", objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [{ ref: `deliverable:${preparedId}` }], verification: [], attentionRequests: [], artifacts: [], }); - await expect(verifyReceipt(receipts)).resolves.toBeUndefined(); + await expect(verifyReceipt()).resolves.toBeUndefined(); for (const field of ["filename", "byteSize"]) { - const corrupted = structuredClone(receipts); - for (const receipt of Object.values(corrupted)) { - const prepared = (receipt.result as { prepared?: Record }).prepared; + const corrupted = structuredClone(receiptRows); + for (const row of corrupted) { + const receipt = row.receipt as { result?: Record }; + const prepared = (receipt.result?.prepared as Record | undefined); if (prepared) prepared[field] = field === "filename" ? "different.txt" : sourceBody.length + 1; } - await expect(verifyReceipt(corrupted)).rejects.toThrow("this run's requested output"); + await writeReceiptRows(corrupted); + await expect(verifyReceipt()).rejects.toThrow("this run's requested output"); + await writeReceiptRows(receiptRows); } - const legacy = structuredClone(receipts); - for (const receipt of Object.values(legacy)) { - const prepared = (receipt.result as { prepared?: Record }).prepared; + const legacyRows = structuredClone(receiptRows); + for (const row of legacyRows) { + const receipt = row.receipt as { result?: Record }; + const prepared = receipt.result?.prepared as Record | undefined; if (prepared) { delete prepared.filename; delete prepared.byteSize; } } - await expect(verifyReceipt(legacy)).resolves.toBeUndefined(); + await writeReceiptRows(legacyRows); + await expect(verifyReceipt()).resolves.toBeUndefined(); + await writeReceiptRows(receiptRows); const [sourceRow] = await db.select({ attachment: issueAttachments, asset: assets }).from(issueAttachments) .innerJoin(assets, eq(assets.id, issueAttachments.assetId)).where(eq(issueAttachments.id, sourceAttachmentId)); const foreignCompanyId = randomUUID(); await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated receipt source" }); try { for (const mutation of ["missing", "filename", "size", "hash", "foreign company"]) { + await writeReceiptRows(legacyRows); if (mutation === "missing") await db.delete(issueAttachments).where(eq(issueAttachments.id, sourceAttachmentId)); if (mutation === "filename") await db.update(assets).set({ originalFilename: "different.txt" }).where(eq(assets.id, sourceRow.asset.id)); if (mutation === "size") await db.update(assets).set({ byteSize: sourceBody.length + 1 }).where(eq(assets.id, sourceRow.asset.id)); if (mutation === "hash") await db.update(assets).set({ sha256: "0".repeat(64) }).where(eq(assets.id, sourceRow.asset.id)); if (mutation === "foreign company") await db.update(issueAttachments).set({ companyId: foreignCompanyId }).where(eq(issueAttachments.id, sourceAttachmentId)); try { - await expect(verifyReceipt(legacy)).rejects.toThrow("this run's requested output"); + await expect(verifyReceipt()).rejects.toThrow("this run's requested output"); // New receipts preserve the verified tuple even after the source is removed. - await expect(verifyReceipt(receipts)).resolves.toBeUndefined(); + await writeReceiptRows(receiptRows); + await expect(verifyReceipt()).resolves.toBeUndefined(); } finally { + await writeReceiptRows(receiptRows); if (mutation === "missing") await db.insert(issueAttachments).values(sourceRow.attachment); else await db.update(issueAttachments).set({ companyId }).where(eq(issueAttachments.id, sourceAttachmentId)); await db.update(assets).set({ originalFilename: sourceRow.asset.originalFilename, @@ -572,6 +592,8 @@ describe("native same-conversation chat attachment reuse", () => { } finally { await db.delete(companies).where(eq(companies.id, foreignCompanyId)); } + const [run] = await db.select({ resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const completedResult = mergeHeartbeatRunResultJson( { ...(run.resultJson ?? {}), @@ -591,14 +613,10 @@ describe("native same-conversation chat attachment reuse", () => { .select({ resultJson: heartbeatRuns.resultJson }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) - )[0]?.resultJson as { - semanticToolReceipts: typeof receipts; - nativeResult: { summary: string }; - }; - expect(persistedCompletion.semanticToolReceipts).toEqual(receipts); - expect(persistedCompletion.nativeResult.summary).toBe( - "Prepared the earlier file again.", - ); + )[0]?.resultJson as Record; + expect(persistedCompletion).not.toHaveProperty("semanticToolReceipts"); + expect(persistedCompletion.nativeResult).toMatchObject({ summary: "Prepared the earlier file again." }); + expect(await readReceiptRows()).toEqual(receiptRows); const reuseActivity = ( await db.select().from(activityLog).where(eq(activityLog.runId, runId)) ).find( diff --git a/server/src/services/native-runtime/continuation-contract.test.ts b/server/src/services/native-runtime/continuation-contract.test.ts index d899f6cba6c..f80aad9c00e 100644 --- a/server/src/services/native-runtime/continuation-contract.test.ts +++ b/server/src/services/native-runtime/continuation-contract.test.ts @@ -159,10 +159,10 @@ const support = await getEmbeddedPostgresTestSupport(); }, }); }); - it("incorporates later direction under the matching fenced recovery claim", async () => { + it.each([2, 2_147_483_647])("incorporates later direction only under the matching fenced recovery claim at generation %i", async (controllerGeneration) => { const s = await setup(); - await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery-owner", controllerGeneration: 2, phase: "observed", recoveryState: "resuming_session" }).where(eq(nativeRunFinalizations.runId, s.runId)); - const claim = { kind: "resume_dead_runner" as const, runId: s.runId, leaseOwner: "recovery-owner", controllerGeneration: 2, providerAttempt: 2, restartKind: "hard" as const, recoveryRequestId: null }; + await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery-owner", controllerGeneration, phase: "observed", recoveryState: "resuming_session" }).where(eq(nativeRunFinalizations.runId, s.runId)); + const claim = { kind: "resume_dead_runner" as const, runId: s.runId, leaseOwner: "recovery-owner", controllerGeneration, providerAttempt: 2, restartKind: "hard" as const, recoveryRequestId: null }; await expect(rebindContinuationContract(db, s.oldInput, s.nextInput, { ...claim, leaseOwner: "other-owner" })).rejects.toThrow("requires_fenced_uncompleted_run"); await rebindContinuationContract(db, s.oldInput, s.nextInput, claim); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, s.runId)); diff --git a/server/src/services/native-runtime/evidence-classifier.ts b/server/src/services/native-runtime/evidence-classifier.ts index bc1a8ef35f4..a307a8df000 100644 --- a/server/src/services/native-runtime/evidence-classifier.ts +++ b/server/src/services/native-runtime/evidence-classifier.ts @@ -1,3 +1,4 @@ +import { parseRunEventCursor } from "../run-event-history.js"; import { and, eq, or } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -107,10 +108,13 @@ async function classifyEvidenceRef(input: { runId: string; ref: string; }): Promise { - const eventMatch = /^event:(\d+)$/.exec(input.ref); + const eventMatch = /^event:((?:e:[0-9a-f-]+:)?[0-9]+)$/.exec(input.ref); if (eventMatch) { - const cursor = Number(eventMatch[1]); - const event = await input.db.select({ + let position: ReturnType; + try { position = parseRunEventCursor(eventMatch[1]); } catch { + return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null }; + } + const matches = await input.db.select({ id: heartbeatRunEvents.id, eventType: heartbeatRunEvents.eventType, payload: heartbeatRunEvents.payload, @@ -119,8 +123,16 @@ async function classifyEvidenceRef(input: { .where(and( eq(heartbeatRunEvents.companyId, input.companyId), eq(heartbeatRunEvents.runId, input.runId), - or(eq(heartbeatRunEvents.seq, cursor), eq(heartbeatRunEvents.sourceSeq, cursor)), - )).limit(1).then((rows) => rows[0] ?? null); + or( + and(eq(heartbeatRunEvents.eventEpoch, position.eventEpoch), eq(heartbeatRunEvents.seq, position.seq)), + // Retained numeric references also allowed normalized ordinals. They + // remain legacy-only; a repeated ordinal in a new namespace cannot + // take over the authority of the original evidence reference. + position.eventEpoch === "" ? and(eq(heartbeatRunEvents.sourceEpoch, ""), eq(heartbeatRunEvents.sourceSeq, position.seq)) : undefined, + ), + )).limit(2); + if (matches.length > 1) return { ref: input.ref, kind: "event", outcome: "unverifiable", reasonCode: "durable_run_event_reference_ambiguous", durableRecordId: null }; + const event = matches[0]; if (!event) { return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null }; } diff --git a/server/src/services/native-runtime/external-chat-wait.integration.test.ts b/server/src/services/native-runtime/external-chat-wait.integration.test.ts index 8264b87519b..67265dbd48c 100644 --- a/server/src/services/native-runtime/external-chat-wait.integration.test.ts +++ b/server/src/services/native-runtime/external-chat-wait.integration.test.ts @@ -3604,6 +3604,7 @@ describe("native external-chat response wait", () => { agentId: fixture.agentId, issueId: fixture.issueId, eventType: "run.presentation.resolved", + commitHint: true, }]); await expect(Promise.all(visibilityChecks)).resolves.toEqual([ [{ runId: fixture.runId }], diff --git a/server/src/services/native-runtime/history-payload-s3.qualification.test.ts b/server/src/services/native-runtime/history-payload-s3.qualification.test.ts new file mode 100644 index 00000000000..8986f1fe946 --- /dev/null +++ b/server/src/services/native-runtime/history-payload-s3.qualification.test.ts @@ -0,0 +1,154 @@ +import { execFile } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { S3Client, CreateBucketCommand } from "@aws-sdk/client-s3"; +import { expect, it, vi } from "vitest"; +import { createS3StorageProvider } from "../../storage/s3-provider.js"; +import { HistoryPayloadStore } from "./history-payload-store.js"; +import { createSegmentedRunLogStore } from "../segmented-run-log-store.js"; +import type { StorageProvider } from "../../storage/types.js"; + +const execute = promisify(execFile); +it.skipIf(process.env.PAPERCLIP_HISTORY_S3_QUALIFICATION !== "1")("publishes verified immutable history through a real S3-compatible object server", async () => { + const owner = randomUUID(), name = `paperclip-history-s3-${owner}`; + const accessKeyId = "history-qualification", secretAccessKey = randomBytes(32).toString("hex"); + const env = { ...process.env, MINIO_ROOT_USER: accessKeyId, MINIO_ROOT_PASSWORD: secretAccessKey }; + const docker = async (args: string[]) => (await execute("docker", args, { env, maxBuffer: 128 * 1024, timeout: 30_000 })).stdout.trim(); + const image = await docker(["image", "inspect", "minio/minio:latest", "--format", "{{.Id}}"]); + if (!/^sha256:[a-f0-9]{64}$/.test(image)) throw new Error("S3 qualification requires an installed immutable image ID"); + const started = Date.now(); + let client: S3Client | undefined, created = false; + let evidence: Record | undefined; + let basePath: string | undefined; + try { + // No host mounts, published non-loopback ports, persistent volumes or real + // cloud credentials. Record intent before create so ambiguous replies clean up. + created = true; + await docker(["create", "--name", name, "--label", `paperclip.history-qualification=${owner}`, + "--publish", "127.0.0.1::9000", "--tmpfs", "/data:rw,size=512m", "--env", "MINIO_ROOT_USER", "--env", "MINIO_ROOT_PASSWORD", + image, "server", "/data", "--address", ":9000"]); + await docker(["start", name]); + const inspected = JSON.parse(await docker(["inspect", name]))[0]; + const port = inspected.NetworkSettings.Ports["9000/tcp"]?.[0]; + if (inspected.Config.Labels["paperclip.history-qualification"] !== owner || port?.HostIp !== "127.0.0.1" || !/^[0-9]+$/.test(port.HostPort)) throw new Error("S3 fixture ownership or port mismatch"); + const endpoint = `http://127.0.0.1:${port.HostPort}`; + const deadline = Date.now() + 20_000; + for (;;) { + if (await fetch(`${endpoint}/minio/health/live`).then(r => r.ok, () => false)) break; + if (Date.now() >= deadline) throw new Error("S3 fixture readiness timeout"); + await new Promise(resolve => setTimeout(resolve, 100)); + } + vi.stubEnv("AWS_ACCESS_KEY_ID", accessKeyId); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", secretAccessKey); + vi.stubEnv("AWS_SESSION_TOKEN", ""); + const bucket = `history-${owner}`; + client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true, credentials: { accessKeyId, secretAccessKey } }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + const config = { endpoint, bucket, region: "us-east-1", forcePathStyle: true, prefix: "qualification" }; + const provider = createS3StorageProvider(config), store = new HistoryPayloadStore(provider); + const scope = { companyId: randomUUID(), runId: randomUUID() }, bytes = randomBytes(1024 * 1024); + const ref = await store.put(scope, bytes, "application/json"); + expect(await store.put(scope, bytes, "application/json")).toEqual(ref); + const reopened = new HistoryPayloadStore(createS3StorageProvider(config)); + expect(await reopened.read(scope, ref, bytes.length)).toEqual(bytes); + await expect(reopened.read({ ...scope, companyId: randomUUID() }, ref, bytes.length)).rejects.toThrow("reference_invalid"); + await provider.deleteObject({ objectKey: ref.objectKey }); + await expect(reopened.read(scope, ref, bytes.length)).rejects.toThrow("not found"); + basePath = await mkdtemp(join(tmpdir(), "paperclip-segmented-s3-")); + const reads: string[] = [], uploads: Array<{ key: string; bytes: number }> = []; + const observed: StorageProvider = { + id: "s3", + async getObject(input) { reads.push(input.objectKey); return provider.getObject(input); }, + async putObject(input) { uploads.push({ key: input.objectKey, bytes: input.contentLength }); return provider.putObject(input); }, + async putObjectConditional(input, expected) { uploads.push({ key: input.objectKey, bytes: input.contentLength }); return provider.putObjectConditional!(input, expected); }, + headObject: provider.headObject, deleteObject: provider.deleteObject, + }; + const binding = { companyId: scope.companyId, agentId: randomUUID(), runId: randomUUID() }; + let log = createSegmentedRunLogStore({ basePath, s3: { provider: observed } }); + const handle = await log.begin(binding), expected = createHash("sha256"); + let totalBytes = 0; + for (let index = 0; index < 18; index++) { + const event = { stream: "stdout" as const, ts: `2026-09-28T00:00:${String(index).padStart(2,"0")}Z`, chunk: randomBytes(3 * 1024 * 1024).toString("base64") }; + expected.update(JSON.stringify(event) + "\n"); totalBytes += await log.append(handle, event); + } + await log.flushInflightMirrors!(); + expect(totalBytes).toBeGreaterThan(64 * 1024 * 1024); + const completedUploads = uploads.filter(value => /\/segments\/.*\.ndjson\.[a-f0-9]{64}$/.test(value.key)); + expect(completedUploads.length).toBe(2); + expect(new Set(completedUploads.map(value => value.key)).size).toBe(2); + await rm(basePath, { recursive: true }); + reads.length = 0; + log = createSegmentedRunLogStore({ basePath, s3: { provider: observed } }); + await log.begin(binding); + expect(reads.some(key => key.includes("/segments/"))).toBe(false); + const headBytes = (await stat(join(basePath, handle.logRef, "head.json"))).size; + expect(headBytes).toBeLessThan(1024); + const resumed = { stream: "stdout" as const, ts: "resumed", chunk: "continued after losing the local volume" }; + expected.update(JSON.stringify(resumed) + "\n"); totalBytes += await log.append(handle, resumed); + await log.flushInflightMirrors!(); + const actual = createHash("sha256"); let cursor = "0"; + do { + const page = await log.read(handle, { cursor, limitBytes: 1024 * 1024 }); + actual.update(page.content); cursor = page.cursor!; + if (!page.hasMore) break; + } while (true); + expect(cursor).toBe(String(totalBytes)); + const digest = actual.digest("hex"); expect(digest).toBe(expected.digest("hex")); + const oldKey = completedUploads[0]!.key, corrupt = Buffer.alloc(32 * 1024 * 1024, "!"); + await provider.putObject({ objectKey: oldKey, body: corrupt, contentType: "application/x-ndjson", contentLength: corrupt.length, sha256: createHash("sha256").update(corrupt).digest("hex") }); + await expect(createSegmentedRunLogStore({ basePath, s3: { provider: observed } }).read(handle, { limitBytes: 64 })).rejects.toThrow("segment_digest_mismatch"); + await provider.deleteObject({ objectKey: oldKey }); + // An unavailable archived segment prevents its historical read, but must + // not prevent reopening, appending or finalizing current work. + log = createSegmentedRunLogStore({ basePath, s3: { provider: observed } }); + await log.begin(binding); + const final = { stream: "stdout" as const, ts: "final", chunk: "current work remains available" }; + const beforeFinal = totalBytes; totalBytes += await log.append(handle, final); + expect((await log.finalize(handle)).bytesExact).toBe(String(totalBytes)); + expect((await log.read(handle, { cursor: String(beforeFinal) })).content).toBe(JSON.stringify(final) + "\n"); + await expect(log.read(handle, { limitBytes: 64 })).rejects.toThrow("not found"); + + // Exercise the actual S3 conditional write, including immutable tails: + // an old upload is paused while a replacement claims and publishes. + let reached!: () => void, release!: () => void, armed = false; + const blocked = new Promise(resolve => { reached = resolve; }); + const resumedUpload = new Promise(resolve => { release = resolve; }); + const old = createSegmentedRunLogStore({ basePath: join(basePath, "old-owner"), segmentBytes: 1024, s3: { provider }, + onBoundary: async boundary => { if (armed && boundary === "remote-tail") { armed = false; reached(); await resumedUpload; } } }); + const raceBinding = { ...binding, runId: randomUUID() }, raceHandle = await old.begin(raceBinding); + const initial = { stream: "stdout" as const, ts: "initial", chunk: "before replacement" }; + const continuation = { stream: "stdout" as const, ts: "next", chunk: "replacement output" }; + await old.append(raceHandle, initial); await old.flushInflightMirrors!(); + await old.append(raceHandle, { ...initial, chunk: "late old output" }); armed = true; + const late = old.flushInflightMirrors!(); void late.catch(() => {}); + try { + await blocked; + const replacement = createSegmentedRunLogStore({ basePath: join(basePath, "replacement-owner"), s3: { provider } }); + await replacement.begin(raceBinding); await replacement.append(raceHandle, continuation); await replacement.flushInflightMirrors!(); + release(); await expect(late).rejects.toMatchObject({ $metadata: { httpStatusCode: 412 } }); + const reader = createSegmentedRunLogStore({ basePath: join(basePath, "independent-reader"), s3: { provider } }); + expect((await reader.read(raceHandle)).content).toBe([initial, continuation].map(value => JSON.stringify(value) + "\n").join("")); + } finally { release(); await late.catch(() => {}); } + evidence = { image, backend: "local MinIO S3-compatible API; not hosted AWS", bytes: bytes.length, + sha256: ref.sha256, uploadVerified: true, reopenedReadVerified: true, tenantRejected: true, missingObjectRejected: true, + segmentedLog: { totalBytes, verifiedHistorySha256: digest, headBytes, completedSegments: completedUploads.length, + fullStreamVerified: true, volumeLossRestored: true, reopenFetchedHistoricalSegments: false, corruptionRejected: true, missingHistoryDoesNotBlockCurrentWork: true, + conditionalOwnerReplacementVerified: true, lateWriterRejected: true } }; + } finally { + client?.destroy(); + vi.unstubAllEnvs(); + if (basePath) await rm(basePath, { recursive: true, force: true }); + if (created) { + const found = await docker(["inspect", name]).then(value => JSON.parse(value)[0], () => null); + if (found) { + if (found.Config.Labels["paperclip.history-qualification"] !== owner) throw new Error("S3 fixture cleanup owner mismatch"); + await docker(["rm", "--force", "--volumes", found.Id]); + } + } + } + if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, + JSON.stringify({ ...evidence, status: "passed", cleanupVerified: true, elapsedMs: Date.now() - started }, null, 2), { mode: 0o600 }); +}, 180_000); diff --git a/server/src/services/native-runtime/history-payload-store.test.ts b/server/src/services/native-runtime/history-payload-store.test.ts new file mode 100644 index 00000000000..7b3be2caa28 --- /dev/null +++ b/server/src/services/native-runtime/history-payload-store.test.ts @@ -0,0 +1,43 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { Readable } from "node:stream"; +import { randomUUID } from "node:crypto"; +import { expect, it, vi } from "vitest"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { HistoryPayloadStore } from "./history-payload-store.js"; + +it("publishes exact immutable bytes, reopens them, and fails closed on scope, path, size and corruption", async () => { + const root = await mkdtemp(resolve(tmpdir(), "paperclip-history-payload-")); + try { + const provider = createLocalDiskStorageProvider(root), store = new HistoryPayloadStore(provider); + const scope = { companyId: randomUUID(), runId: randomUUID() }, bytes = Buffer.from("binary\0é".repeat(8192)); + const ref = await store.put(scope, bytes, "application/json"); + expect(await store.put(scope, bytes, "application/json")).toEqual(ref); + expect(await new HistoryPayloadStore(provider).read(scope, ref, bytes.length)).toEqual(bytes); + const get = vi.spyOn(provider, "getObject"); + for (const changed of [ + { ...ref, companyId: randomUUID() }, { ...ref, runId: randomUUID() }, + { ...ref, objectKey: `${scope.companyId}/../some-file` }, { ...ref, provider: "s3" }, + { ...ref, byteLength: bytes.length + 1 }, { ...ref, sha256: "0".repeat(64) }, + ]) await expect(store.read(scope, changed, bytes.length)).rejects.toThrow("history_payload_reference_invalid"); + expect(get).not.toHaveBeenCalled(); + await writeFile(resolve(root, ref.objectKey), Buffer.alloc(bytes.length, 0x78)); + await expect(store.read(scope, ref, bytes.length)).rejects.toThrow("history_payload_integrity_mismatch"); + await rm(resolve(root, ref.objectKey)); + await expect(store.read(scope, ref, bytes.length)).rejects.toThrow("not found"); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("never publishes an upload which the backend lost or changed, and caps a lying stream", async () => { + const scope = { companyId: randomUUID(), runId: randomUUID() }, bytes = Buffer.from("original"); + const put = vi.fn(async () => {}); + const stream = Readable.from([Buffer.alloc(100)]); + const provider = { id: "s3" as const, putObject: put, + getObject: vi.fn(async () => ({ stream })), headObject: vi.fn(), deleteObject: vi.fn() }; + await expect(new HistoryPayloadStore(provider).put(scope, bytes, "application/json")).rejects.toThrow("history_payload_length_mismatch"); + expect(stream.destroyed).toBe(true); + provider.getObject.mockRejectedValueOnce(new Error("lost upload")); + await expect(new HistoryPayloadStore(provider).put(scope, bytes, "application/json")).rejects.toThrow("lost upload"); + expect(put).toHaveBeenCalledTimes(2); +}); diff --git a/server/src/services/native-runtime/history-payload-store.ts b/server/src/services/native-runtime/history-payload-store.ts new file mode 100644 index 00000000000..deb69209e5b --- /dev/null +++ b/server/src/services/native-runtime/history-payload-store.ts @@ -0,0 +1,83 @@ +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; +import { loadConfig } from "../../config.js"; +import { createStorageProviderFromConfig } from "../../storage/provider-registry.js"; +import type { StorageProvider } from "../../storage/types.js"; +import { DurableAuthorityStoreError } from "../../vendor/paperclip-runner/index.js"; + +export interface HistoryPayloadScope { companyId: string; runId: string } +export interface HistoryPayloadReference extends HistoryPayloadScope { + schema: "paperclip.history-payload.v1"; + provider: StorageProvider["id"]; + objectKey: string; + byteLength: number; + sha256: string; + mediaType: "application/json" | "text/plain; charset=utf-8"; +} +const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i; +const hex = /^[a-f0-9]{64}$/; +const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +function objectKey(scope: HistoryPayloadScope, sha256: string): string { + if (!uuid.test(scope.companyId) || !uuid.test(scope.runId) || !hex.test(sha256)) throw new Error("history_payload_scope_invalid"); + return `${scope.companyId}/native-history/${scope.runId}/${sha256.slice(0, 2)}/${sha256}`; +} + +/** Immutable content addressed objects. A reference never accepts a caller's + * path or URL, and every read verifies scope, length and content before use. + * The caller chooses the per-record allocation limit; there is no history cap. */ +export class HistoryPayloadStore { + constructor(private readonly provider: StorageProvider) {} + + private validate(scope: HistoryPayloadScope, value: unknown, maxBytes: number): HistoryPayloadReference { + const ref = value as HistoryPayloadReference | null; + if (!ref || ref.schema !== "paperclip.history-payload.v1" || ref.companyId !== scope.companyId || ref.runId !== scope.runId || + ref.provider !== this.provider.id || !Number.isSafeInteger(ref.byteLength) || ref.byteLength <= 0 || ref.byteLength > maxBytes || + !["application/json", "text/plain; charset=utf-8"].includes(ref.mediaType) || ref.objectKey !== objectKey(scope, ref.sha256)) { + throw new Error("history_payload_reference_invalid"); + } + return ref; + } + + async put(scope: HistoryPayloadScope, bytes: Buffer, mediaType: HistoryPayloadReference["mediaType"]): Promise { + if (!bytes.length) throw new Error("history_payload_empty"); + const sha256 = hash(bytes); + const ref: HistoryPayloadReference = { schema: "paperclip.history-payload.v1", ...scope, + provider: this.provider.id, objectKey: objectKey(scope, sha256), byteLength: bytes.length, sha256, mediaType }; + try { + await this.provider.putObject({ objectKey: ref.objectKey, body: Readable.from([bytes]), + contentType: mediaType, contentLength: bytes.length, sha256 }); + // Publication follows an exact read-back, including on S3-compatible stores + // which do not implement the AWS checksum header. Lost publication leaves + // only a reusable immutable orphan, never a dangling database reference. + await this.read(scope, ref, bytes.length); + } catch (error) { + const code = (error as NodeJS.ErrnoException)?.code; + if (code === "ENOSPC" || code === "EDQUOT") throw new DurableAuthorityStoreError("storage_pressure", "immutable history storage is full"); + throw error; + } + return ref; + } + + async read(scope: HistoryPayloadScope, value: unknown, maxBytes: number): Promise { + const ref = this.validate(scope, value, maxBytes); + const result = await this.provider.getObject({ objectKey: ref.objectKey }); + let length = 0; + const chunks: Buffer[] = []; + try { + if (result.contentLength !== undefined && result.contentLength !== ref.byteLength) throw new Error("history_payload_length_mismatch"); + for await (const value of result.stream) { + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value); + length += bytes.length; + if (length > ref.byteLength) throw new Error("history_payload_length_mismatch"); + chunks.push(bytes); + } + const bytes = Buffer.concat(chunks, length); + if (length !== ref.byteLength || hash(bytes) !== ref.sha256) throw new Error("history_payload_integrity_mismatch"); + return bytes; + } finally { result.stream.destroy(); } + } +} + +export function createHistoryPayloadStore(): HistoryPayloadStore { + return new HistoryPayloadStore(createStorageProviderFromConfig(loadConfig())); +} diff --git a/server/src/services/native-runtime/native-authority-read-context.ts b/server/src/services/native-runtime/native-authority-read-context.ts new file mode 100644 index 00000000000..0124d10962f --- /dev/null +++ b/server/src/services/native-runtime/native-authority-read-context.ts @@ -0,0 +1,50 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { dirname } from "node:path"; +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { authorityJson, parseAuthorityLocator, readIndexedLocalState, assertNoPendingLegacyMigration } from "../../vendor/paperclip-runner/index.js"; +import { readPostgresAuthority } from "./postgres-authority-store.js"; + +const context = new AsyncLocalStorage(); +/** Scope follows the owning recovery operation, never a global database singleton. */ +export function withNativeAuthorityReadContext(db: Db, operation: () => Promise): Promise { + return context.run(db, operation); +} + +export async function readIndexedAuthorityProof(path: string) { + assertNoPendingLegacyMigration(dirname(path)); + const fd = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let locator; + let value: unknown; + let unsafe = false; + try { + const stat = await fd.stat(); + if (!stat.isFile()) throw new Error("native_state_file_unsafe"); + unsafe = process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()); + if (stat.size > 8192) return null; + const buffer = Buffer.alloc(stat.size); + let offset = 0; + while (offset < buffer.length) { const { bytesRead } = await fd.read(buffer, offset, buffer.length - offset, offset); if (!bytesRead) throw new Error("native_state_file_changed"); offset += bytesRead; } + const after = await fd.stat(); + if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("native_state_file_changed"); + try { value = JSON.parse(buffer.toString("utf8")); } catch { return null; } + locator = parseAuthorityLocator(value); + } finally { await fd.close(); } + const schema = (value as { schema?: string } | null)?.schema; + if (unsafe && (locator || schema?.includes(".indexed.v1"))) throw new Error("native_indexed_state_file_unsafe"); + let snapshot; + if (schema === "paperclip.runner.durable.state.indexed.v1" || schema === "paperclip.runner.codex-provider-state.indexed.v1") { + snapshot = await readIndexedLocalState(path); + } else { + if (!locator) return null; + const db = context.getStore(); + if (!db || locator.location.kind !== "postgres") throw new Error("native_indexed_authority_unavailable"); + snapshot = await readPostgresAuthority(db, locator.location); + } + // Include generation in proof hashes: the locator itself is never evidence. + const proof = { ...snapshot.state, indexedGeneration: snapshot.generation }; + const bytes = Buffer.from(authorityJson(proof)); + return { value: proof, bytes, sha256: createHash("sha256").update(bytes).digest("hex"), byteSize: bytes.length, retainedBudgetBytes: bytes.length }; +} diff --git a/server/src/services/native-runtime/native-authority-storage-error.test.ts b/server/src/services/native-runtime/native-authority-storage-error.test.ts new file mode 100644 index 00000000000..d1a1c159df1 --- /dev/null +++ b/server/src/services/native-runtime/native-authority-storage-error.test.ts @@ -0,0 +1,17 @@ +import { expect, it } from "vitest"; +import { classifyNativeAuthorityStorageError } from "./native-authority-storage-error.js"; + +it("classifies bounded structured database/filesystem causes without parsing messages", () => { + for (const code of ["53100", "53200", "53300", "ENOSPC", "EDQUOT"]) { + expect(classifyNativeAuthorityStorageError(new Error("query failed", { cause: Object.assign(new Error("private backend details"), { code }) }))) + .toMatchObject({ code: "storage_pressure", message: "storage_pressure: authority storage capacity is temporarily unavailable" }); + } + for (const code of ["08006", "57P01", "25006", "EROFS", "ECONNRESET"]) { + expect(classifyNativeAuthorityStorageError({ code })).toMatchObject({ code: "storage_unavailable" }); + } + for (const error of [new Error("53100: disk full"), { code: "23505" }, { code: "42501" }, { cause: null }]) { + expect(classifyNativeAuthorityStorageError(error)).toBe(error); + } + const cycle = { cause: null as unknown }; cycle.cause = cycle; + expect(classifyNativeAuthorityStorageError(cycle)).toBe(cycle); +}); diff --git a/server/src/services/native-runtime/native-authority-storage-error.ts b/server/src/services/native-runtime/native-authority-storage-error.ts new file mode 100644 index 00000000000..87cc625899b --- /dev/null +++ b/server/src/services/native-runtime/native-authority-storage-error.ts @@ -0,0 +1,25 @@ +import { DurableAuthorityStoreError } from "../../vendor/paperclip-runner/index.js"; + +/** Classify machine-readable backend conditions, never exception text. This + * does not establish rollback: the controller must separately read and compare + * its exact durable generation before allowing any same-owner retry. */ +export function classifyNativeAuthorityStorageError(error: unknown): unknown { + if (error instanceof DurableAuthorityStoreError) return error; + let cause = error; + for (let depth = 0; depth < 8 && cause && typeof cause === "object"; depth++) { + const detail = cause as { code?: unknown; name?: unknown; cause?: unknown }; + if (detail instanceof DurableAuthorityStoreError) return detail; + // PostgreSQL: disk_full, out_of_memory, too_many_connections. The latter + // two can reject admission without exhausting persistent storage. + if (["53100", "53200", "53300", "ENOSPC", "EDQUOT"].includes(String(detail.code))) { + return new DurableAuthorityStoreError("storage_pressure", "authority storage capacity is temporarily unavailable"); + } + // Preserve an unavailable/indeterminate disposition for connection loss, + // read-only storage and shutdown. A successful COMMIT reply may be lost. + if (["25006", "57P01", "57P02", "57P03", "08000", "08003", "08006", "08007", "08P01", "EROFS", "EIO", "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT"].includes(String(detail.code))) { + return new DurableAuthorityStoreError("storage_unavailable", "authority storage operation could not be confirmed"); + } + cause = detail.cause; + } + return error; +} diff --git a/server/src/services/native-runtime/native-checkpoint-command.test.ts b/server/src/services/native-runtime/native-checkpoint-command.test.ts new file mode 100644 index 00000000000..d7e933b123f --- /dev/null +++ b/server/src/services/native-runtime/native-checkpoint-command.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { runCheckpointCommand } from "./native-checkpoint-command.js"; +import { loopbackCheckpointChannel as localChannel } from "./native-checkpoint-transfer.test-support.js"; + +describe("history-sized checkpoint command", () => { + it("uses an owned long-lived process instead of a short execute RPC", async () => { + const execute = vi.fn().mockRejectedValue(new Error("single-command RPC deadline")); + const openDuplexChannel = vi.fn(async ({ command }) => localChannel(command)); + const runner = { execute, openDuplexChannel } as CommandManagedRuntimeRunner; + expect(await runCheckpointCommand(runner, "sleep 0.05; printf checkpoint-finished")).toBe("checkpoint-finished"); + expect(execute).not.toHaveBeenCalled(); + expect(openDuplexChannel).toHaveBeenCalledOnce(); + }); + it("rejects a real failed command and reaps its owned process", async () => { + const close = vi.fn(); + const runner = { execute: vi.fn(), openDuplexChannel: async ({ command }) => { + const channel = localChannel(command); + const finish = channel.close; + channel.close = async () => { await finish(); close(); }; + return channel; + } } as CommandManagedRuntimeRunner; + await expect(runCheckpointCommand(runner, "exit 7")).rejects.toThrow("checkpoint_transfer_failed"); + expect(close).toHaveBeenCalledOnce(); + }); + it("fails closed on truncated transport and bounds diagnostic output", async () => { + for (const flood of [false, true]) { + const stop = vi.fn(), close = vi.fn(async () => {}); + const runner = { execute: vi.fn(), openDuplexChannel: async () => ({ + stop, close, write() {}, + onData: (listener: (chunk: Uint8Array) => void) => { if (flood) queueMicrotask(() => listener(Buffer.alloc(65 * 1024))); }, + onExit: (listener: (exit: { exitCode: number | null; transportClosed?: boolean }) => void) => { + if (!flood) queueMicrotask(() => listener({ exitCode: null, transportClosed: true })); + }, + }) } as CommandManagedRuntimeRunner; + await expect(runCheckpointCommand(runner, "private checkpoint")).rejects.toThrow("checkpoint_transfer_failed"); + expect(stop).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); + } + }); +}); diff --git a/server/src/services/native-runtime/native-checkpoint-command.ts b/server/src/services/native-runtime/native-checkpoint-command.ts new file mode 100644 index 00000000000..3b8861fb51c --- /dev/null +++ b/server/src/services/native-runtime/native-checkpoint-command.ts @@ -0,0 +1,47 @@ +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; + +/** Archive/hash commands write their large payload to a private remote spool. + * Only this bounded completion output crosses the channel. A long-lived owned + * channel avoids the provider's single-RPC execution deadline. */ +export async function runCheckpointCommand( + runner: CommandManagedRuntimeRunner, + script: string, +): Promise { + if (!runner.openDuplexChannel) { + // SSH and local runners implement execute's explicit zero deadline. A + // provider without duplex still owns its execute contract; its restrictions + // must be reported as provider limitations, not unlimited qualification. + const result = await runner.execute({ command: "sh", args: ["-c", script], bypassSession: true, timeoutMs: 0 }); + if (result.exitCode !== 0 || result.timedOut || Buffer.byteLength(result.stdout) > 64 * 1024) { + throw new Error("runner_remote_checkpoint_transfer_failed"); + } + return result.stdout; + } + const channel = await runner.openDuplexChannel({ command: ["sh", "-c", script] }); + const chunks: Buffer[] = []; + let size = 0; + try { + return await new Promise((resolve, reject) => { + let settled = false; + const fail = () => { + if (settled) return; + settled = true; + // Release only this channel's owned child, never a PID/name search. + try { channel.stop(); } catch { /* The transport can already be closed. */ } + reject(new Error("runner_remote_checkpoint_transfer_failed")); + }; + channel.onData(chunk => { + if (settled) return; + size += chunk.byteLength; + if (size > 64 * 1024) { fail(); return; } + chunks.push(Buffer.from(chunk)); + }); + channel.onExit(exit => { + if (settled) return; + if (exit.transportClosed || exit.exitCode !== 0) { fail(); return; } + settled = true; + resolve(Buffer.concat(chunks, size).toString("utf8")); + }); + }); + } finally { await channel.close(); } +} diff --git a/server/src/services/native-runtime/native-checkpoint-transfer.test-support.ts b/server/src/services/native-runtime/native-checkpoint-transfer.test-support.ts new file mode 100644 index 00000000000..e5c2731d99d --- /dev/null +++ b/server/src/services/native-runtime/native-checkpoint-transfer.test-support.ts @@ -0,0 +1,32 @@ +import { execFile, spawn } from "node:child_process"; +import type { CommandManagedRuntimeRunner, CommandManagedDuplexChannel } from "@paperclipai/adapter-utils/command-managed-runtime"; + +export function loopbackCheckpointChannel(command: readonly string[]): CommandManagedDuplexChannel { + const child = spawn(command[0]!, command.slice(1), { stdio: ["pipe", "pipe", "pipe"] }); + const closed = new Promise(resolve => child.once("close", () => resolve())); + return { + write: bytes => { child.stdin.write(bytes); }, + onData: listener => { child.stdout.on("data", listener); child.stderr.on("data", listener); }, + onExit: listener => { child.once("close", exitCode => listener({ exitCode })); }, + stop: () => { child.kill("SIGKILL"); }, + close: async () => { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); await closed; }, + }; +} + +export function loopbackCheckpointDuplexRunner(): CommandManagedRuntimeRunner { + return { ...loopbackCheckpointRunner(), openDuplexChannel: async ({ command }) => loopbackCheckpointChannel(command) }; +} + +/** Execute the actual command transport on a private test directory. No SSH, + * provider, API keys or Docker resources are borrowed by these protocol tests. */ +export function loopbackCheckpointRunner(onCommand?: (input: Parameters[0], stdout: string) => string): CommandManagedRuntimeRunner { + return { execute: input => new Promise((resolve, reject) => { + const startedAt = new Date().toISOString(); + const child = execFile(input.command, input.args ?? [], { cwd: input.cwd, timeout: input.timeoutMs, maxBuffer: 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { + if (error) { reject(error); return; } + resolve({ exitCode: 0, signal: null, timedOut: false, stdout: onCommand?.(input, stdout) ?? stdout, stderr, pid: child.pid ?? null, startedAt }); + }); + child.stdin?.on("error", () => undefined); + child.stdin?.end(input.stdin); + }) }; +} diff --git a/server/src/services/native-runtime/native-checkpoint-transfer.test.ts b/server/src/services/native-runtime/native-checkpoint-transfer.test.ts new file mode 100644 index 00000000000..7312ad76f18 --- /dev/null +++ b/server/src/services/native-runtime/native-checkpoint-transfer.test.ts @@ -0,0 +1,58 @@ +import { createHash, randomBytes } from "node:crypto"; +import { mkdir, mkdtemp, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { checkpointTar, downloadCheckpointDirectory, uploadCheckpointArchive } from "./native-checkpoint-transfer.js"; +import { loopbackCheckpointRunner } from "./native-checkpoint-transfer.test-support.js"; + +it("round-trips over 64 MiB and 20,000 entries through bounded command chunks", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-checkpoint-scale-")); + try { + const source = join(root, "source"), remote = join(root, "remote's home"), restored = join(root, "restored"), archive = join(root, "history.tar.gz"); + await mkdir(source); await mkdir(join(source, "receipts")); + const chunk = randomBytes(192 * 1024), expected = createHash("sha256"), file = await open(join(source, "history"), "wx", 0o600); + try { for (let index = 0; index < 350; index++) { await file.writeFile(chunk); expected.update(chunk); } } finally { await file.close(); } + for (let first = 0; first < 20_100; first += 100) await Promise.all(Array.from({ length: 100 }, (_, index) => writeFile(join(source, "receipts", `r-${first + index}`), "receipt"))); + await writeFile(join(source, "auth.json"), "must remain local"); + await checkpointTar(["--exclude=./auth.json", "-czf", archive, "-C", source, "."]); + expect((await stat(archive)).size).toBeGreaterThan(64 * 1024 * 1024); + let maxRequest = 0, maxResponse = 0, transfers = 0; + const runner = loopbackCheckpointRunner((input, stdout) => { + maxRequest = Math.max(maxRequest, input.stdin?.length ?? 0); maxResponse = Math.max(maxResponse, stdout.length); + if (input.args?.[1]?.includes("dd ")) transfers++; + return stdout; + }); + await uploadCheckpointArchive({ runner, archive, targetPath: remote, mode: 0o700 }); + await downloadCheckpointDirectory({ runner, sourcePath: remote, targetPath: restored, mode: 0o700 }); + const actual = createHash("sha256"), restoredFile = await open(join(restored, "history"), "r"); + try { for await (const bytes of restoredFile.createReadStream({ autoClose: false })) actual.update(bytes); } finally { await restoredFile.close(); } + expect(actual.digest("hex")).toBe(expected.digest("hex")); + expect(await readdir(join(restored, "receipts"))).toHaveLength(20_100); + await expect(stat(join(restored, "auth.json"))).rejects.toThrow(); + expect(maxRequest).toBeLessThan(300_000); expect(maxResponse).toBeLessThan(300_000); expect(transfers).toBeGreaterThan(600); + expect((await readdir(root)).filter(name => name.startsWith(".paperclip-checkpoint-"))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } +}, 240_000); + +it("rejects a corrupted chunk and preserves the previously durable checkpoint", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-checkpoint-corrupt-")); + try { + const source = join(root, "source"), target = join(root, "target"); await mkdir(source); await mkdir(target); + await writeFile(join(source, "history"), "new"); await writeFile(join(target, "history"), "preserved"); + const runner = loopbackCheckpointRunner((input, stdout) => input.args?.[1]?.includes("dd if=") ? stdout.replace(/./, "!") : stdout); + await expect(downloadCheckpointDirectory({ runner, sourcePath: source, targetPath: target, mode: 0o700 })).rejects.toThrow("chunk_invalid"); + expect(await readFile(join(target, "history"), "utf8")).toBe("preserved"); + expect((await readdir(root)).filter(name => name.startsWith(".paperclip-checkpoint-"))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("rejects linked remote entries before replacing the local checkpoint", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-checkpoint-link-")); + try { + const source = join(root, "source"), target = join(root, "target"); await mkdir(source); await mkdir(target); + await writeFile(join(target, "history"), "preserved"); await symlink("/etc/passwd", join(source, "escape")); + await expect(downloadCheckpointDirectory({ runner: loopbackCheckpointRunner(), sourcePath: source, targetPath: target, mode: 0o700 })).rejects.toThrow("unsafe_entry"); + expect(await readFile(join(target, "history"), "utf8")).toBe("preserved"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/native-checkpoint-transfer.ts b/server/src/services/native-runtime/native-checkpoint-transfer.ts new file mode 100644 index 00000000000..e145e9fd86d --- /dev/null +++ b/server/src/services/native-runtime/native-checkpoint-transfer.ts @@ -0,0 +1,204 @@ +import { spawn, execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { chmod, lstat, mkdir, mkdtemp, open, opendir, rename, rm } from "node:fs/promises"; +import { dirname, join, posix } from "node:path"; +import { promisify } from "node:util"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { syncNativeHarnessDirectory } from "./native-harness-tree.js"; +import { runCheckpointCommand } from "./native-checkpoint-command.js"; + +const execute = promisify(execFile); +const CHUNK_BYTES = 192 * 1024; // Base64 remains below ordinary 1 MiB command output budgets. +const MAX_LINE_BYTES = 64 * 1024; +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +const shaCommand = (path: string) => `(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${quote(path)}; else shasum -a 256 ${quote(path)}; fi)`; + +async function remote(runner: CommandManagedRuntimeRunner, script: string, stdin?: string, timeoutMs = 30_000): Promise { + if (timeoutMs === 0 && stdin === undefined) return runCheckpointCommand(runner, `set -e; umask 077; ${script}`); + const result = await runner.execute({ command: "sh", args: ["-c", `set -e; umask 077; ${script}`], stdin, bypassSession: true, timeoutMs }); + if (result.exitCode !== 0 || result.timedOut) throw new Error("runner_remote_checkpoint_transfer_failed"); + return result.stdout; +} + +export function checkpointExclusions(entries: readonly string[]): string[] { + for (const entry of entries) { + if (!entry || entry.startsWith("/") || entry.split("/").some(part => !/^[A-Za-z0-9._-]+$/.test(part) || part === "." || part === "..")) { + throw new Error("runner_remote_checkpoint_exclusion_invalid"); + } + } + return entries.map(entry => `--exclude=./${entry}`); +} + +/** Only diagnostic output is buffered. Archive bytes always remain on disk. */ +export async function checkpointTar(args: string[]): Promise { + await execute("tar", args, { env: { ...process.env, COPYFILE_DISABLE: "1", LC_ALL: "C" }, maxBuffer: MAX_LINE_BYTES }); +} + +async function digest(path: string): Promise<{ bytes: bigint; sha256: string }> { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const hash = createHash("sha256"); let bytes = 0n; + try { + if (!(await file.stat()).isFile()) throw new Error("runner_remote_checkpoint_archive_invalid"); + for await (const chunk of file.createReadStream({ highWaterMark: CHUNK_BYTES, autoClose: false })) { + hash.update(chunk); bytes += BigInt(chunk.length); + } + } finally { await file.close(); } + return { bytes, sha256: hash.digest("hex") }; +} + +/** Strict archive validation before extraction, with bounded listing memory. + * Links/devices are excluded from checkpoints. Tar receives the same private, + * immutable spool for validation and extraction. No total bytes/entry limit. */ +export async function validateCheckpointArchive(path: string): Promise { + const child = spawn("tar", ["-tvzf", path], { env: { ...process.env, LC_ALL: "C" }, stdio: ["ignore", "pipe", "pipe"] }); + let stderrBytes = 0, pending = "", count = 0n; + const completed = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", code => code === 0 ? resolve() : reject(new Error("runner_remote_checkpoint_archive_invalid"))); + }); + // Attach immediately: validation can fail before the child exits. + void completed.catch(() => undefined); + child.stderr.on("data", (chunk: Buffer) => { stderrBytes += chunk.length; if (stderrBytes > MAX_LINE_BYTES) child.kill(); }); + const validate = (line: string) => { + if (!line) return; + // GNU and BSD tar listings. Fail closed on an unknown dialect or an entry + // whose path contains a raw line break, rather than trusting a partial name. + const parsed = line.match(/^(\S+)\s+\S+\/\S+\s+\d+\s+\S+\s+\S+\s+(.*)$/) + ?? line.match(/^(\S+)\s+\d+\s+\S+\s+\S+\s+\d+\s+\S+\s+\d{1,2}\s+(?:\d{4}|\d{1,2}:\d{2}(?::\d{2})?)\s+(.*)$/); + if (!parsed) throw new Error("runner_remote_checkpoint_archive_invalid"); + if (parsed[1]![0] !== "-" && parsed[1]![0] !== "d") throw new Error("runner_remote_checkpoint_archive_unsafe_entry"); + const name = parsed[2]!; + if (name.startsWith("/") || name.includes("\0") || name.split("/").includes("..")) throw new Error("runner_remote_checkpoint_archive_unsafe_path"); + count++; + }; + try { + child.stdout.setEncoding("utf8"); + for await (const chunk of child.stdout) { + pending += String(chunk); + let end: number; + while ((end = pending.indexOf("\n")) >= 0) { + if (end > MAX_LINE_BYTES) throw new Error("runner_remote_checkpoint_archive_invalid"); + validate(pending.slice(0, end)); pending = pending.slice(end + 1); + } + if (pending.length > MAX_LINE_BYTES) throw new Error("runner_remote_checkpoint_archive_invalid"); + } + if (pending) validate(pending); + await completed; + if (!count || stderrBytes > MAX_LINE_BYTES) throw new Error("runner_remote_checkpoint_archive_invalid"); + } catch (error) { child.kill(); await completed.catch(() => undefined); throw error; } +} + +async function verifyExtracted(root: string, depth = 0): Promise { + if (depth > 128 || !(await lstat(root)).isDirectory()) throw new Error("runner_remote_checkpoint_unsafe_entry"); + const files: string[] = []; + const flush = async () => { + // Keep fsync and descriptor concurrency bounded independently of history + // length. Await every operation before cleanup or syncing its directory. + const results = await Promise.allSettled(files.splice(0).map(async path => { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + if (!(await file.stat()).isFile()) throw new Error("runner_remote_checkpoint_unsafe_entry"); + await file.sync(); + } finally { await file.close(); } + })); + for (const result of results) if (result.status === "rejected") throw result.reason; + }; + for await (const entry of await opendir(root, { bufferSize: 128 })) { + const path = join(root, entry.name), stat = await lstat(path); + if (stat.isDirectory()) { + await flush(); + await verifyExtracted(path, depth + 1); + } + else if (stat.isFile()) { + files.push(path); + if (files.length === 8) await flush(); + } else throw new Error("runner_remote_checkpoint_unsafe_entry"); + } + await flush(); + const directory = await open(root, constants.O_RDONLY | constants.O_NOFOLLOW); + try { await directory.sync(); } finally { await directory.close(); } +} + +export async function extractCheckpointArchive(archive: string, target: string, mode: number): Promise { + await validateCheckpointArchive(archive); + await mkdir(target, { mode }); + await checkpointTar(["--no-same-owner", "--no-same-permissions", "-xzf", archive, "-C", target]); + await verifyExtracted(target); + await chmod(target, mode); +} + +/** Upload bounded chunks to a private remote spool, verify the whole digest, + * then extract. A slow large archive has no total transfer deadline. */ +export async function uploadCheckpointArchive(input: { runner: CommandManagedRuntimeRunner; archive: string; targetPath: string; mode: number }): Promise { + const expected = await digest(input.archive); + const scratch = posix.join(posix.dirname(input.targetPath), `.paperclip-checkpoint-${randomUUID()}`), archive = posix.join(scratch, "payload.tar.gz"); + let owned = false; + try { + await remote(input.runner, `mkdir -p ${quote(posix.dirname(input.targetPath))}; mkdir ${quote(scratch)}; : > ${quote(archive)}`); owned = true; + const file = await open(input.archive, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const buffer = Buffer.alloc(CHUNK_BYTES); let index = 0n, length = 0n; + for (;;) { + // Fill each block; file.read is allowed to return a short read. + let filled = 0; + while (filled < buffer.length) { const read = await file.read(buffer, filled, buffer.length - filled, null); if (!read.bytesRead) break; filled += read.bytesRead; } + if (!filled) break; + await remote(input.runner, `test ! -L ${quote(archive)}; test "$(wc -c < ${quote(archive)} | tr -d '[:space:]')" = ${quote(String(length))}; base64 -d | dd of=${quote(archive)} bs=${CHUNK_BYTES} seek=${index} conv=notrunc 2>/dev/null`, buffer.subarray(0, filled).toString("base64")); + length += BigInt(filled); index++; + } + } finally { await file.close(); } + const actual = await remote(input.runner, `wc -c < ${quote(archive)}; ${shaCommand(archive)}`, undefined, 0); + assertRemoteDigest(actual, expected); + await remote(input.runner, `mkdir -p ${quote(input.targetPath)}; tar -xzf ${quote(archive)} -C ${quote(input.targetPath)}; chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`, undefined, 0); + } finally { if (owned) await remote(input.runner, `rm -rf -- ${quote(scratch)}`).catch(() => undefined); } +} + +function assertRemoteDigest(value: string, expected: { bytes: bigint; sha256: string }): void { + const lines = value.trim().split("\n"); + if (lines.length !== 2 || !/^\s*\d+\s*$/.test(lines[0]!) || BigInt(lines[0]!.trim()) !== expected.bytes || lines[1]!.split(/\s+/)[0] !== expected.sha256) { + throw new Error("runner_remote_checkpoint_digest_mismatch"); + } +} + +/** Remote tar is a fixed disk snapshot. Transfer/read-back are independently + * verified before the prior local checkpoint can be replaced. */ +export async function downloadCheckpointDirectory(input: { runner: CommandManagedRuntimeRunner; sourcePath: string; targetPath: string; mode: number; excludeEntries?: readonly string[] }): Promise { + const excludes = checkpointExclusions(input.excludeEntries ?? []).map(quote).join(" "); + const scratch = posix.join(posix.dirname(input.sourcePath), `.paperclip-checkpoint-${randomUUID()}`), archive = posix.join(scratch, "payload.tar.gz"); + await mkdir(dirname(input.targetPath), { recursive: true, mode: 0o700 }); + const local = await mkdtemp(join(dirname(input.targetPath), ".paperclip-checkpoint-")); + const localArchive = join(local, "payload.tar.gz"), staged = join(local, "payload"), previous = join(local, "previous"); + let owned = false, moved = false, installed = false; + try { + await remote(input.runner, `mkdir ${quote(scratch)}`); owned = true; + const manifest = await remote(input.runner, `tar ${excludes} -czf ${quote(archive)} -C ${quote(input.sourcePath)} .; wc -c < ${quote(archive)}; ${shaCommand(archive)}`, undefined, 0); + const lines = manifest.trim().split("\n"), size = lines[0]?.trim(), sha256 = lines[1]?.split(/\s+/)[0]; + if (lines.length !== 2 || !size || !/^\d+$/.test(size) || !sha256 || !/^[a-f0-9]{64}$/.test(sha256)) throw new Error("runner_remote_checkpoint_archive_invalid"); + const total = BigInt(size), hash = createHash("sha256"), file = await open(localArchive, "wx", 0o600); + try { + for (let offset = 0n, index = 0n; offset < total; index++) { + const encoded = await remote(input.runner, `test ! -L ${quote(archive)}; dd if=${quote(archive)} bs=${CHUNK_BYTES} skip=${index} count=1 2>/dev/null | base64`); + if (encoded.length > CHUNK_BYTES * 2) throw new Error("runner_remote_checkpoint_chunk_invalid"); + const compact = encoded.replace(/\s/g, ""), chunk = Buffer.from(compact, "base64"); + const expected = Number(total - offset < BigInt(CHUNK_BYTES) ? total - offset : BigInt(CHUNK_BYTES)); + if (chunk.length !== expected || chunk.toString("base64") !== compact) throw new Error("runner_remote_checkpoint_chunk_invalid"); + hash.update(chunk); await file.writeFile(chunk); offset += BigInt(chunk.length); + } + if (hash.digest("hex") !== sha256) throw new Error("runner_remote_checkpoint_digest_mismatch"); + await file.sync(); + } finally { await file.close(); } + await extractCheckpointArchive(localArchive, staged, input.mode); + try { await rename(input.targetPath, previous); moved = true; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + await rename(staged, input.targetPath); + await syncNativeHarnessDirectory(dirname(input.targetPath)); + installed = true; + } catch (error) { + if (moved && !installed) { try { await rename(previous, input.targetPath); moved = false; } catch { /* Preserve the previous checkpoint in its private staging root. */ } } + throw error; + } finally { + if (owned) await remote(input.runner, `rm -rf -- ${quote(scratch)}`).catch(() => undefined); + if (!moved || installed) await rm(local, { recursive: true, force: true }); + } +} diff --git a/server/src/services/native-runtime/native-cleanup-tree.test.ts b/server/src/services/native-runtime/native-cleanup-tree.test.ts new file mode 100644 index 00000000000..60800ebbfb3 --- /dev/null +++ b/server/src/services/native-runtime/native-cleanup-tree.test.ts @@ -0,0 +1,52 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdir, mkdtemp, open, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { canonicalNativeJson } from "./canonical.js"; +import { snapshotCleanupProviderHome, copyCleanupProviderHomeTree, findCleanupRollout, readCleanupRolloutHeader } from "./native-cleanup-tree.js"; + +it("preserves the retained metadata/content fingerprints while copying in bounded chunks", async () => { + const root = await mkdtemp(join(tmpdir(), "cleanup-tree-")), source = join(root, "source"), destination = join(root, "copy"); + try { + await mkdir(source); await mkdir(join(source, "sessions")); await mkdir(join(source, "empty")); + await writeFile(join(source, "sessions", "b"), "retained"); await writeFile(join(source, "Z"), "before lowercase"); await writeFile(join(source, "auth.json"), "exclude"); + const metadata: unknown[] = [], content: unknown[] = []; + const oracle = async (relative: string) => { + const path = join(source, relative), stat = await lstat(path), directory = stat.isDirectory(), size = directory ? 0 : stat.size; + metadata.push({ path: relative, directory, dev: stat.dev, ino: stat.ino, size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs }); + content.push({ path: relative, directory, size, ...(!directory ? { sha256: createHash("sha256").update(await readFile(path)).digest("hex") } : {}) }); + if (directory) for (const name of (await readdir(path)).sort()) { if (!relative && name === "auth.json") continue; await oracle(relative ? `${relative}/${name}` : name); } + }; + await oracle(""); + const snapshot = await snapshotCleanupProviderHome(source, true, ["auth.json"]); + const sha = (value: unknown) => createHash("sha256").update(canonicalNativeJson(value)).digest("hex"); + expect(snapshot.metadataFingerprint).toBe(sha(metadata)); expect(snapshot.fingerprint).toBe(sha(content)); + await copyCleanupProviderHomeTree(source, destination, snapshot, ["auth.json"]); + expect((await snapshotCleanupProviderHome(destination, true, ["auth.json"])).fingerprint).toBe(snapshot.fingerprint); + await expect(readFile(join(destination, "auth.json"))).rejects.toThrow(); + await writeFile(join(source, "Z"), "changed"); + await expect(copyCleanupProviderHomeTree(source, join(root, "changed-copy"), snapshot, ["auth.json"])).rejects.toThrow("unproven"); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("reads only the bounded header of a multi-gigabyte rollout and rejects duplicate identity", async () => { + const root = await mkdtemp(join(tmpdir(), "cleanup-rollout-")); + try { + await mkdir(join(root, "sessions")); + const path = join(root, "sessions", "rollout-thread.jsonl"), file = await open(path, "wx", 0o600); + const header = JSON.stringify({ type: "session_meta", payload: { id: "thread" } }); + try { await file.writeFile(header + "\n"); await file.truncate(3 * 1024 ** 3); } finally { await file.close(); } + const rollout = await findCleanupRollout(root, "thread", []); + expect(rollout.size).toBe(3 * 1024 ** 3); + expect((await readCleanupRolloutHeader(root, rollout)).toString()).toBe(header); + await writeFile(join(root, "sessions", "another-thread.jsonl"), header + "\n"); + await expect(findCleanupRollout(root, "thread", [])).rejects.toThrow("unproven"); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("refuses symlinks in the source inventory", async () => { + const root = await mkdtemp(join(tmpdir(), "cleanup-tree-link-")); + try { await symlink("/etc/passwd", join(root, "escape")); await expect(snapshotCleanupProviderHome(root, true, [])).rejects.toThrow("unproven"); } + finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/native-cleanup-tree.ts b/server/src/services/native-runtime/native-cleanup-tree.ts new file mode 100644 index 00000000000..d6d39da672a --- /dev/null +++ b/server/src/services/native-runtime/native-cleanup-tree.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open } from "node:fs/promises"; +import { resolve } from "node:path"; +import { canonicalNativeJson } from "./canonical.js"; +import { sortedNativeHarnessNames, syncNativeHarnessDirectory } from "./native-harness-tree.js"; + +interface Entry { + path: string; directory: boolean; dev: number; ino: number; size: number; + mtimeMs: number; ctimeMs: number; sha256?: string; +} +const denied = () => new Error("native_cleanup_maintenance_unproven"); +const order = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0; +function unchanged(a: Awaited>, b: Awaited>): boolean { + return a.dev === b.dev && a.ino === b.ino && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs && a.size === b.size && a.mode === b.mode; +} + +/** Exact retained-format scan. Only one bounded file buffer and a bounded + * external-sort page are resident; neither entries nor file bodies accumulate. */ +async function* entries(home: string, content: boolean, exclude: readonly string[], relative = "", depth = 0): AsyncGenerator { + if (depth > 128) throw denied(); + const path = resolve(home, relative), before = await lstat(path); + if (before.isSymbolicLink() || (!before.isFile() && !before.isDirectory())) throw denied(); + const entry: Entry = { path: relative, directory: before.isDirectory(), dev: before.dev, ino: before.ino, + size: before.isDirectory() ? 0 : before.size, mtimeMs: before.mtimeMs, ctimeMs: before.ctimeMs }; + if (!entry.directory && content) { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + if (!unchanged(before, await file.stat())) throw denied(); + const hash = createHash("sha256"); let bytes = 0; + for await (const chunk of file.createReadStream({ highWaterMark: 64 * 1024, autoClose: false })) { bytes += chunk.length; hash.update(chunk); } + if (bytes !== before.size || !unchanged(before, await file.stat())) throw denied(); + entry.sha256 = hash.digest("hex"); + } finally { await file.close(); } + } + yield entry; + if (entry.directory) for await (const name of sortedNativeHarnessNames(path, order)) { + if (!relative && exclude.includes(name)) continue; + yield* entries(home, content, exclude, relative ? `${relative}/${name}` : name, depth + 1); + } + if (!unchanged(before, await lstat(path))) throw denied(); +} + +export async function snapshotCleanupProviderHome(home: string, content: boolean, exclude: readonly string[]) { + const metadataHash = createHash("sha256").update("["), contentHash = createHash("sha256").update("["); + let bytes = 0, first = true; + for await (const entry of entries(home, content, exclude)) { + const { sha256: _sha, ...metadata } = entry; + if (!first) { metadataHash.update(","); contentHash.update(","); } first = false; + metadataHash.update(canonicalNativeJson(metadata)); + contentHash.update(canonicalNativeJson({ path: entry.path, directory: entry.directory, size: entry.size, ...(entry.sha256 ? { sha256: entry.sha256 } : {}) })); + bytes += entry.size; + } + return { bytes, metadataFingerprint: metadataHash.update("]").digest("hex"), + fingerprint: content ? contentHash.update("]").digest("hex") : null, + entries: () => entries(home, content, exclude) }; +} + +export async function copyCleanupProviderHomeTree(source: string, destination: string, snapshot: Awaited>, exclude: readonly string[]): Promise { + for await (const entry of snapshot.entries()) { + const target = resolve(destination, entry.path); + if (entry.directory) { await mkdir(target, { mode: 0o700 }); continue; } + const file = await open(resolve(source, entry.path), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let output: Awaited> | undefined; + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.dev !== entry.dev || stat.ino !== entry.ino || stat.size !== entry.size || stat.mtimeMs !== entry.mtimeMs || stat.ctimeMs !== entry.ctimeMs) throw denied(); + output = await open(target, "wx", 0o600); const hash = createHash("sha256"); + for await (const chunk of file.createReadStream({ highWaterMark: 64 * 1024, autoClose: false })) { hash.update(chunk); await output.writeFile(chunk); } + if (hash.digest("hex") !== entry.sha256 || !unchanged(stat, await file.stat())) throw denied(); + await output.sync(); + } finally { try { await output?.close(); } finally { await file.close(); } } + } + if ((await snapshotCleanupProviderHome(source, false, exclude)).metadataFingerprint !== snapshot.metadataFingerprint || + (await snapshotCleanupProviderHome(destination, true, exclude)).fingerprint !== snapshot.fingerprint) throw denied(); + // Sync directories after their child entries have been installed. + const sync = async (path: string): Promise => { + for await (const name of sortedNativeHarnessNames(path, order)) { const child = resolve(path, name); if ((await lstat(child)).isDirectory()) await sync(child); } + await syncNativeHarnessDirectory(path); + }; + await sync(destination); +} + +/** Find one exact rollout without retaining a catalog of historical files. */ +export async function findCleanupRollout(home: string, threadId: string, exclude: readonly string[]): Promise { + let rollout: Entry | undefined; + for await (const entry of entries(home, false, exclude)) { + if (/^state_\d+\.sqlite$/.test(entry.path) && entry.path !== "state_5.sqlite") throw denied(); + if (!entry.directory && entry.path.startsWith("sessions/") && entry.path.endsWith(`-${threadId}.jsonl`)) { + if (rollout) throw denied(); rollout = entry; + } + } + if (!rollout) throw denied(); return rollout; +} + +export async function readCleanupRolloutHeader(home: string, entry: Entry): Promise { + const file = await open(resolve(home, entry.path), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.ino !== entry.ino || stat.dev !== entry.dev || stat.size !== entry.size || stat.mtimeMs !== entry.mtimeMs || stat.ctimeMs !== entry.ctimeMs) throw denied(); + const buffer = Buffer.alloc(64 * 1024 + 1); let length = 0; + while (length < buffer.length) { + const read = await file.read(buffer, length, buffer.length - length, null); if (!read.bytesRead) break; + length += read.bytesRead; const newline = buffer.subarray(0, length).indexOf(10); + if (newline >= 0) { if (!unchanged(stat, await file.stat())) throw denied(); return buffer.subarray(0, newline); } + } + throw denied(); + } finally { await file.close(); } +} diff --git a/server/src/services/native-runtime/native-codex-runner.integration.test.ts b/server/src/services/native-runtime/native-codex-runner.integration.test.ts index 13d8c73b71c..25b20726f9c 100644 --- a/server/src/services/native-runtime/native-codex-runner.integration.test.ts +++ b/server/src/services/native-runtime/native-codex-runner.integration.test.ts @@ -1,3 +1,4 @@ +import { resolveRunnerCargoTestBinary } from "./cargo-test-binary.js"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdtemp, readFile, rm } from "node:fs/promises"; @@ -43,19 +44,8 @@ const runnerWorkspace = resolve( import.meta.dirname, "../../../../packages/paperclip-runner/runner", ); -const executableSuffix = process.platform === "win32" ? ".exe" : ""; -const runnerBinary = resolve( - runnerWorkspace, - "target", - "release", - `paperclip-runnerd${executableSuffix}`, -); -const fakeCodexBinary = resolve( - runnerWorkspace, - "target", - "release", - `fake-codex-app-server${executableSuffix}`, -); +const runnerBinary = resolveRunnerCargoTestBinary(runnerWorkspace, "release", "paperclip-runnerd"); +const fakeCodexBinary = resolveRunnerCargoTestBinary(runnerWorkspace, "release", "fake-codex-app-server"); function ensureRunnerTestBinaries(): void { // Cargo's freshness check is necessary even when the files exist: an older diff --git a/server/src/services/native-runtime/native-codex-runner.ts b/server/src/services/native-runtime/native-codex-runner.ts index 64d3044b4f0..90b4f1ce557 100644 --- a/server/src/services/native-runtime/native-codex-runner.ts +++ b/server/src/services/native-runtime/native-codex-runner.ts @@ -347,22 +347,22 @@ export async function executeNativeCodexRunner(input: { runnerDigest, }); - prepared.queueCommand("run.prepare", buildNativeRunnerPreparePayload({ + (await prepared.queueCommand("run.prepare", buildNativeRunnerPreparePayload({ cwd: input.cwd, model: input.model, resumeProviderSessionId: input.resumeProviderSessionId, completionContract: input.completionContract, semanticTools: prepared.semanticTools, ...(input.providerLaunch ? { providerLaunch: input.providerLaunch } : {}), - }), `prepare_${input.runId}`); - prepared.queueCommand("session.open", {}, `open_${input.runId}`); + }), `prepare_${input.runId}`)); + (await prepared.queueCommand("session.open", {}, `open_${input.runId}`)); const recoveryGoalRequestId = input.resumeSessionGoalHeartbeat ? `recovery_${input.runId}` : null; if (goalControl) { const commandKey = createHash("sha256").update(goalControl.requestId).digest("hex").slice(0, 20); - nativeGoalCommands(goalControl).forEach((command, index) => { - prepared.queueCommand(command.type, command.payload, `goal_${commandKey}_${index + 1}`); + nativeGoalCommands(goalControl).forEach(async (command, index) => { + (await prepared.queueCommand(command.type, command.payload, `goal_${commandKey}_${index + 1}`)); }); await input.db.update(agentSessionGoalActions).set({ status: "delivering", @@ -372,13 +372,13 @@ export async function executeNativeCodexRunner(input: { eq(agentSessionGoalActions.requestId, goalControl.requestId), )); } else if (recoveryGoalRequestId) { - prepared.queueCommand( + (await prepared.queueCommand( "session.goal.set", { requestId: recoveryGoalRequestId, status: "active" }, `goal_${recoveryGoalRequestId}`, - ); + )); } else { - prepared.queueCommand("turn.start", { text: input.prompt }, `turn_${input.runId}`); + (await prepared.queueCommand("turn.start", { text: input.prompt }, `turn_${input.runId}`)); } const child = spawn(binary, buildNativeRunnerArguments({ @@ -449,8 +449,8 @@ export async function executeNativeCodexRunner(input: { ); }), ]); - prepared.queueCommand("session.close", {}, `close_${input.runId}`); - prepared.queueCommand("runner.shutdown", {}, `shutdown_${input.runId}`); + (await prepared.queueCommand("session.close", {}, `close_${input.runId}`)); + (await prepared.queueCommand("runner.shutdown", {}, `shutdown_${input.runId}`)); await stopChild(child, exit, true); const succeeded = completed === null || completed.terminal.runTerminalState === "succeeded"; diff --git a/server/src/services/native-runtime/native-deliverable-feedback.ts b/server/src/services/native-runtime/native-deliverable-feedback.ts index 48aae1ace33..d796ea93c21 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.ts @@ -1,6 +1,7 @@ import { and, eq } from "drizzle-orm"; import { assets, issueAttachments, issueWorkProducts, type Db } from "@paperclipai/db"; import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js"; +import { findNativeToolReceiptReference, type NativeToolReceiptBinding } from "./native-tool-receipts.js"; function evidenceRefs(value: unknown): string[] { if (!Array.isArray(value)) return []; @@ -16,10 +17,13 @@ function record(value: unknown): Record { ? value as Record : {}; } -async function hasCurrentPublicationReceipt(db: Db, companyId: string, receipts: unknown, attachment: { +async function hasCurrentPublicationReceipt(db: Db, binding: NativeToolReceiptBinding, receipts: unknown, attachment: { id: string; filename: string | null; byteSize: number; sha256: string; }): Promise { - for (const value of Object.values(record(receipts))) { + const indexed = await findNativeToolReceiptReference(db, binding, "publication", attachment.id); + // Retained old rows remain readable while migration is admitted separately. + // Fresh writes use one indexed lookup, independent of earlier tool count. + for (const value of indexed ? [indexed] : Object.values(record(receipts))) { const receipt = record(value); const input = record(receipt.input); const result = record(receipt.result); @@ -49,8 +53,8 @@ async function hasCurrentPublicationReceipt(db: Db, companyId: string, receipts: if (typeof source.attachmentId !== "string" || !uuid.test(source.attachmentId) || typeof source.commentId !== "string" || !uuid.test(source.commentId)) continue; const [original] = await db.select({ filename: assets.originalFilename, byteSize: assets.byteSize, sha256: assets.sha256 }) - .from(issueAttachments).innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, companyId))) - .where(and(eq(issueAttachments.id, source.attachmentId), eq(issueAttachments.companyId, companyId), + .from(issueAttachments).innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, binding.companyId))) + .where(and(eq(issueAttachments.id, source.attachmentId), eq(issueAttachments.companyId, binding.companyId), eq(issueAttachments.issueCommentId, source.commentId))).limit(1); if (original?.filename === attachment.filename && original.byteSize === attachment.byteSize && original.sha256.toLowerCase() === attachment.sha256.toLowerCase()) return true; @@ -123,7 +127,7 @@ export async function validateNativeDeliverableEvidence( // controller restart of this run; a replacement can re-register preserved // workspace bytes internally rather than asking the user to confirm them. if (fileRequested && attachment.originatingRunId !== binding.runId) continue; - if (fileRequested && !await hasCurrentPublicationReceipt(db, binding.companyId, binding.semanticToolReceipts, attachment)) { + if (fileRequested && !await hasCurrentPublicationReceipt(db, binding, binding.semanticToolReceipts, attachment)) { throw new Error("This attachment has no matching verified publication receipt for this run's requested output. Inspect any preserved file and use register_deliverable to verify its current filename, size, and SHA-256, then cite the new receipt. No human completion approval was created."); } registeredAttachment = true; diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts index 1c969758ac7..059b73d39c4 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { expect, it } from "vitest"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; -it("removes retired backups containing immutable skills without modifying live files or symlink targets", () => { +it("removes retired backups containing immutable skills without modifying live files or symlink targets", async () => { const base = mkdtempSync(join(tmpdir(), "paperclip-backup-cleanup-")); const retired = join(base, "previous"); const nested = join(retired, "claude", "skills", "first-task"); @@ -19,12 +19,12 @@ it("removes retired backups containing immutable skills without modifying live f chmodSync(retired, 0o555); chmodSync(live, 0o555); try { - removeNativeHarnessBackup(retired); + await removeNativeHarnessBackup(retired); expect(existsSync(retired)).toBe(false); expect(readFileSync(join(live, "SKILL.md"), "utf8")).toBe("live instructions"); expect(lstatSync(live).mode & 0o777).toBe(0o555); expect(lstatSync(join(live, "SKILL.md")).mode & 0o777).toBe(0o444); - expect(() => removeNativeHarnessBackup(retired)).not.toThrow(); + await expect(removeNativeHarnessBackup(retired)).resolves.toBeUndefined(); } finally { for (const directory of [retired, join(retired, "claude", "skills"), nested, live]) { if (existsSync(directory)) chmodSync(directory, 0o700); @@ -32,3 +32,18 @@ it("removes retired backups containing immutable skills without modifying live f rmSync(base, { recursive: true, force: true }); } }); + +it("removes a wide history asynchronously without collecting all directory entries", async () => { + const root = mkdtempSync(join(tmpdir(), "paperclip-backup-cleanup-wide-")); + for (let index = 0; index < 4_097; index++) writeFileSync(join(root, `receipt-${index}`), "old"); + let ticks = 0; + const timer = setInterval(() => ticks++, 1); + try { + await removeNativeHarnessBackup(root); + expect(existsSync(root)).toBe(false); + expect(ticks).toBeGreaterThan(0); + } finally { + clearInterval(timer); + rmSync(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.ts index d52e50ba5c4..c72492f9ff7 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.ts @@ -1,18 +1,26 @@ -import { chmodSync, lstatSync, readdirSync, rmSync } from "node:fs"; +import { chmod, lstat, rmdir, unlink } from "node:fs/promises"; import { join } from "node:path"; +import { sortedNativeHarnessNames } from "./native-harness-tree.js"; /** Remove a retired controller-owned backup, never a live harness directory. */ -export function removeNativeHarnessBackup(root: string): void { +export async function removeNativeHarnessBackup(root: string): Promise { // Archives preserve immutable skill directory modes. Deleting their children // requires write permission on those directories, even with rm's force flag. // Only change directories in the backup being discarded. lstat deliberately // avoids following provider-created symlinks into live or unrelated trees. - const prepareDirectory = (path: string): void => { - const metadata = lstatSync(path, { throwIfNoEntry: false }); - if (!metadata?.isDirectory()) return; - chmodSync(path, (metadata.mode & 0o777) | 0o700); - for (const name of readdirSync(path)) prepareDirectory(join(path, name)); + const remove = async (path: string, depth: number): Promise => { + // Published harness trees use the same path-depth bound. File count and + // historical bytes are unbounded; directory names spill to private disk. + if (depth > 128) throw new Error("runner_harness_backup_path_too_deep"); + const metadata = await lstat(path).catch(error => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + }); + if (!metadata) return; + if (!metadata.isDirectory()) { await unlink(path); return; } + await chmod(path, (metadata.mode & 0o777) | 0o700); + for await (const name of sortedNativeHarnessNames(path)) await remove(join(path, name), depth + 1); + await rmdir(path); }; - prepareDirectory(root); - rmSync(root, { recursive: true, force: true }); + await remove(root, 0); } diff --git a/server/src/services/native-runtime/native-harness-backup-stamp.ts b/server/src/services/native-runtime/native-harness-backup-stamp.ts index 202bf8a9cfe..a95e0062b57 100644 --- a/server/src/services/native-runtime/native-harness-backup-stamp.ts +++ b/server/src/services/native-runtime/native-harness-backup-stamp.ts @@ -1,5 +1,6 @@ +import { digestNativeHarnessBackupDirectory, readNativeHarnessBackupManifestBytes } from "./native-harness-tree.js"; import { createHash } from "node:crypto"; -import { lstatSync, readFileSync, readdirSync, readlinkSync } from "node:fs"; +import { lstatSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; @@ -51,40 +52,6 @@ function isRealFile(path: string): boolean { } } -function digestDirectory(directory: string): { sha256: string; bytes: number } { - const hash = createHash("sha256"); - let bytes = 0; - const visit = (current: string, relative: string) => { - const entries = readdirSync(current, { withFileTypes: true }).sort( - (left, right) => left.name.localeCompare(right.name), - ); - if (entries.length === 0) hash.update(`directory:${relative}\0`); - for (const entry of entries) { - const path = resolve(current, entry.name); - const relativePath = relative ? `${relative}/${entry.name}` : entry.name; - const stats = lstatSync(path); - if (entry.isDirectory()) { - hash.update(`directory:${relativePath}:${stats.mode & 0o777}\0`); - visit(path, relativePath); - } else if (entry.isSymbolicLink()) { - hash.update(`symlink:${relativePath}:${readlinkSync(path)}\0`); - } else if (entry.isFile()) { - const contents = readFileSync(path); - bytes += contents.byteLength; - hash.update( - `file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`, - ); - hash.update(contents); - } else { - throw new Error( - `runner_harness_backup_unsupported_entry:${relativePath}`, - ); - } - } - }; - visit(directory, ""); - return { sha256: `sha256:${hash.digest("hex")}`, bytes }; -} export function createNativeHarnessBackupStamp(input: { manifestPath: string; @@ -115,7 +82,7 @@ export function createNativeHarnessBackupStamp(input: { ) { throw new Error("runner_harness_backup_scope_invalid"); } - const manifestBytes = readFileSync(input.manifestPath); + const manifestBytes = readNativeHarnessBackupManifestBytes(input.manifestPath); const manifest = JSON.parse(manifestBytes.toString("utf8")) as Record< string, unknown @@ -142,10 +109,10 @@ export function createNativeHarnessBackupStamp(input: { }; } -export function verifyNativeHarnessBackupStamp( +export async function verifyNativeHarnessBackupStamp( value: unknown, expectedProviderLeaseId: string, -): boolean { +): Promise { if (!expectedProviderLeaseId) return false; if (!value || typeof value !== "object" || Array.isArray(value)) return false; const stamp = value as Record; @@ -173,7 +140,7 @@ export function verifyNativeHarnessBackupStamp( const manifestPath = resolve(candidate, "manifest.json"); if (!isRealDirectory(candidate) || !isRealFile(manifestPath)) continue; try { - const bytes = readFileSync(manifestPath); + const bytes = readNativeHarnessBackupManifestBytes(manifestPath); const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; if (manifestSha256 !== stamp.manifestSha256) continue; const manifest = JSON.parse(bytes.toString("utf8")) as Record< @@ -186,10 +153,11 @@ export function verifyNativeHarnessBackupStamp( manifest.runnerInstanceId !== stamp.runnerInstanceId || manifest.sourceProviderLeaseId !== stamp.sourceProviderLeaseId || !Array.isArray(manifest.directories) || - manifest.directories.length === 0 + manifest.directories.length === 0 || manifest.directories.length > 4 ) continue; let valid = true; + const names = new Set(); for (const entry of manifest.directories) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { valid = false; @@ -198,19 +166,20 @@ export function verifyNativeHarnessBackupStamp( const declared = entry as Record; if ( typeof declared.name !== "string" || - !/^[A-Za-z0-9._-]+$/.test(declared.name) || + !["runner", "codex-home", "opencode", "acpx"].includes(declared.name) || names.has(declared.name) || typeof declared.sha256 !== "string" || typeof declared.bytes !== "number" ) { valid = false; break; } + names.add(declared.name); const directory = resolve(candidate, declared.name); if (!isRealDirectory(directory)) { valid = false; break; } - const actual = digestDirectory(directory); + const actual = await digestNativeHarnessBackupDirectory(directory); if ( actual.sha256 !== declared.sha256 || actual.bytes !== declared.bytes @@ -219,7 +188,9 @@ export function verifyNativeHarnessBackupStamp( break; } } - if (valid) return true; + // Async streaming yields to other owners; replacement cannot rely on a + // manifest that changed while its historical files were being verified. + if (valid && bytes.equals(readNativeHarnessBackupManifestBytes(manifestPath))) return true; } catch { // Try the previous atomically-published backup. } diff --git a/server/src/services/native-runtime/native-harness-tree.test.ts b/server/src/services/native-runtime/native-harness-tree.test.ts new file mode 100644 index 00000000000..7dcd022e37c --- /dev/null +++ b/server/src/services/native-runtime/native-harness-tree.test.ts @@ -0,0 +1,66 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, open, readFile, readdir, lstat, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { digestNativeHarnessBackupDirectory } from "./native-harness-tree.js"; + +// Independent retained-format oracle, deliberately only used on small files. +async function retainedDigest(root: string) { + const hash = createHash("sha256"); let bytes = 0; + async function visit(directory: string, relative: string) { + const entries = (await readdir(directory)).sort((a, b) => a.localeCompare(b)); + if (!entries.length) hash.update(`directory:${relative}\0`); + for (const name of entries) { + const path = join(directory, name), entry = relative ? `${relative}/${name}` : name, metadata = await lstat(path); + if (metadata.isDirectory()) { hash.update(`directory:${entry}:${metadata.mode & 0o777}\0`); await visit(path, entry); } + else if (metadata.isSymbolicLink()) hash.update(`symlink:${entry}:${await readlink(path)}\0`); + else { const body = await readFile(path); bytes += body.length; hash.update(`file:${entry}:${metadata.mode & 0o777}:${body.length}\0`).update(body); } + } + } + await visit(root, ""); return { sha256: `sha256:${hash.digest("hex")}`, bytes }; +} + +it("preserves retained backup digests across multiple external-sort merge levels", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-tree-test-")); + try { + await mkdir(join(root, "empty")); + await mkdir(join(root, "nested")); + await writeFile(join(root, "nested", "line\nquote\".json"), "content"); + await symlink("absent-target", join(root, "link")); + // More than 32 full 128-name pages. No in-memory array may contain the + // whole directory while computing the production digest. + for (let batch = 0; batch < 33; batch++) await Promise.all(Array.from({ length: 128 }, (_, index) => + writeFile(join(root, `receipt-${(batch * 128 + index).toString().padStart(6, "0")}`), "x"))); + const expected = await retainedDigest(root); + expect(await digestNativeHarnessBackupDirectory(root)).toEqual(expected); + await writeFile(join(root, "receipt-000000"), "y"); + expect((await digestNativeHarnessBackupDirectory(root)).sha256).not.toBe(expected.sha256); + } finally { await rm(root, { recursive: true, force: true }); } +}, 60_000); + +it("streams a large file while the event loop remains available", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-tree-large-")); + const path = join(root, "history"), file = await open(path, "wx", 0o600); + const chunk = Buffer.alloc(64 * 1024, "a"), repeats = 2048; + try { + for (let index = 0; index < repeats; index++) await file.writeFile(chunk); + await file.close(); + const hash = createHash("sha256").update(`file:history:384:${chunk.length * repeats}\0`); + for (let index = 0; index < repeats; index++) hash.update(chunk); + let ticks = 0; + const timer = setInterval(() => ticks++, 1); + try { + expect(await digestNativeHarnessBackupDirectory(root)).toEqual({ sha256: `sha256:${hash.digest("hex")}`, bytes: chunk.length * repeats }); + expect(ticks).toBeGreaterThan(1); + } finally { clearInterval(timer); } + } finally { await file.close(); await rm(root, { recursive: true, force: true }); } +}, 30_000); + +it("refuses a root symlink instead of following it into unrelated history", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-tree-link-")); + try { + await mkdir(join(root, "real")); await symlink(join(root, "real"), join(root, "link")); + await expect(digestNativeHarnessBackupDirectory(join(root, "link"))).rejects.toThrow("unsafe_directory"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/native-harness-tree.ts b/server/src/services/native-runtime/native-harness-tree.ts new file mode 100644 index 00000000000..b87c029770f --- /dev/null +++ b/server/src/services/native-runtime/native-harness-tree.ts @@ -0,0 +1,170 @@ +import { createHash } from "node:crypto"; +import { constants, closeSync, fstatSync, openSync, readSync } from "node:fs"; +import { lstat, mkdtemp, open, opendir, readlink, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const SORT_PAGE = 128; +const MERGE_FAN_IN = 32; +const CHUNK_BYTES = 64 * 1024; + +/** A manifest lists the fixed provider directories, never their history. */ +export function readNativeHarnessBackupManifestBytes(path: string): Buffer { + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = fstatSync(fd); + if (!before.isFile() || before.size > 64 * 1024) throw new Error("runner_harness_backup_manifest_invalid"); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error("runner_harness_backup_changed"); + offset += count; + } + if (!sameMetadata(before, fstatSync(fd))) throw new Error("runner_harness_backup_changed"); + return bytes; + } finally { closeSync(fd); } +} + +async function* names(path: string): AsyncGenerator { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + let pending = ""; + for await (const chunk of file.createReadStream({ encoding: "utf8", highWaterMark: CHUNK_BYTES, autoClose: false })) { + pending += chunk; + let end: number; + while ((end = pending.indexOf("\n")) >= 0) { + yield JSON.parse(pending.slice(0, end)) as string; + pending = pending.slice(end + 1); + } + if (pending.length > 16 * 1024) throw new Error("runner_harness_backup_name_invalid"); + } + if (pending) throw new Error("runner_harness_backup_sort_incomplete"); + } finally { await file.close(); } +} + +async function merge(paths: string[], destination: string, compare: (a: string, b: string) => number): Promise { + const streams = paths.map(path => names(path)); + let output: Awaited> | undefined; + try { + // Keep partially opened readers under the finally guard as well. + const heads = []; + for (const stream of streams) heads.push(await stream.next()); + output = await open(destination, "wx", 0o600); + for (;;) { + let selected = -1; + for (let index = 0; index < heads.length; index++) { + const head = heads[index]!; + if (!head.done && (selected === -1 || compare(head.value, heads[selected]!.value!) < 0)) selected = index; + } + if (selected === -1) break; + await output.writeFile(`${JSON.stringify(heads[selected]!.value)}\n`); + heads[selected] = await streams[selected]!.next(); + } + } finally { + // A failed output close must still release every input descriptor. + const closed = await Promise.allSettled([ + output?.close(), + ...streams.map(stream => stream.return(undefined)), + ]); + const failure = closed.find(result => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + } +} + +/** Preserve the v1 manifest's locale ordering without materializing an entire + * directory. Runs and merge levels live in private scratch files; RAM depends + * on page/fan-in limits, never the number of retained files. */ +export async function* sortedNativeHarnessNames(directory: string, compare = (a: string, b: string) => a.localeCompare(b)): AsyncGenerator { + let scratch: string | undefined, count = 0n; + let page: string[] = []; + const flush = async () => { + scratch ??= await mkdtemp(join(tmpdir(), "paperclip-backup-sort-")); + const file = await open(join(scratch, `0-${count++}`), "wx", 0o600); + try { await file.writeFile(page.sort(compare).map(name => JSON.stringify(name) + "\n").join("")); } + finally { await file.close(); } + page = []; + }; + try { + for await (const entry of await opendir(directory, { bufferSize: SORT_PAGE })) { + page.push(entry.name); + if (page.length === SORT_PAGE) await flush(); + } + if (!scratch) { yield* page.sort(compare); return; } + if (page.length) await flush(); + let level = 0; + while (count > 1n) { + let next = 0n; + for (let first = 0n; first < count; first += BigInt(MERGE_FAN_IN)) { + const inputs: string[] = []; + for (let index = first; index < count && inputs.length < MERGE_FAN_IN; index++) inputs.push(join(scratch, `${level}-${index}`)); + await merge(inputs, join(scratch, `${level + 1}-${next++}`), compare); + await Promise.all(inputs.map(path => rm(path))); + } + count = next; + level++; + } + yield* names(join(scratch, `${level}-0`)); + } finally { if (scratch) await rm(scratch, { recursive: true, force: true }); } +} + +function sameMetadata(a: Awaited>, b: Awaited>): boolean { + return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mode === b.mode && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs; +} + +/** Streaming, asynchronous v1-compatible directory digest. This is a full + * backup verification, so I/O necessarily scales with bytes being verified; + * ordinary continuation uses current authority and does not call it. The byte + * total is reporting metadata; the digest includes each exact file length and + * all its bytes. Callers still need exclusive stopped-session admission for a + * consistent multi-store backup, not merely stable individual files. */ +export async function digestNativeHarnessBackupDirectory(directory: string, options: { sync?: boolean } = {}): Promise<{ sha256: string; bytes: number }> { + const hash = createHash("sha256"), buffer = Buffer.alloc(CHUNK_BYTES); + let bytes = 0; + const visit = async (current: string, relative: string, depth: number): Promise => { + if (depth > 128) throw new Error("runner_harness_backup_path_too_deep"); + const before = await lstat(current); + if (!before.isDirectory() || before.isSymbolicLink()) throw new Error("runner_harness_backup_unsafe_directory"); + let empty = true; + for await (const name of sortedNativeHarnessNames(current)) { + empty = false; + const path = join(current, name), entry = relative ? `${relative}/${name}` : name; + const metadata = await lstat(path); + if (metadata.isDirectory()) { + hash.update(`directory:${entry}:${metadata.mode & 0o777}\0`); + await visit(path, entry, depth + 1); + } else if (metadata.isSymbolicLink()) { + // Preserve the retained format: hash the link itself, never its target. + hash.update(`symlink:${entry}:${await readlink(path)}\0`); + } else if (metadata.isFile()) { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const opened = await file.stat(); + if (!opened.isFile() || !sameMetadata(metadata, opened)) throw new Error("runner_harness_backup_changed"); + hash.update(`file:${entry}:${metadata.mode & 0o777}:${metadata.size}\0`); + let length = 0; + for (;;) { + const read = await file.read(buffer, 0, buffer.length, null); + if (!read.bytesRead) break; + hash.update(buffer.subarray(0, read.bytesRead)); length += read.bytesRead; + } + if (length !== metadata.size || !sameMetadata(metadata, await file.stat())) throw new Error("runner_harness_backup_changed"); + if (options.sync) await file.sync(); + bytes += length; + } finally { await file.close(); } + } else throw new Error(`runner_harness_backup_unsupported_entry:${entry}`); + if (!sameMetadata(metadata, await lstat(path))) throw new Error("runner_harness_backup_changed"); + } + if (empty) hash.update(`directory:${relative}\0`); + if (!sameMetadata(before, await lstat(current))) throw new Error("runner_harness_backup_changed"); + if (options.sync) await syncNativeHarnessDirectory(current); + }; + await visit(directory, "", 0); + return { sha256: `sha256:${hash.digest("hex")}`, bytes }; +} + +export async function syncNativeHarnessDirectory(directory: string): Promise { + const file = await open(directory, constants.O_RDONLY | constants.O_NOFOLLOW); + try { if (!(await file.stat()).isDirectory()) throw new Error("runner_harness_backup_unsafe_directory"); await file.sync(); } + finally { await file.close(); } +} diff --git a/server/src/services/native-runtime/native-indexed-activation.test.ts b/server/src/services/native-runtime/native-indexed-activation.test.ts new file mode 100644 index 00000000000..0ba9b8fc8d0 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-activation.test.ts @@ -0,0 +1,27 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { expect, it } from "vitest"; +import { useIndexedNativeAuthority } from "./native-indexed-activation.js"; + +it("keeps retained indexed sessions recoverable when fresh activation is disabled", () => { + const root = mkdtempSync(resolve(tmpdir(), "indexed-activation-")); + const control = resolve(root, "control-plane"); + mkdirSync(control, { mode: 0o700 }); + mkdirSync(resolve(root, "runner"), { mode: 0o700 }); + const file = resolve(control, "control-plane-state.json"); + const write = (path: string, schema: string) => writeFileSync(path, JSON.stringify({ schema }), { mode: 0o600 }); + try { + expect(useIndexedNativeAuthority(control, false)).toBe(false); + expect(useIndexedNativeAuthority(control, true)).toBe(true); + write(file, "paperclip.runner.control-plane-state.v1"); + expect(useIndexedNativeAuthority(control, true)).toBe(false); + write(file, "paperclip.runner.authority-locator.v1"); + expect(useIndexedNativeAuthority(control, false)).toBe(true); + rmSync(file); + write(resolve(root, "runner/codex-provider-state.json"), "paperclip.runner.codex-provider-state.indexed.v1"); + expect(useIndexedNativeAuthority(control, false)).toBe(true); + write(resolve(root, "runner/codex-provider-state.json"), "paperclip.runner.codex-provider-state.v1"); + expect(useIndexedNativeAuthority(control, true)).toBe(false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/native-indexed-activation.ts b/server/src/services/native-runtime/native-indexed-activation.ts new file mode 100644 index 00000000000..7255994a27b --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-activation.ts @@ -0,0 +1,41 @@ +import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs"; +import { resolve } from "node:path"; +import { assertNoPendingLegacyMigration } from "../../vendor/paperclip-runner/index.js"; + +function schema(path: string): string | null { + let fd: number; + try { fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; } + try { + const stat = fstatSync(fd); + if (!stat.isFile()) throw new Error("native_indexed_locator_unsafe"); + // Large files are legacy checkpoints, never indexed locators. + if (stat.size > 8192) return "legacy"; + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(fd, bytes, offset, bytes.length - offset, offset); + if (!count) throw new Error("native_indexed_locator_changed"); + offset += count; + } + const after = fstatSync(fd); + if (stat.size !== after.size || stat.mtimeMs !== after.mtimeMs || stat.ctimeMs !== after.ctimeMs) throw new Error("native_indexed_locator_changed"); + const value = JSON.parse(bytes.toString("utf8")) as { schema?: string }; + if (value.schema?.includes("indexed.v1") || value.schema === "paperclip.runner.authority-locator.v1") { + if (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.())) throw new Error("native_indexed_locator_unsafe"); + } + return value.schema ?? "legacy"; + } finally { closeSync(fd); } +} + +/** The rollout flag controls fresh format selection, never retained recovery. */ +export function useIndexedNativeAuthority(directory: string, enableFresh: boolean): boolean { + assertNoPendingLegacyMigration(directory); + const controller = schema(resolve(directory, "control-plane-state.json")); + if (controller !== null) return controller === "paperclip.runner.authority-locator.v1"; + const provider = schema(resolve(directory, "../runner/codex-provider-state.json")); + if (provider !== null) return provider === "paperclip.runner.codex-provider-state.indexed.v1"; + const runner = schema(resolve(directory, "../runner/runner-state.json")); + if (runner !== null) return runner === "paperclip.runner.durable.state.indexed.v1"; + return enableFresh; +} diff --git a/server/src/services/native-runtime/native-indexed-checkpoint-proof.ts b/server/src/services/native-runtime/native-indexed-checkpoint-proof.ts new file mode 100644 index 00000000000..c283891fd82 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-checkpoint-proof.ts @@ -0,0 +1,48 @@ +import { lstatSync } from "node:fs"; +import { join } from "node:path"; +import { readIndexedLocalState, type IndexedLocalSnapshot } from "../../vendor/paperclip-runner/index.js"; +import { readNativeHarnessBackupManifestBytes } from "./native-harness-tree.js"; + +export interface NativeCheckpointBindings { + schema: "paperclip.native-checkpoint-bindings.v1"; + runId: string; + runner: { generation: string; sha256: string }; + provider: { generation: string; sha256: string }; +} +const invalid = () => new Error("runner_harness_state_mismatch: indexed_snapshot_binding"); +const object = (value: unknown): value is Record => !!value && typeof value === "object" && !Array.isArray(value); +const present = (path: string) => !!lstatSync(path, { throwIfNoEntry: false }); + +/** Verify bounded native current-state bindings before a transferred checkpoint + * can be published or restored. Full file/receipt integrity belongs to the + * native snapshot and outer streamed archive digest, not this current read. */ +export async function readNativeIndexedCheckpointProof(directory: string): Promise<{ + bindings: NativeCheckpointBindings; + runner: IndexedLocalSnapshot; + provider: IndexedLocalSnapshot; +} | null> { + const manifest = join(directory, "indexed-session-snapshot.json"); + if (!present(manifest)) { + // A legacy backup has no native databases. A naked indexed directory is + // never accepted as a legacy backup merely because its marker is missing. + if (["runner-state.sqlite", "codex-provider-state.sqlite"].some(name => present(join(directory, name)))) throw invalid(); + return null; + } + const value: unknown = JSON.parse(readNativeHarnessBackupManifestBytes(manifest).toString("utf8")); + if (!object(value) || value.schema !== "paperclip.runner.indexed-session-snapshot.v1" || !Array.isArray(value.states) || value.states.length !== 2 || + !Array.isArray(value.locators) || value.locators.length !== 2 || value.locators.some(locator => typeof locator !== "string" || Buffer.byteLength(locator) > 4096)) throw invalid(); + const runner = await readIndexedLocalState(join(directory, "runner-state.json")); + const provider = await readIndexedLocalState(join(directory, "codex-provider-state.json")); + const states = [runner, provider]; + for (let index = 0; index < 2; index++) { + const expected: unknown = value.states[index], locator: unknown = JSON.parse(value.locators[index]); + const actualLocator: unknown = JSON.parse(readNativeHarnessBackupManifestBytes(join(directory, index ? "codex-provider-state.json" : "runner-state.json")).toString("utf8")); + if (!object(expected) || expected.generation !== states[index]!.generation || expected.sha256 !== states[index]!.stateDigest || + !object(locator) || !object(actualLocator) || locator.schema !== actualLocator.schema || locator.binding !== actualLocator.binding) throw invalid(); + } + const runId = runner.state.runId; + if (typeof runId !== "string" || !runId || runId.length > 240 || runner.state.lifecycle !== "suspended" || + !Array.isArray(runner.state.outbox) || runner.state.outbox.length !== 0) throw invalid(); + return { bindings: { schema: "paperclip.native-checkpoint-bindings.v1", runId, + runner: { generation: runner.generation, sha256: runner.stateDigest }, provider: { generation: provider.generation, sha256: provider.stateDigest } }, runner, provider }; +} diff --git a/server/src/services/native-runtime/native-indexed-checkpoint.test.ts b/server/src/services/native-runtime/native-indexed-checkpoint.test.ts new file mode 100644 index 00000000000..6fa1e906e28 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-checkpoint.test.ts @@ -0,0 +1,95 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { buildNativeHarnessBackupManifest, verifyNativeHarnessBackup } from "./native-session-executor.js"; +import { readNativeIndexedCheckpointProof } from "./native-indexed-checkpoint-proof.js"; +import { digestNativeHarnessBackupDirectory } from "./native-harness-tree.js"; +import { createRemoteIndexedSessionCheckpoint } from "./native-indexed-checkpoint.js"; +import { inspectRemoteIndexedState } from "./native-indexed-inspection.js"; +import { downloadCheckpointDirectory } from "./native-checkpoint-transfer.js"; +import { loopbackCheckpointDuplexRunner } from "./native-checkpoint-transfer.test-support.js"; +import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js"; + +const names = ["runner-state", "codex-provider-state"], keys = ["runner", "codex-provider"], bindings = ["runner/r/s", "provider/r/s"]; +function fixture(root: string, runnerBinary: string) { + const directory = join(root, "source"); mkdirSync(directory, { mode: 0o700 }); + const states = [{ runId: "run-1", lifecycle: "suspended", runnerInstanceId: "r", normalizedSessionId: "s", nextSourceSeq: 2, ackedSourceSeq: 1 }, + { schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "session_open", config: { provider: "codex", driver: "codex_app_server" }, activeProviderTurnId: null, pendingEvents: [], queuedEvents: [], threadId: "original-provider-thread" }]; + const ancient = "界".repeat(400_000); + for (let index = 0; index < 2; index++) { + writeFileSync(join(directory, `${names[index]}.json`), JSON.stringify({ schema: index === 0 ? "paperclip.runner.durable.state.indexed.v1" : "paperclip.runner.codex-provider-state.indexed.v1", binding: bindings[index] }), { mode: 0o600 }); + const requests = [{ id: 1, operation: "commit", input: { key: keys[index], expectedGeneration: "0", bytes: Buffer.from(JSON.stringify(states[index])).toString("base64"), receipts: [{ namespace: "ancient", key: "exact", bytes: Buffer.from(ancient).toString("base64") }] } }, { id: 2, operation: "close" }]; + const output = execFileSync(runnerBinary, ["storage", "rpc", "--path", join(directory, `${names[index]}.sqlite`), "--binding", bindings[index]!, "--create", "true"], { input: requests.map(request => JSON.stringify(request)).join("\n") + "\n", maxBuffer: 8 * 1024 * 1024, encoding: "utf8" }); + expect(output.split("\n").filter(Boolean).map(line => JSON.parse(line).error)).toEqual([undefined, undefined]); + } + return { directory, states, ancient }; +} +it("snapshots, transfers, and reopens both native stores after source loss without an execute RPC deadline", async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "paperclip-indexed-checkpoint-"))), runnerBinary = resolvePaperclipRunnerBinary(); + const runner = loopbackCheckpointDuplexRunner(), execute = runner.execute.bind(runner); + runner.execute = vi.fn(async options => { + if (options.timeoutMs === 0) throw new Error("ordinary RPC deadline"); + return execute(options); + }); + const duplex = vi.spyOn(runner, "openDuplexChannel"); + try { + const source = fixture(root, runnerBinary), input = { runner, runnerBinary, directory: source.directory }; + const checkpoint = await createRemoteIndexedSessionCheckpoint(input); + expect((await createRemoteIndexedSessionCheckpoint(input)).directory).toBe(checkpoint.directory); + const current = join(root, "failover-backups", "current"), restored = join(current, "runner"); + mkdirSync(join(current, "codex-home"), { recursive: true, mode: 0o700 }); + await downloadCheckpointDirectory({ runner, sourcePath: checkpoint.directory, targetPath: restored, mode: 0o700 }); + await checkpoint.dispose(); expect(existsSync(checkpoint.directory)).toBe(false); + const before = await digestNativeHarnessBackupDirectory(restored); + const proof = await readNativeIndexedCheckpointProof(restored); + expect(proof?.bindings).toMatchObject({ runId: "run-1", runner: { generation: "1" }, provider: { generation: "1" } }); + expect(await digestNativeHarnessBackupDirectory(restored)).toEqual(before); + const execution = { provider: { kind: "codex", model: "gpt-5.6-sol", approvalPolicy: "never" }, + binding: { companyId: "company", runId: "run-1", issueId: "issue", agentId: "agent", executionWorkspaceId: "workspace" }, + workspace: { cwd: "/workspace", repoUrl: "https://example.test/repo.git", repoRef: "main", branchName: "snapshot-test" }, + session: { normalizedSessionId: "s", driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + } as unknown as NativeExecutionInput; + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: current, execution, runnerInstanceId: "r", sourceProviderLeaseId: "sandbox", providerSessionIdentity: { providerSessionId: "original-provider-thread", providerBackendSessionId: null, providerSessionIdentity: null } }); + const manifestPath = join(current, "manifest.json"); writeFileSync(manifestPath, JSON.stringify(manifest), { mode: 0o600 }); + expect(manifest.nativeState).toEqual(proof!.bindings); + const next = { ...execution, binding: { ...execution.binding, runId: "run-2" } }; + expect(await verifyNativeHarnessBackup({ root, execution: next, runnerInstanceId: "r" })).not.toBeNull(); + expect(await verifyNativeHarnessBackup({ root, execution: next, runnerInstanceId: "r" })).not.toBeNull(); + writeFileSync(manifestPath, JSON.stringify({ ...manifest, nativeState: { ...manifest.nativeState, provider: { ...manifest.nativeState!.provider, generation: "2" } } })); + expect(await verifyNativeHarnessBackup({ root, execution: next, runnerInstanceId: "r" })).toBeNull(); + writeFileSync(manifestPath, JSON.stringify(manifest)); + const nativeManifestPath = join(restored, "indexed-session-snapshot.json"), nativeBytes = readFileSync(nativeManifestPath); + const native = JSON.parse(nativeBytes.toString()); native.states[1].generation = "2"; + writeFileSync(nativeManifestPath, JSON.stringify(native)); + await expect(readNativeIndexedCheckpointProof(restored)).rejects.toThrow("indexed_snapshot_binding"); + writeFileSync(nativeManifestPath, nativeBytes); + rmSync(source.directory, { recursive: true }); + for (let index = 0; index < 2; index++) { + const current = await inspectRemoteIndexedState({ runner, runnerBinary, path: join(restored, `${names[index]}.json`) }); + expect(current.generation).toBe("1"); + expect(current.stateDigest).toBe(createHash("sha256").update(JSON.stringify(source.states[index])).digest("hex")); + const requests = [{ id: 1, operation: "get", input: { namespace: "ancient", key: "exact" } }, { id: 2, operation: "close" }]; + const result = execFileSync(runnerBinary, ["storage", "rpc", "--path", join(restored, `${names[index]}.sqlite`), "--binding", bindings[index]!], { input: requests.map(request => JSON.stringify(request)).join("\n") + "\n", encoding: "utf8", maxBuffer: 4 * 1024 * 1024 }); + expect(Buffer.from(JSON.parse(result.split("\n")[0]!).value, "base64").toString("utf8")).toBe(source.ancient); + } + expect(duplex.mock.calls.some(([request]) => request.command.some(value => value.includes("snapshot-session")))).toBe(true); + expect(vi.mocked(runner.execute).mock.calls.every(([request]) => request.timeoutMs !== 0)).toBe(true); + } finally { rmSync(root, { recursive: true, force: true }); } +}, 120_000); +it("rejects unsupported snapshot transport before opening or copying state", async () => { + const runner = loopbackCheckpointDuplexRunner(); runner.openDuplexChannel = undefined; + await expect(createRemoteIndexedSessionCheckpoint({ runner, runnerBinary: "/private/runnerd", directory: "/private/session" })).rejects.toThrow("transport_unavailable"); +}); + +it("does not classify indexed stores with a missing snapshot marker as a legacy backup", async () => { + const root = mkdtempSync(join(tmpdir(), "paperclip-missing-snapshot-marker-")); + try { + expect(await readNativeIndexedCheckpointProof(root)).toBeNull(); + writeFileSync(join(root, "runner-state.sqlite"), "unproven", { mode: 0o600 }); + await expect(readNativeIndexedCheckpointProof(root)).rejects.toThrow("indexed_snapshot_binding"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/native-indexed-checkpoint.ts b/server/src/services/native-runtime/native-indexed-checkpoint.ts new file mode 100644 index 00000000000..f27d809aef9 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-checkpoint.ts @@ -0,0 +1,45 @@ +import { createHash } from "node:crypto"; +import { posix } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { inspectRemoteIndexedState } from "./native-indexed-inspection.js"; +import { runCheckpointCommand } from "./native-checkpoint-command.js"; + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** Replace raw copies of live SQLite files with native, verified snapshots. + * Caller owns the stopped-session/maintenance admission and must bind the + * provider home and controller authority before publishing a harness backup. */ +export async function createRemoteIndexedSessionCheckpoint(input: { + runner: CommandManagedRuntimeRunner; + runnerBinary: string; + directory: string; +}): Promise<{ directory: string; dispose(): Promise }> { + if (!input.runner.openDuplexChannel || ![input.runnerBinary, input.directory].every(path => path.startsWith("/") && !path.includes("\0"))) { + throw new Error("native_indexed_checkpoint_transport_unavailable"); + } + const states = []; + for (const name of ["runner-state.json", "codex-provider-state.json"]) { + const current = await inspectRemoteIndexedState({ ...input, path: posix.join(input.directory, name) }); + states.push({ generation: current.generation, sha256: current.stateDigest }); + } + // A controller restart retries the same durable native copy. No aggregate + // transfer deadline or in-memory history is introduced here. + const id = createHash("sha256").update(JSON.stringify({ directory: input.directory, states })).digest("hex").slice(0, 32); + const directory = posix.join(posix.dirname(input.directory), `.paperclip-indexed-snapshot-${id}`); + const command = [input.runnerBinary, "storage", "snapshot-session", "--directory", input.directory, "--destination", directory, + "--runner-generation", states[0]!.generation, "--runner-sha256", states[0]!.sha256, + "--provider-generation", states[1]!.generation, "--provider-sha256", states[1]!.sha256]; + const result: unknown = JSON.parse(await runCheckpointCommand(input.runner, command.map(quote).join(" "))); + const response = result as { snapshot?: unknown; destination?: unknown; states?: unknown } | null; + if (!response || response.snapshot !== "complete" || response.destination !== directory || JSON.stringify(response.states) !== JSON.stringify(states)) { + throw new Error("native_indexed_checkpoint_incomplete"); + } + return { directory, async dispose() { + // Only a successfully copied and published host backup calls this. Failed + // transfers retain native source pins/staging for an exact retry. These + // two fixed job paths were created by the command above; never delete the + // live runner directory or discover cleanup targets by a wildcard. + const staging = posix.join(posix.dirname(directory), `.${posix.basename(directory)}.preparing`); + await runCheckpointCommand(input.runner, `rm -rf -- ${quote(directory)} ${quote(staging)}`); + } }; +} diff --git a/server/src/services/native-runtime/native-indexed-inspection.test.ts b/server/src/services/native-runtime/native-indexed-inspection.test.ts new file mode 100644 index 00000000000..747a145b3b4 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-inspection.test.ts @@ -0,0 +1,83 @@ +import { createHash } from "node:crypto"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { inspectRemoteIndexedState } from "./native-indexed-inspection.js"; +import { loopbackCheckpointDuplexRunner } from "./native-checkpoint-transfer.test-support.js"; +import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js"; + +const snapshot = { state: { lifecycle: "ready", outbox: [{ indexedPending: true }], activeInput: "界".repeat(500_000) }, + generation: "9007199254740992", stateDigest: "a".repeat(64), preparation: null }; +function frames() { + const bytes = Buffer.from(JSON.stringify(snapshot)); + const rows: Record[] = [{ schema: "paperclip.indexed-inspection.v1", byteLength: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }]; + for (let start = 0; start < bytes.length; start += 192 * 1024) rows.push({ index: rows.length - 1, bytes: bytes.subarray(start, start + 192 * 1024).toString("base64") }); + rows.push({ complete: true }); return rows; +} +function runner(rows: Record[], exit = { exitCode: 0, transportClosed: false }, hold = false) { + const wire = Buffer.from(rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + const stop = vi.fn(), close = vi.fn(async () => {}), execute = vi.fn(); + const openDuplexChannel = vi.fn(async () => ({ + stop, close, write() {}, + onData(callback: (bytes: Buffer) => void) { + if (!hold) queueMicrotask(() => { + // Fragment JSON, UTF-8 base64 and headers at arbitrary byte boundaries. + for (let start = 0; start < wire.length; start += 7919) callback(wire.subarray(start, start + 7919)); + }); + }, + onExit(callback: (value: typeof exit) => void) { if (!hold) queueMicrotask(() => callback(exit)); }, + })); + return { runner: { execute, openDuplexChannel } as CommandManagedRuntimeRunner, stop, close, execute, openDuplexChannel }; +} +const inspect = (fixture: ReturnType, timeoutMs?: number) => inspectRemoteIndexedState({ runner: fixture.runner, runnerBinary: "/private/runnerd", path: "/private/runner/runner-state.json", timeoutMs }); + +describe("remote indexed current-state inspection", () => { + it("reads a real runner-owned WAL database through the native binary and duplex transport", async () => { + const root = mkdtempSync(join(tmpdir(), "paperclip-native-inspection-")); + const path = join(root, "runner-state.json"), database = join(root, "runner-state.sqlite"); + const db = new DatabaseSync(database); + try { + chmodSync(database, 0o600); + writeFileSync(`${database}.lifetime`, "", { mode: 0o600 }); + writeFileSync(path, JSON.stringify({ schema: "paperclip.runner.durable.state.indexed.v1", binding: "inspection" }), { mode: 0o600 }); + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE store_binding(singleton INTEGER PRIMARY KEY,schema_version INTEGER,binding TEXT); INSERT INTO store_binding VALUES(1,2,'inspection'); CREATE TABLE current_state(key TEXT PRIMARY KEY,generation INTEGER,bytes BLOB,digest BLOB); CREATE TABLE receipt_prepare(state_key TEXT,generation INTEGER,body BLOB,digest BLOB)"); + const state = { nextSourceSeq: 3, ackedSourceSeq: 1, activeInput: snapshot.state.activeInput }; + const bytes = Buffer.from(JSON.stringify(state)), stateDigest = createHash("sha256").update(bytes).digest(); + db.prepare("INSERT INTO current_state VALUES('runner',?,?,?)").run(9007199254740992n, bytes, stateDigest); + const input = { runner: loopbackCheckpointDuplexRunner(), runnerBinary: resolvePaperclipRunnerBinary(), path }; + expect(await inspectRemoteIndexedState(input)).toEqual({ state: { ...state, outbox: [{ indexedPending: true }] }, generation: "9007199254740992", stateDigest: stateDigest.toString("hex"), preparation: null }); + db.exec("UPDATE current_state SET digest=zeroblob(32)"); + await expect(inspectRemoteIndexedState(input)).rejects.toThrow("invalid_or_incomplete"); + } finally { db.close(); rmSync(root, { recursive: true, force: true }); } + }); + it("streams over ordinary RPC output capacity, preserves exact generation and pending work", async () => { + const fixture = runner(frames()); + expect(await inspect(fixture)).toEqual(snapshot); + expect(fixture.execute).not.toHaveBeenCalled(); expect(fixture.stop).not.toHaveBeenCalled(); expect(fixture.close).toHaveBeenCalledOnce(); + expect(fixture.openDuplexChannel).toHaveBeenCalledWith({ command: ["/private/runnerd", "storage", "inspect-state", "--path", "/private/runner/runner-state.json"] }); + }); + it.each(["missing-footer", "digest", "reordered", "over-capacity", "short-chunk", "after-footer"])("rejects %s instead of treating missing output as settled state", async kind => { + const rows = frames(); + if (kind === "missing-footer") rows.pop(); + if (kind === "digest") rows[0]!.sha256 = "b".repeat(64); + if (kind === "reordered") rows[1]!.index = 1; + if (kind === "over-capacity") rows[0]!.byteLength = 40 * 1024 * 1024 + 1; + if (kind === "short-chunk") rows[1]!.bytes = "eA=="; + if (kind === "after-footer") rows.push({ complete: true }); + const fixture = runner(rows); + await expect(inspect(fixture)).rejects.toThrow("invalid_or_incomplete"); + expect(fixture.stop).toHaveBeenCalledOnce(); expect(fixture.close).toHaveBeenCalledOnce(); + }); + it.each([{ exitCode: 7, transportClosed: false }, { exitCode: 0, transportClosed: true }])("requires a clean command exit (%j)", async exit => { + const fixture = runner(frames(), exit); + await expect(inspect(fixture)).rejects.toThrow("invalid_or_incomplete"); expect(fixture.close).toHaveBeenCalledOnce(); + }); + it("stops and closes its own channel on a stalled bounded current read", async () => { + const fixture = runner([], undefined, true); + await expect(inspect(fixture, 25)).rejects.toThrow("invalid_or_incomplete"); + expect(fixture.stop).toHaveBeenCalledOnce(); expect(fixture.close).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/services/native-runtime/native-indexed-inspection.ts b/server/src/services/native-runtime/native-indexed-inspection.ts new file mode 100644 index 00000000000..cb7a6c118c7 --- /dev/null +++ b/server/src/services/native-runtime/native-indexed-inspection.ts @@ -0,0 +1,41 @@ +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { IndexedInspectionDecoder, type IndexedLocalSnapshot } from "../../vendor/paperclip-runner/index.js"; + +export type NativeIndexedInspection = IndexedLocalSnapshot; + +/** A single read-only native snapshot, streamed through an owned remote + * channel. The wire limit bounds current state, never lifetime history. */ +export async function inspectRemoteIndexedState(input: { + runner: CommandManagedRuntimeRunner; + runnerBinary: string; + path: string; + timeoutMs?: number; +}): Promise { + if (!input.runner.openDuplexChannel || ![input.runnerBinary, input.path].every(path => path.startsWith("/") && !path.includes("\0"))) { + throw new Error("native_indexed_inspection_transport_unavailable"); + } + const channel = await input.runner.openDuplexChannel({ command: [input.runnerBinary, "storage", "inspect-state", "--path", input.path] }); + let timer: ReturnType | undefined; + try { + return await new Promise((resolve, reject) => { + const decoder = new IndexedInspectionDecoder(); + let settled = false; + const fail = () => { + if (settled) return; + settled = true; + try { channel.stop(); } catch { /* Still await owned close below. */ } + reject(new Error("native_indexed_inspection_invalid_or_incomplete")); + }; + timer = setTimeout(fail, input.timeoutMs ?? 30_000); + channel.onData(chunk => { if (!settled) { try { decoder.push(chunk); } catch { fail(); } } }); + channel.onExit(exit => { + if (settled) return; + try { + if (exit.transportClosed || exit.exitCode !== 0) throw new Error(); + const value = decoder.finish(); + settled = true; resolve(value); + } catch { fail(); } + }); + }); + } finally { if (timer) clearTimeout(timer); await channel.close(); } +} diff --git a/server/src/services/native-runtime/native-journal-projection-async.ts b/server/src/services/native-runtime/native-journal-projection-async.ts index 0a1713b4604..e6fe951168e 100644 --- a/server/src/services/native-runtime/native-journal-projection-async.ts +++ b/server/src/services/native-runtime/native-journal-projection-async.ts @@ -1,3 +1,7 @@ +import { readIndexedAuthorityProof } from "./native-authority-read-context.js"; +import { createHash } from "node:crypto"; +import { open } from "node:fs/promises"; +import { constants } from "node:fs"; import { Worker } from "node:worker_threads"; import { performance } from "node:perf_hooks"; import type { readNativeJournalProjection as readProjection } from "./native-journal-projection.js"; @@ -153,6 +157,8 @@ export async function readNativeJournalProjection( path: string, purpose: "identity" | "evidence" = "evidence", ) { + const indexed = await readIndexedAuthorityProof(path); + if (indexed) return indexed; return (await reader.read({ kind: "projection", path, @@ -160,6 +166,8 @@ export async function readNativeJournalProjection( })) as ReturnType; } export async function scanNativeStateFile(path: string, maximum: number) { + const indexed = await readIndexedAuthorityProof(path); + if (indexed) return { sha256: indexed.sha256, byteSize: indexed.byteSize }; return (await reader.read({ kind: "scan", path, maximum })) as { sha256: string; byteSize: number; @@ -170,6 +178,8 @@ export async function nativeStateFileIncludes( maximum: number, needles: string[], ) { + const indexed = await readIndexedAuthorityProof(path); + if (indexed) return needles.some((needle) => indexed.bytes.includes(needle)); return (await reader.read({ kind: "includes", path, @@ -178,6 +188,25 @@ export async function nativeStateFileIncludes( })) as boolean; } export async function nativeStateBase64Fingerprint(files: File[]) { + const indexed = await Promise.all(files.map((file) => readIndexedAuthorityProof(file.path))); + if (indexed.some(Boolean)) { + const bytes = await Promise.all(files.map(async (file, index) => { + if (indexed[index]) return indexed[index]!.bytes; + const proof = await scanNativeStateFile(file.path, file.maximum); + const fd = await open(file.path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let value: Buffer; + try { + const stat = await fd.stat(); + if (!stat.isFile() || stat.size > file.maximum) throw new Error("native_state_file_changed"); + value = Buffer.alloc(stat.size); + let offset = 0; + while (offset < value.length) { const { bytesRead } = await fd.read(value, offset, value.length - offset, offset); if (!bytesRead) throw new Error("native_state_file_changed"); offset += bytesRead; } + } finally { await fd.close(); } + if (value.length > file.maximum || createHash("sha256").update(value).digest("hex") !== proof.sha256) throw new Error("native_state_file_changed"); + return value; + })); + return { sha256: createHash("sha256").update(JSON.stringify(bytes.map((value) => value.toString("base64")))).digest("hex"), fileSha256: bytes.map((value) => createHash("sha256").update(value).digest("hex")) }; + } return (await reader.read({ kind: "fingerprint", files })) as { sha256: string; fileSha256: string[]; diff --git a/server/src/services/native-runtime/native-legacy-migration.ts b/server/src/services/native-runtime/native-legacy-migration.ts new file mode 100644 index 00000000000..b470f8849d7 --- /dev/null +++ b/server/src/services/native-runtime/native-legacy-migration.ts @@ -0,0 +1,117 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { resolve } from "node:path"; +import { and, eq, gt, sql } from "drizzle-orm"; +import { nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { migrateLegacySessionAuthority, readRunnerdArtifactBinding, type DurableRecoveryIdentity, type PreparedLocalSession } from "../../vendor/paperclip-runner/index.js"; +import { createNativePostgresAuthorityStore, type AuthorityTransaction, type PostgresAuthorityBinding } from "./postgres-authority-store.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { startNativeMaintenanceLeaseRenewal } from "./native-maintenance-lease.js"; + +const execute = promisify(execFile); + +/** Retains the exact admitted migration owner for a job of any duration. + * Short timing options allow real database fault tests without a minute-long + * fixture; production callers use the 60-second renewable lease. */ +export async function startNativeLegacyMigrationLease(input: { + db: Db; + owner: PostgresAuthorityBinding; + lease: { owner: string; attempt: number }; + assertStoppedProcessTree(): Promise; +}, options: { leaseMs?: number; intervalMs?: number; timeoutMs?: number } = {}) { + const leaseMs = options.leaseMs ?? 60_000; + const intervalMs = options.intervalMs ?? 15_000; + const timeoutMs = options.timeoutMs ?? 30_000; + if (![leaseMs, intervalMs, timeoutMs].every(value => Number.isSafeInteger(value) && value > 0) || intervalMs + timeoutMs >= leaseMs) throw new Error("native_legacy_migration_lease_policy_invalid"); + const owner = { ...input.owner }; + const lease = { ...input.lease }; + const controller = await currentNativeControllerIdentity(); + const condition = () => and(eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId), + eq(nativeRunFinalizations.runId, owner.runId), eq(nativeRunFinalizations.leaseOwner, lease.owner), eq(nativeRunFinalizations.attempt, lease.attempt), + eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid), + eq(nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt), gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`)); + const renewLease = async (connection: Db | AuthorityTransaction = input.db) => { + const rows = await connection.update(nativeRunFinalizations) + .set({ leaseExpiresAt: sql`clock_timestamp() + ${leaseMs} * interval '1 millisecond'`, updatedAt: sql`clock_timestamp()` }) + .where(condition()).returning({ runId: nativeRunFinalizations.runId }); + if (rows.length !== 1) throw new Error("native_legacy_migration_lease_lost"); + }; + await input.assertStoppedProcessTree(); + await renewLease(); + const maintenance = startNativeMaintenanceLeaseRenewal(renewLease, intervalMs, { renewalTimeoutMs: timeoutMs }); + const assertLease = async (tx?: AuthorityTransaction) => { + if (tx) { + maintenance.assertKnown(); + // This update both fences publication and renews using the transaction's + // own connection. Waiting for the background renewal while holding its + // target row could deadlock. No history-sized work occurs in this tx. + await renewLease(tx); + maintenance.assertKnown(); + return; + } + await maintenance.assert(); + const query = input.db.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(condition()); + const rows = await query; + if (rows.length !== 1) throw new Error("native_legacy_migration_lease_lost"); + }; + return { ...maintenance, assertLease }; +} + +/** Server entry point for an already-quiesced local migration. The caller's + * normal recovery admission must supply the exact stopped process-tree proof; + * this service does not infer death from an expired lease or invent a new run. + * Both the filesystem steps and each PostgreSQL commit recheck ownership. */ +export async function migrateNativeLegacySessionAuthority(input: { + db: Db; + root: string; + owner: PostgresAuthorityBinding; + identity: DurableRecoveryIdentity; + agentId: string; + runnerBinary: string; + maxOutboxBytes: number; + p0ReserveBytes: number; + fenceId: string; + lease: { owner: string; attempt: number }; + assertStoppedProcessTree(): Promise; +}): Promise { + const { owner, identity } = input; + if (identity.runId !== owner.runId || identity.runnerInstanceId !== owner.runnerInstanceId || identity.normalizedSessionId !== owner.normalizedSessionId || identity.environmentLeaseId !== owner.environmentLeaseId) throw new Error("native_legacy_migration_binding_mismatch"); + if (!Number.isSafeInteger(input.maxOutboxBytes) || input.maxOutboxBytes <= 0 || input.maxOutboxBytes > 512 * 1024 * 1024 || + !Number.isSafeInteger(input.p0ReserveBytes) || input.p0ReserveBytes <= 0 || input.p0ReserveBytes >= input.maxOutboxBytes) throw new Error("native_legacy_migration_capacity_invalid"); + const maintenance = await startNativeLegacyMigrationLease(input); + const { assertLease } = maintenance; + try { + const authority = createNativePostgresAuthorityStore(input.db, owner, input.agentId, async tx => { await assertLease(tx); }); + const artifact = readRunnerdArtifactBinding(input.runnerBinary); + await migrateLegacySessionAuthority({ + root: input.root, identity, authority, fenceId: input.fenceId, runnerBinary: input.runnerBinary, + assertExclusiveFence: async () => { await assertLease(); await input.assertStoppedProcessTree(); await assertLease(); }, + prepareLocal: async destination => { + await assertLease(); + const preparation = execute(input.runnerBinary, ["storage", "stage-legacy", + "--state-dir", resolve(input.root, "runner"), "--destination", destination, "--fence-id", input.fenceId, + "--connect-url", `ws://127.0.0.1:1/api/runner/v1/connect/${identity.runId}`, + "--runner-version", artifact.version, "--runner-digest", artifact.digest, + "--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId, + "--session-id", identity.normalizedSessionId, "--run-id", identity.runId, "--turn-id", identity.turnId, "--item-id", identity.itemId, + "--max-outbox-bytes", String(input.maxOutboxBytes), "--p0-reserve-bytes", String(input.p0ReserveBytes), + ], { maxBuffer: 128 * 1024, signal: maintenance.signal, killSignal: "SIGKILL", env: { PATH: process.env.PATH, LANG: "C.UTF-8" } }); + // An AbortError can precede child exit. Keep the migration lifetime + // until the native storage writer is actually reaped. + const closed = new Promise(resolve => preparation.child.once("close", () => resolve())); + let stdout: string; + try { ({ stdout } = await preparation); } finally { await closed; } + await assertLease(); + // The session coordinator validates every returned field against the + // actual staged stores. No provider credentials cross this subprocess. + return JSON.parse(stdout) as PreparedLocalSession; + }, + }); + await maintenance.assert(); + } catch (error) { + // Preserve lease loss rather than reporting the subprocess's derivative + // AbortError. Staging never launches provider children or publishes peers. + maintenance.assertKnown(); + throw error; + } finally { await maintenance.stop(); } +} diff --git a/server/src/services/native-runtime/native-maintenance-lease.test.ts b/server/src/services/native-runtime/native-maintenance-lease.test.ts new file mode 100644 index 00000000000..032236479bd --- /dev/null +++ b/server/src/services/native-runtime/native-maintenance-lease.test.ts @@ -0,0 +1,66 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { startNativeMaintenanceLeaseRenewal } from "./native-maintenance-lease.js"; +afterEach(() => vi.useRealTimers()); + +it("renews for arbitrarily many intervals without queuing work during a slow database write", async () => { + vi.useFakeTimers(); + let release!: () => void; + const renew = vi.fn(() => new Promise(resolve => { release = resolve; })); + const lease = startNativeMaintenanceLeaseRenewal(renew); + try { + await vi.advanceTimersByTimeAsync(15_000); expect(renew).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(24 * 60 * 60_000); expect(renew).toHaveBeenCalledTimes(1); + release(); await lease.assert(); + await vi.advanceTimersByTimeAsync(15_000); expect(renew).toHaveBeenCalledTimes(2); + release(); await lease.stop(); + await vi.advanceTimersByTimeAsync(60_000); expect(renew).toHaveBeenCalledTimes(2); + } finally { release?.(); await lease.stop(); } +}); + +it("retains renewal failure so no later authority check can ignore lease loss", async () => { + vi.useFakeTimers(); + const renew = vi.fn(async () => { throw new Error("ownership lost"); }); + const lease = startNativeMaintenanceLeaseRenewal(renew); + try { + await vi.advanceTimersByTimeAsync(15_000); + await expect(lease.assert()).rejects.toThrow("ownership lost"); + await vi.advanceTimersByTimeAsync(60_000); expect(renew).toHaveBeenCalledTimes(1); + await expect(lease.assert()).rejects.toThrow("ownership lost"); + } finally { await lease.stop(); } +}); + +it("cancels attached staging work when renewal fails and preserves the original reason", async () => { + vi.useFakeTimers(); + const reason = new Error("migration owner changed"); + const lease = startNativeMaintenanceLeaseRenewal(async () => { throw reason; }); + const abort = vi.fn(); + lease.signal.addEventListener("abort", abort); + try { + expect(lease.signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(15_000); + expect(lease.signal.aborted).toBe(true); + expect(lease.signal.reason).toBe(reason); + expect(abort).toHaveBeenCalledTimes(1); + expect(() => lease.assertKnown()).toThrow(reason); + } finally { await lease.stop(); } +}); + +it("bounds a lost database response and never revives the owner when it arrives late", async () => { + vi.useFakeTimers(); + let complete!: () => void; + const renew = vi.fn(() => new Promise(resolve => { complete = resolve; })); + const lease = startNativeMaintenanceLeaseRenewal(renew, 15_000, { renewalTimeoutMs: 30_000 }); + try { + await vi.advanceTimersByTimeAsync(15_000); + // A transaction must be able to check the sticky failure without awaiting + // a background query blocked on a row lock owned by that transaction. + lease.assertKnown(); + await vi.advanceTimersByTimeAsync(30_000); + await expect(lease.assert()).rejects.toThrow("native_maintenance_lease_renewal_timeout"); + expect(lease.signal.aborted).toBe(true); + complete(); + await vi.advanceTimersByTimeAsync(60_000); + expect(renew).toHaveBeenCalledTimes(1); + expect(() => lease.assertKnown()).toThrow("native_maintenance_lease_renewal_timeout"); + } finally { complete?.(); await lease.stop(); } +}); diff --git a/server/src/services/native-runtime/native-maintenance-lease.ts b/server/src/services/native-runtime/native-maintenance-lease.ts new file mode 100644 index 00000000000..3840f809c28 --- /dev/null +++ b/server/src/services/native-runtime/native-maintenance-lease.ts @@ -0,0 +1,33 @@ +/** Keep a bounded lease alive during asynchronous history scans/copies. Never + * queue renewals, revive an expired lease, or conceal a failed ownership check. */ +export function startNativeMaintenanceLeaseRenewal(renew: () => Promise, intervalMs = 15_000, options: { renewalTimeoutMs?: number } = {}) { + let pending: Promise | null = null, failure: unknown, stopped = false; + const cancellation = new AbortController(); + const attempt = async () => { + if (options.renewalTimeoutMs === undefined) return renew(); + let timeout: NodeJS.Timeout | undefined; + try { + await Promise.race([renew(), new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error("native_maintenance_lease_renewal_timeout")), options.renewalTimeoutMs); + timeout.unref(); + })]); + } finally { if (timeout) clearTimeout(timeout); } + }; + const timer = setInterval(() => { + if (stopped || pending || failure) return; + pending = Promise.resolve().then(attempt).catch(error => { + failure = error ?? new Error("native_cleanup_maintenance_lease_lost"); + cancellation.abort(failure); + }).finally(() => { pending = null; }); + }, intervalMs); + timer.unref(); + return { + signal: cancellation.signal, + /** Transaction-local fences must not await a renewal that may itself be + * waiting for that transaction's row lock. They still check actual owner + * and expiry in their own transaction before publishing. */ + assertKnown() { if (failure) throw failure; }, + async assert() { await pending; if (failure) throw failure; }, + async stop() { stopped = true; clearInterval(timer); await pending; }, + }; +} diff --git a/server/src/services/native-runtime/native-maintenance-no-launch.ts b/server/src/services/native-runtime/native-maintenance-no-launch.ts index 6a159a962c0..64f640d7e0b 100644 --- a/server/src/services/native-runtime/native-maintenance-no-launch.ts +++ b/server/src/services/native-runtime/native-maintenance-no-launch.ts @@ -1,3 +1,4 @@ +import { isProviderProcessGeneration } from "../../vendor/paperclip-runner/index.js"; import { createHash } from "node:crypto"; import { validatePrpEvent } from "../../vendor/paperclip-runner/index.js"; import { canonicalNativeJson, nativeSha256 } from "./canonical.js"; @@ -214,7 +215,7 @@ export function verifyRetainedMaintenanceNoLaunch( ); requireProof( original.provider.lifecycle === "turn_active" && - Number.isSafeInteger(original.provider.providerProcessGeneration), + isProviderProcessGeneration(original.provider.providerProcessGeneration), ); requireProof( typeof original.provider.threadId === "string" && diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index ee69e3f494a..1be13545e47 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -8,6 +8,20 @@ import { } from "./native-restart-recovery.js"; describe("native restart recovery classification", () => { + it("requires indexed process-tree retirement even when all recorded PIDs are gone", () => { + const evidence = { + runnerPidAlive: false, runnerGroupAlive: false, processStartMatches: false, + knownProviderProcessAlive: false, knownProviderProcessIdentityAmbiguous: false, + unretiredIndexedProcessOwner: true, hasCheckpoint: true, + checkpointIdentityMatches: true, hasProviderEvidence: true, + }; + expect(classifyNativeRunnerRecoveryEvidence(evidence)).toEqual({ + claimKind: null, reason: "provider_process_tree_retirement_unproven", + }); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, runnerPidAlive: true, processStartMatches: true }).claimKind).toBe("reattach_existing_runner"); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: true }).claimKind).toBe("reattach_remote_runner"); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, unretiredIndexedProcessOwner: false }).claimKind).toBe("resume_dead_runner"); + }); it("does not reopen a failed checkpoint or reset an exhausted provider budget", () => { const evidence = { runnerPidAlive: false, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 2fc54b97a25..5be334f8ba0 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1,14 +1,15 @@ import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import { recordNativeLocalProcessStop } from "../native-local-process-stop.js"; -import { randomUUID } from "node:crypto"; -import { and, desc, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { createHash, randomUUID } from "node:crypto"; +import { and, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, - heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, + nativeAuthorityWork, + nativeSessionAuthorities, } from "@paperclipai/db"; import { readProcessStartedAt } from "../hot-restart.js"; import { getServerInfoSnapshot } from "../../server-info.js"; @@ -16,6 +17,7 @@ import { redactSensitiveText } from "../../redaction.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; +import { readRunEventLane } from "../run-event-history.js"; export type NativeControllerIdentity = { bootId: string; @@ -77,6 +79,17 @@ export type NativeRestartRecoveryDisposition = reason: string; }; +/** Kept as a bounded diagnostic for retained rows and API compatibility. + * Ownership is the exact random leaseOwner plus controller identity, never + * this counter alone. Saturation must not prevent another controller claim + * or make an old lease valid again. */ +export function nextNativeControllerGeneration(current: number): number { + if (!Number.isInteger(current) || current < 0 || current > 2_147_483_647) { + throw new Error("native_controller_generation_invalid"); + } + return Math.min(current + 1, 2_147_483_647); +} + export function nextNativeProviderAttempt( currentAttempt: number, recoveryKind?: NativeRestartRecoveryClaim["kind"], @@ -289,6 +302,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: { processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; + unretiredIndexedProcessOwner?: boolean; hasCheckpoint: boolean; checkpointIdentityMatches?: boolean; hasProviderEvidence: boolean; @@ -330,6 +344,12 @@ export function classifyNativeRunnerRecoveryEvidence(input: { : "live_runner_process_group_without_exact_pid", }; } + if (input.unretiredIndexedProcessOwner) { + return { + claimKind: null, + reason: "provider_process_tree_retirement_unproven", + }; + } if (input.knownProviderProcessAlive) { return { claimKind: null, @@ -659,28 +679,25 @@ export async function claimNativeRestartRecoveries(input: { typeof checkpointRecord.providerIdentity === "object" && !Array.isArray(checkpointRecord.providerIdentity) && Object.keys(checkpointRecord.providerIdentity).length > 0); - const providerEvents = await tx - .select({ - id: heartbeatRunEvents.id, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, row.run.id), - inArray(heartbeatRunEvents.eventType, [ - "harness.ready", - "session.started", - "session.resumed", - "session.updated", - "turn.started", - "provider.event", - "provider.rpc_result", - ]), - ), - ) - .orderBy(desc(heartbeatRunEvents.id)) - .limit(100); + const [indexedAuthority] = row.run.nativeSessionId ? await tx.select().from(nativeSessionAuthorities).where(and( + eq(nativeSessionAuthorities.companyId, row.run.companyId), + eq(nativeSessionAuthorities.issueId, row.coordinator.issueId), + eq(nativeSessionAuthorities.normalizedSessionId, row.run.nativeSessionId), + eq(nativeSessionAuthorities.runId, row.run.id), + )).for("share") : []; + // A filtered LIMIT over the transcript can still inspect all history when + // recent output contains none of these event types. Indexed recovery uses + // the materialized owner relation and sticky bootstrap fact exclusively. + const providerEvents = indexedAuthority + ? [] + : (await readRunEventLane( + tx as unknown as Db, + row.run.id, + "provider-identity", + 100, + )) + .filter((event) => event.companyId === row.run.companyId) + .map(({ id, payload }) => ({ id, payload })); const checkpointProcess = checkpointRecord.process && typeof checkpointRecord.process === "object" && @@ -746,8 +763,40 @@ export async function claimNativeRestartRecoveries(input: { (identity) => identity.pid !== row.run.processPid, ), }); + // Indexed owners are current authority, independent of transcript rows. + // Admission checks their presence without loading historical launch facts. + let indexedOwnerEvidence = false; + let unretiredIndexedProcessOwner = false; + if (indexedAuthority) { + const authority = indexedAuthority; + if (authority.successorRunId !== null) throw new Error("native_process_owner_authority_superseded"); + if (authority.stateSha256 !== createHash("sha256").update(authority.state).digest("hex")) throw new Error("native_process_owner_authority_corrupt"); + const currentState = JSON.parse(authority.state); + const currentIdentity = currentState.identity; + if (currentIdentity?.normalizedSessionId !== row.run.nativeSessionId || currentIdentity?.runId !== row.run.id) throw new Error("native_process_owner_authority_binding_invalid"); + if (currentState.indexedState?.processOwnerIndexVersion !== 1 || typeof currentState.indexedState?.providerEverStarted !== "boolean") throw new Error("native_process_owner_index_not_ready"); + indexedOwnerEvidence = currentState.indexedState.providerEverStarted; + // A surviving runner retains ownership; reattachment only restores + // its authenticated transport. Replacement requires retirement of + // every earlier launch, including children which escaped a process + // group. A missing PID cannot establish that proof. Any unresolved + // owner therefore holds replacement; an indexed existence query is + // sufficient regardless of the number of earlier launches. Remote + // reattachment separately verifies the original sandbox and runner. + if (!exactRunnerIdentity && !remoteSandbox) { + const unresolved = await tx.select({ workId: nativeAuthorityWork.workId }) + .from(nativeAuthorityWork).where(and( + eq(nativeAuthorityWork.companyId, row.run.companyId), + eq(nativeAuthorityWork.issueId, row.coordinator.issueId), + eq(nativeAuthorityWork.normalizedSessionId, authority.normalizedSessionId), + eq(nativeAuthorityWork.collection, "process-owner"), + )).limit(1); + unretiredIndexedProcessOwner = unresolved.length > 0; + indexedOwnerEvidence ||= unretiredIndexedProcessOwner; + } + } const hasProviderEvidence = - hasCheckpointProviderIdentity || providerEvents.length > 0; + hasCheckpointProviderIdentity || providerEvents.length > 0 || indexedOwnerEvidence; const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox, @@ -757,6 +806,7 @@ export async function claimNativeRestartRecoveries(input: { knownProviderProcessAlive: providerProcesses.livePids.length > 0, knownProviderProcessIdentityAmbiguous: providerProcesses.ambiguousLivePids.length > 0, + unretiredIndexedProcessOwner, hasCheckpoint, checkpointIdentityMatches: checkpointIdentityMatches && @@ -888,7 +938,7 @@ export async function claimNativeRestartRecoveries(input: { return { kind: "blocked", runId: row.run.id, reason } as const; } - const generation = row.coordinator.controllerGeneration + 1; + const generation = nextNativeControllerGeneration(row.coordinator.controllerGeneration); const leaseOwner = `${controller.bootId}:${generation}:${randomUUID()}`; const event = historyEntry({ now, diff --git a/server/src/services/native-runtime/native-run-coordinator-store.ts b/server/src/services/native-runtime/native-run-coordinator-store.ts index ad9f772023c..f77309a6b94 100644 --- a/server/src/services/native-runtime/native-run-coordinator-store.ts +++ b/server/src/services/native-runtime/native-run-coordinator-store.ts @@ -1,3 +1,4 @@ +import { readRunEventLane, runEventLane, runEventCursor, runEventSettlementLane } from "../run-event-history.js"; import { createHash } from "node:crypto"; import { and, desc, eq, inArray, lt } from "drizzle-orm"; @@ -9,6 +10,7 @@ import { issues, nativeRunFinalizations, nativeRunResults, + nativeOutputBodyChunks, } from "@paperclipai/db"; import { NativeSessionProtocolIntegrityError, @@ -18,6 +20,7 @@ import { validatePrpEvent, validatePrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; +import { appendHeartbeatRunEventInTransaction, HeartbeatRunEventConflictError } from "../heartbeat-run-events.js"; export interface NativeRunStoreBinding { readonly companyId: string; @@ -160,20 +163,7 @@ export class NativeRunCoordinatorStore { } async readProviderSessionId(): Promise { - const rows = await this.#db - .select({ payload: heartbeatRunEvents.payload }) - .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.runId, this.#binding.runId), - eq(heartbeatRunEvents.sourceInstanceId, this.#binding.runnerSourceInstanceId), - inArray(heartbeatRunEvents.eventType, [ - "session.started", - "session.resumed", - "session.reconciled", - ]), - )) - .orderBy(desc(heartbeatRunEvents.sourceSeq)) - .limit(3); + const rows = await readRunEventLane(this.#db, this.#binding.runId, runEventLane("provider-session", this.#binding.runnerSourceInstanceId), 3); for (const row of rows) { const event = (row.payload as Record | null)?.prpEvent; const payload = typeof event === "object" && event !== null && !Array.isArray(event) @@ -198,9 +188,18 @@ export class NativeRunCoordinatorStore { async appendEvent(value: PrpEvent): Promise<{ readonly disposition: "committed" | "duplicate"; - readonly cursor: number; + readonly cursor: number | string; readonly highestContiguousSourceSeq: number; + readonly highestContiguousSourceEpoch?: string; }> { + return this.#db.transaction((tx) => this.appendEventInTransaction(tx, value)); + } + + /** Used by the authority store: event, receipt and delivery cursor have one commit. */ + async appendEventInTransaction( + tx: Parameters[0]>[0], + value: PrpEvent, + ): Promise<{ disposition: "committed" | "duplicate"; cursor: number | string; highestContiguousSourceSeq: number; highestContiguousSourceEpoch?: string }> { const validated = validatePrpEvent(value); if (!validated.ok) throw new Error("native_event_schema_invalid"); const event = validated.event; @@ -215,7 +214,6 @@ export class NativeRunCoordinatorStore { const canonicalPayload = event as unknown as Record; const payloadDigest = sha256(canonicalPayload); - return this.#db.transaction(async (tx) => { const [run] = await tx .select() .from(heartbeatRuns) @@ -246,80 +244,35 @@ export class NativeRunCoordinatorStore { ), ) .limit(1); - if (existing) { - if ( - existing.sourcePayloadSha256 !== payloadDigest || - existing.sourceInstanceId !== event.sourceInstanceId || - existing.sourceSeq !== event.sourceSeq - ) { - throw new NativeSessionProtocolIntegrityError( - "source_event_replay_conflict", - ); - } - const [latest] = await tx - .select({ sourceSeq: heartbeatRunEvents.sourceSeq }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, this.#binding.runId), - eq(heartbeatRunEvents.sourceInstanceId, event.sourceInstanceId), - ), - ) - .orderBy(desc(heartbeatRunEvents.sourceSeq)) - .limit(1); - return { - disposition: "duplicate" as const, - cursor: existing.seq, - highestContiguousSourceSeq: latest?.sourceSeq ?? event.sourceSeq, - }; + if (existing && (existing.sourcePayloadSha256 !== payloadDigest || + existing.sourceInstanceId !== event.sourceInstanceId || existing.sourceSeq !== event.sourceSeq || + existing.sourceEpoch !== (event.sourceEpoch ?? ""))) { + throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); } - - const [previous] = await tx - .select({ sourceSeq: heartbeatRunEvents.sourceSeq }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, this.#binding.runId), - eq(heartbeatRunEvents.sourceInstanceId, event.sourceInstanceId), - ), - ) - .orderBy(desc(heartbeatRunEvents.sourceSeq)) - .limit(1); - const expectedSourceSeq = (previous?.sourceSeq ?? 0) + 1; - if (event.sourceSeq !== expectedSourceSeq) { - throw new Error("native_event_source_gap"); + if (!existing && !event.sourceEpochTransition) { + const [previous] = await tx.select({ sourceSeq: heartbeatRunEvents.sourceSeq }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, this.#binding.runId), eq(heartbeatRunEvents.sourceInstanceId, event.sourceInstanceId), + eq(heartbeatRunEvents.sourceEpoch, event.sourceEpoch ?? ""))) + .orderBy(desc(heartbeatRunEvents.sourceSeq)).limit(1); + if (event.sourceSeq !== (previous?.sourceSeq ?? 0) + 1) throw new Error("native_event_source_gap"); + } + try { + const receipt = await appendHeartbeatRunEventInTransaction(tx, { + companyId: this.#binding.companyId, runId: this.#binding.runId, agentId: this.#binding.agentId, + eventType: event.eventType, stream: "system", level: event.eventType.includes("failed") ? "error" : "info", + payload: { prpEvent: canonicalPayload }, nativeSource: { + sourceInstanceId: event.sourceInstanceId, sourceEventId: event.sourceEventId, sourceSeq: event.sourceSeq, + sourceEpoch: event.sourceEpoch, sourceEpochTransition: event.sourceEpochTransition, + protocolSchemaVersion: event.schemaVersion, canonicalPayload, hashEncoding: "sha256", + }, + }); + return { disposition: receipt.disposition, cursor: receipt.row.eventEpoch ? runEventCursor(receipt.row) : receipt.row.seq, + highestContiguousSourceSeq: receipt.highestContiguousSourceSeq, + ...(receipt.highestContiguousSourceEpoch ? { highestContiguousSourceEpoch: receipt.highestContiguousSourceEpoch } : {}) }; + } catch (error) { + if (error instanceof HeartbeatRunEventConflictError) throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); + throw error; } - - const cursor = run.nextEventSeq; - const [inserted] = await tx - .insert(heartbeatRunEvents) - .values({ - companyId: this.#binding.companyId, - runId: this.#binding.runId, - agentId: this.#binding.agentId, - seq: cursor, - eventType: event.eventType, - stream: "system", - level: event.eventType.includes("failed") ? "error" : "info", - payload: { prpEvent: canonicalPayload }, - sourceInstanceId: event.sourceInstanceId, - sourceEventId: event.sourceEventId, - sourceSeq: event.sourceSeq, - sourcePayloadSha256: payloadDigest, - protocolSchemaVersion: event.schemaVersion, - }) - .returning({ seq: heartbeatRunEvents.seq }); - if (!inserted) throw new Error("native_event_not_persisted"); - await tx - .update(heartbeatRuns) - .set({ nextEventSeq: cursor + 1, updatedAt: new Date() }) - .where(eq(heartbeatRuns.id, this.#binding.runId)); - return { - disposition: "committed" as const, - cursor: inserted.seq, - highestContiguousSourceSeq: event.sourceSeq, - }; - }); } async completeRun(input: CompleteNativeRunInput): Promise<{ @@ -444,22 +397,14 @@ export class NativeRunCoordinatorStore { } const terminal = event.payload as PrpTerminalState; assertTerminal(terminal); - const [proposed] = await this.#db - .select({ payload: heartbeatRunEvents.payload }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, this.#binding.runId), - eq( - heartbeatRunEvents.sourceInstanceId, - this.#binding.runnerSourceInstanceId, - ), - eq(heartbeatRunEvents.eventType, "run.result.proposed"), - lt(heartbeatRunEvents.sourceSeq, event.sourceSeq), - ), - ) - .orderBy(desc(heartbeatRunEvents.sourceSeq)) - .limit(1); + const [terminalRow] = await this.#db.select().from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.runId, this.#binding.runId), eq(heartbeatRunEvents.sourceEventId, event.sourceEventId), + eq(heartbeatRunEvents.sourceInstanceId, this.#binding.runnerSourceInstanceId), + )).limit(1); + if (!terminalRow || terminalRow.sourcePayloadSha256 !== sha256(event)) throw new Error("native_terminal_not_committed"); + const [proposed] = await readRunEventLane(this.#db, this.#binding.runId, + runEventSettlementLane(event.sourceInstanceId, event.turnId), 1, terminalRow.id); + if (proposed?.eventType !== "run.result.proposed") throw new Error("native_terminal_result_missing"); const proposedEnvelope = ( proposed?.payload as Record | undefined )?.prpEvent as Record | undefined; diff --git a/server/src/services/native-runtime/native-runner-file-handoff.test.ts b/server/src/services/native-runtime/native-runner-file-handoff.test.ts index 367671dd2e3..4bd45144b52 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.test.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.test.ts @@ -15,7 +15,7 @@ import { import { tmpdir } from "node:os"; import path from "node:path"; -import { eq, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { @@ -29,6 +29,7 @@ import { issueComments, issues, issueWorkProducts, + nativeToolReceipts, } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -42,6 +43,7 @@ import { stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { insertNativeToolReceipt, readNativeToolReceipt } from "./native-tool-receipts.js"; describe("native runner file handoff", () => { let temporary: Awaited< @@ -992,16 +994,13 @@ describe("native runner file handoff", () => { CREATE OR REPLACE FUNCTION paperclip_test_fail_native_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - IF NEW.result_json IS DISTINCT FROM OLD.result_json THEN - RAISE EXCEPTION 'forced_native_receipt_failure'; - END IF; - RETURN NEW; + RAISE EXCEPTION 'forced_native_receipt_failure'; END; $$ `); await db.execute(sql` CREATE TRIGGER paperclip_test_fail_native_receipt - BEFORE UPDATE ON heartbeat_runs + BEFORE INSERT ON native_tool_receipts FOR EACH ROW EXECUTE FUNCTION paperclip_test_fail_native_receipt() `); try { @@ -1028,7 +1027,7 @@ describe("native runner file handoff", () => { } expect(receiptFailure).toMatchObject({ message: expect.stringContaining( - 'Failed query: update "heartbeat_runs"', + 'Failed query: insert into "native_tool_receipts"', ), cause: { message: expect.stringContaining("forced_native_receipt_failure"), @@ -1036,7 +1035,7 @@ describe("native runner file handoff", () => { }); } finally { await db.execute( - sql`DROP TRIGGER IF EXISTS paperclip_test_fail_native_receipt ON heartbeat_runs`, + sql`DROP TRIGGER IF EXISTS paperclip_test_fail_native_receipt ON native_tool_receipts`, ); await db.execute( sql`DROP FUNCTION IF EXISTS paperclip_test_fail_native_receipt()`, @@ -1133,24 +1132,16 @@ describe("native runner file handoff", () => { preparationState: "prepared", providerDeliveryConfirmed: false, }); - const [persisted] = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)); - const result = structuredClone(persisted.resultJson) as Record< - string, - unknown - >; - const receipts = result.semanticToolReceipts as Record< - string, - { result: Record } - >; - expect(receipts[call.arguments.idempotencyKey]?.result).toEqual(first); - delete receipts[call.arguments.idempotencyKey]!.result.fileDelivery; - await db - .update(heartbeatRuns) - .set({ resultJson: result }) - .where(eq(heartbeatRuns.id, runId)); + const receiptBinding = { companyId, issueId, runId }; + const persistedReceipt = await readNativeToolReceipt(db, receiptBinding, call.arguments.idempotencyKey); + expect(persistedReceipt?.result).toEqual(first); + const legacyReceipt = structuredClone(persistedReceipt!); + delete (legacyReceipt.result as Record).fileDelivery; + await db.update(nativeToolReceipts).set({ receipt: legacyReceipt }).where(and( + eq(nativeToolReceipts.companyId, companyId), + eq(nativeToolReceipts.runId, runId), + eq(nativeToolReceipts.idempotencyKey, call.arguments.idempotencyKey), + )); const replay = await authority().execute({ ...call, callId: `${call.callId}-replay`, @@ -1161,11 +1152,7 @@ describe("native runner file handoff", () => { }, }); expect(replay).toEqual(first); - const [afterReplay] = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)); - expect(afterReplay.resultJson).toEqual(result); + expect(await readNativeToolReceipt(db, receiptBinding, call.arguments.idempotencyKey)).toEqual(legacyReceipt); } } finally { await db @@ -1224,25 +1211,32 @@ describe("native runner file handoff", () => { const result = await authority().execute(callFor(`${key}.txt`, body, key)) as { entityRefs: string[] }; const [attachment] = await db.select().from(issueAttachments).where(eq(issueAttachments.id, result.entityRefs[0])); const [asset] = await db.select().from(assets).where(eq(assets.id, attachment.assetId)); - const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const receiptBinding = { companyId, issueId, runId }; + const originalReceipt = await readNativeToolReceipt(db, receiptBinding, key); + expect(originalReceipt).not.toBeNull(); if (mutation === "filename") await db.update(assets).set({ originalFilename: "different.txt" }).where(eq(assets.id, asset.id)); if (mutation === "size") await db.update(assets).set({ byteSize: asset.byteSize + 1 }).where(eq(assets.id, asset.id)); if (mutation === "hash") await db.update(assets).set({ sha256: "0".repeat(64) }).where(eq(assets.id, asset.id)); if (mutation === "origin") await db.update(issueAttachments).set({ originatingRunId: null }).where(eq(issueAttachments.id, attachment.id)); - if (mutation === "missing receipt" || mutation === "wrong operation") { - const changed = structuredClone(run.resultJson!); - const receipts = changed.semanticToolReceipts as Record; - if (mutation === "missing receipt") delete receipts[key]; - else receipts[key]!.operationId = "report_progress"; - await db.update(heartbeatRuns).set({ resultJson: changed }).where(eq(heartbeatRuns.id, runId)); - } + if (mutation === "missing receipt") await db.delete(nativeToolReceipts).where(and( + eq(nativeToolReceipts.companyId, companyId), eq(nativeToolReceipts.runId, runId), + eq(nativeToolReceipts.idempotencyKey, key), + )); + if (mutation === "wrong operation") await db.update(nativeToolReceipts) + .set({ receipt: { ...originalReceipt!, operationId: "report_progress" } }) + .where(and(eq(nativeToolReceipts.companyId, companyId), eq(nativeToolReceipts.runId, runId), + eq(nativeToolReceipts.idempotencyKey, key))); try { await expect(nativeCompletionFeedback(db, runId, doneReport([`deliverable:${attachment.id}`]))) - .rejects.toThrow(/current run|this run/); + .rejects.toThrow(/current run|this run|native_tool_receipt_reference_mismatch/); } finally { await db.update(assets).set({ originalFilename: asset.originalFilename, byteSize: asset.byteSize, sha256: asset.sha256 }).where(eq(assets.id, asset.id)); await db.update(issueAttachments).set({ originatingRunId: attachment.originatingRunId }).where(eq(issueAttachments.id, attachment.id)); - await db.update(heartbeatRuns).set({ resultJson: run.resultJson }).where(eq(heartbeatRuns.id, runId)); + if (mutation === "missing receipt") await insertNativeToolReceipt(db, receiptBinding, key, originalReceipt!); + else if (mutation === "wrong operation") await db.update(nativeToolReceipts) + .set({ receipt: originalReceipt! }) + .where(and(eq(nativeToolReceipts.companyId, companyId), eq(nativeToolReceipts.runId, runId), + eq(nativeToolReceipts.idempotencyKey, key))); } }); }); diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 820661f8893..24f4df7fe11 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1,5 +1,6 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "./cargo-test-binary.js"; import { hasNativeLocalProcessStop } from "../native-local-process-stop.js"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import { mkdtemp, readFile, rename, rm } from "node:fs/promises"; @@ -7,7 +8,7 @@ import { createServer, type Server } from "node:http"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { agents, companies, @@ -17,12 +18,13 @@ import { issues, nativeRunFinalizations, nativeRunResults, + nativeSessionAuthorities, } from "@paperclipai/db"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createRunnerdCodexTransport, - defaultCapabilityRunnerdBinary, + defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary, } from "../../vendor/paperclip-runner/index.js"; import { getEmbeddedPostgresTestSupport, @@ -55,12 +57,20 @@ const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; -const fakeCodexAppServer = resolve( - import.meta.dirname, - "../../../../packages/paperclip-runner/runner/target/debug/fake-codex-app-server", +const runnerWorkspace = resolve(import.meta.dirname, "../../../../packages/paperclip-runner/runner"); +const runnerBinary = () => resolveRunnerCargoTestBinaryOrDefault( + runnerWorkspace, + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); +const fakeCodexAppServer = resolveRunnerCargoTestBinary( + runnerWorkspace, + "debug", + "fake-codex-app-server", ); const binariesAvailable = - existsSync(defaultCapabilityRunnerdBinary()) && existsSync(fakeCodexAppServer); + existsSync(runnerBinary()) && existsSync(fakeCodexAppServer); const realProcessIt = binariesAvailable ? it : it.skip; async function closeServer(server: Server | null): Promise { @@ -244,7 +254,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( stateDirectory: string, ) { return { - runnerBinary: defaultCapabilityRunnerdBinary(), + runnerBinary: runnerBinary(), codexCommand: fakeCodexAppServer, codexArgs: [ "--state-file", @@ -885,6 +895,49 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( ]); }); + it("reattaches an exact surviving runner without reading historical process owners", async () => { + const fixture = await seedRun("INDEXED-LIVE-OWNERS"); + const stateDirectory = resolve(runtimeRoot, fixture.runId); + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", stateDirectory], { + detached: process.platform !== "win32", stdio: "ignore", + }); + child.unref(); + const pid = child.pid; + if (!pid) throw new Error("Failed to launch indexed runner sentinel"); + try { + await waitForCondition("indexed runner sentinel startup", () => processAlive(pid)); + const startedAt = await readProcessStartedAt(pid); + if (!startedAt) throw new Error("Sentinel fingerprint unavailable"); + await fixture.db.update(heartbeatRuns).set({ + processPid: pid, processGroupId: process.platform === "win32" ? null : pid, + processStartedAt: new Date(startedAt), + }).where(eq(heartbeatRuns.id, fixture.runId)); + const state = JSON.stringify({ + identity: { runId: fixture.runId, normalizedSessionId: fixture.native.normalizedSessionId }, + indexedState: { processOwnerIndexVersion: 1, providerEverStarted: true }, + }); + await fixture.db.insert(nativeSessionAuthorities).values({ + companyId, issueId: fixture.issueId, runId: fixture.runId, + normalizedSessionId: fixture.native.normalizedSessionId, + binding: JSON.stringify({ companyId, issueId: fixture.issueId, normalizedSessionId: fixture.native.normalizedSessionId }), + generation: "1", state, stateSha256: createHash("sha256").update(state).digest("hex"), + }); + // Make the entire historical owner relation unavailable. A continuation + // that consults it hits the production one-second lock timeout; a claim + // for the exact live runner needs only its bounded current authority. + await fixture.db.transaction(async blocker => { + await blocker.execute(sql`lock table native_authority_work in access exclusive mode`); + const [claim] = await claimNativeRestartRecoveries({ + db: fixture.db, controller: successor, restartKind: "hard", runIds: [fixture.runId], + }); + expect(claim).toMatchObject({ kind: "reattach_existing_runner", runId: fixture.runId, process: { pid } }); + expect(processAlive(pid)).toBe(true); + }); + } finally { + await stopOwnedProcessGroup(pid, stateDirectory); + } + }); + it("fails closed for a live recycled PID without signalling or spawning", async () => { const fixture = await seedRun("MISMATCH"); const stateDirectory = resolve(runtimeRoot, fixture.runId); @@ -945,8 +998,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( } }); - it("allows only one of two concurrent successor controllers to claim a dead session", async () => { - const fixture = await seedRun("CONCURRENT"); + it.each([4, 2_147_483_646, 2_147_483_647])("allows only one of two concurrent successor controllers to claim a dead session at generation %i", async (controllerGeneration) => { + const fixture = await seedRun(`CONCURRENT-${controllerGeneration}`); const now = new Date(); await fixture.db .update(heartbeatRuns) @@ -978,7 +1031,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( controllerBootId: "dead-controller-boot", controllerPid: 2_000_000_002, controllerProcessStartedAt: new Date("2026-09-04T10:00:00.000Z"), - controllerGeneration: 4, + controllerGeneration, }) .where(eq(nativeRunFinalizations.runId, fixture.runId)); @@ -1014,7 +1067,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, fixture.runId)), ).resolves.toEqual([ - { controllerGeneration: 5, providerAttempt: 0 }, + { controllerGeneration: Math.min(controllerGeneration + 1, 2_147_483_647), providerAttempt: 0 }, ]); }); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fa21d302bd2..ab3356d41d4 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,3 +1,4 @@ +import { loopbackCheckpointRunner, loopbackCheckpointDuplexRunner } from "./native-checkpoint-transfer.test-support.js"; import * as nativeJournalProof from "./native-journal-projection-async.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -1377,7 +1378,7 @@ describe("verified native harness backups", () => { join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state", ); - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, runnerInstanceId: "runner-1", @@ -1392,7 +1393,7 @@ describe("verified native harness backups", () => { await writeFile(join(current, "manifest.json"), JSON.stringify(manifest)); expect( - verifyNativeHarnessBackup({ + await verifyNativeHarnessBackup({ root, execution: backupExecution, runnerInstanceId: "runner-1", @@ -1417,7 +1418,7 @@ describe("verified native harness backups", () => { }, } as NativeExecutionInputV1; expect( - verifyNativeHarnessBackup({ + await verifyNativeHarnessBackup({ root, execution: continuationExecution, runnerInstanceId: "runner-1", @@ -1429,7 +1430,7 @@ describe("verified native harness backups", () => { "corrupt", ); expect( - verifyNativeHarnessBackup({ + await verifyNativeHarnessBackup({ root, execution: backupExecution, runnerInstanceId: "runner-1", @@ -1452,7 +1453,7 @@ describe("verified native harness backups", () => { join(current, "runner", "runner-state.json"), "runner-state", ); - expect(() => + await expect( buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, @@ -1464,7 +1465,7 @@ describe("verified native harness backups", () => { }, sourceProviderLeaseId: "sandbox-1", }), - ).toThrow("runner_harness_state_mismatch"); + ).rejects.toThrow("runner_harness_state_mismatch"); } finally { await rm(root, { recursive: true, force: true }); } @@ -1491,7 +1492,7 @@ describe("verified native harness backups", () => { join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state", ); - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, runnerInstanceId: "runner-1", @@ -1513,8 +1514,8 @@ describe("verified native harness backups", () => { completedAt: manifest.completedAt, }); - expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(true); - expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-2")).toBe(false); + expect(await verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(true); + expect(await verifyNativeHarnessBackupStamp(stamp, "sandbox-2")).toBe(false); const reboundStamp = createNativeHarnessBackupStamp({ manifestPath, sessionScopeId, @@ -1523,11 +1524,11 @@ describe("verified native harness backups", () => { runnerInstanceId: "runner-1", completedAt: manifest.completedAt, }); - expect(verifyNativeHarnessBackupStamp(reboundStamp, "sandbox-2")).toBe( + expect(await verifyNativeHarnessBackupStamp(reboundStamp, "sandbox-2")).toBe( true, ); await writeFile(join(current, "runner", "runner-state.json"), "corrupt"); - expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(false); + expect(await verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(false); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; @@ -1563,7 +1564,7 @@ describe("verified native harness backups", () => { join(legacyRoot, "codex-home", "sessions", "thread.jsonl"), "thread-state", ); - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: legacyRoot, execution: backupExecution, runnerInstanceId: "runner-1", @@ -1578,7 +1579,7 @@ describe("verified native harness backups", () => { await writeFile(join(legacyRoot, "manifest.json"), manifestBytes); expect( - verifyNativeHarnessBackupStamp( + await verifyNativeHarnessBackupStamp( { schema: "paperclip.native-harness-backup-stamp.v1", normalizedSessionId: "native-session", @@ -1804,6 +1805,37 @@ describe("split durable provider checkpoint identity", () => { }); describe("remote provider checkpoint snapshots", () => { + it("uses bounded checkpoint chunks instead of capped provider bulk sync when a command channel exists", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-checkpoint-duplex-")); + const sourcePath = join(root, "source"), targetPath = join(root, "checkpoint"), restored = join(root, "restored"); + const runner = loopbackCheckpointDuplexRunner(); + const syncIn = vi.fn().mockRejectedValue(new Error("provider bulk archive limit")); + const syncOut = vi.fn().mockRejectedValue(new Error("provider bulk archive limit")); + runner.syncIn = syncIn; runner.syncOut = syncOut; + const execute = vi.spyOn(runner, "execute"); + const channel = vi.spyOn(runner, "openDuplexChannel"); + try { + await mkdir(join(sourcePath, "sessions"), { recursive: true }); + await mkdir(join(sourcePath, "tmp")); + await writeFile(join(sourcePath, "sessions", "history.jsonl"), "retained history\n".repeat(100_000)); + await writeFile(join(sourcePath, "auth.json"), "private credential"); + await symlink("/etc/passwd", join(sourcePath, "tmp", "scratch-link")); + await syncRemoteRunnerDirectoryOut({ runner, sourcePath, targetPath, mode: 0o700, excludeEntries: ["tmp", "auth.json"] }); + await stageRemoteRunnerDirectory({ target: { kind: "remote" } as never, runner, sourcePath: targetPath, targetPath: restored, mode: 0o700 }); + expect(await readFile(join(restored, "sessions", "history.jsonl"))).toEqual(await readFile(join(sourcePath, "sessions", "history.jsonl"))); + await expect(access(join(restored, "auth.json"))).rejects.toThrow(); + await expect(access(join(restored, "tmp"))).rejects.toThrow(); + await expect(lstat(join(sourcePath, "tmp", "scratch-link"))).resolves.toMatchObject({}); + expect(syncIn).not.toHaveBeenCalled(); expect(syncOut).not.toHaveBeenCalled(); + expect(channel).toHaveBeenCalled(); + expect(execute.mock.calls.every(([input]) => input.timeoutMs! > 0)).toBe(true); + // A corrupt/unsafe subsequent snapshot cannot replace the good checkpoint. + await symlink("/etc/passwd", join(sourcePath, "unsafe-link")); + await expect(syncRemoteRunnerDirectoryOut({ runner, sourcePath, targetPath, mode: 0o700, excludeEntries: ["tmp", "auth.json"] })).rejects.toThrow("archive_unsafe_entry"); + expect(await readFile(join(targetPath, "sessions", "history.jsonl"))).toEqual(await readFile(join(restored, "sessions", "history.jsonl"))); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("excludes Codex scratch and credential state without mutating the live provider home", async () => { const execute = vi .fn() @@ -1938,30 +1970,12 @@ describe("remote provider checkpoint snapshots", () => { await mkdir(targetPath, { recursive: true }); await writeFile(join(targetPath, "preserved.txt"), "preserved"); await symlink("/etc/passwd", join(archiveSource, "host-secret")); - const archive = execFileSync( - "tar", - ["-czf", "-", "-C", archiveSource, "."], - { maxBuffer: 8 * 1024 * 1024 }, - ); - const execute = vi - .fn() - .mockResolvedValueOnce({ - exitCode: 0, - timedOut: false, - stdout: "", - stderr: "", - }) - .mockResolvedValueOnce({ - exitCode: 0, - timedOut: false, - stdout: archive.toString("base64"), - stderr: "", - }); + await expect( syncRemoteRunnerDirectoryOut({ - runner: { execute } as never, - sourcePath: "/remote/codex-home", + runner: loopbackCheckpointRunner(), + sourcePath: archiveSource, targetPath, mode: 0o700, }), @@ -2815,6 +2829,7 @@ describe("retained native cleanup activation", () => { resultId: "result", assessmentId: "assessment", decisionId: "decision", + controllerGeneration: 0, leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: null, @@ -3670,7 +3685,6 @@ describe("retained native cleanup activation", () => { "canonical_prepared_bad_hash", "canonical_prepared_bad_inode", "home_symlink", - "home_oversized", "legacy_busy_copy", "legacy_bad_proof", "legacy_extra_attempt", @@ -3688,6 +3702,7 @@ describe("retained native cleanup activation", () => { "wrong_provider_account", ].includes(mode); const succeeds = [ + "home_oversized", "home_paginated", "canonical_source", "canonical_claim_commit_stalled", @@ -3966,7 +3981,7 @@ describe("retained native cleanup activation", () => { describe("stopped native conversation physical cleanup", () => { it.each(["codex", "acpx"].flatMap(provider => [ - "stopped", "provider_alive", "worker_alive", "missing_receipt", "wrong_receipt", "new_launch", + "stopped", "indexed_authority", "provider_alive", "worker_alive", "missing_receipt", "wrong_receipt", "new_launch", "foreign_run", "foreign_company", "foreign_runner", "remote", "unreleased", "changed_state", "changed_pid", "symlink", "startup_intent", "pending_identity", "wrong_schema", "replacement", "replacement_alive", "agent_alive", "checkpoint_owner_alive", "diagnostic_owner_alive", "normalized_session_receipt", ].map(mode => ({ provider, mode }))))("$provider $mode", async ({ provider, mode }) => { const base = await mkdtemp(join(tmpdir(), "native-conversation-cleanup-")); @@ -4000,6 +4015,7 @@ describe("stopped native conversation physical cleanup", () => { if (mode === "diagnostic_owner_alive") providerEvents.push({ ...event, eventType: "harness.diagnostic", payload: { providerMethod: "acpx/process", role: "acp_agent", pid: process.pid } } as unknown as typeof event); const providerState = { schema: mode === "wrong_schema" ? "unknown" : `paperclip.runner.${provider}-provider-state.${provider === "codex" ? "v1" : "v3"}`, lifecycle: "turn_active", + ...(mode === "indexed_authority" ? { indexedGeneration: "1" } : {}), ...(mode === "startup_intent" ? { startupAttempt: { phase: "intent" } } : {}), ...(mode === "pending_identity" ? { pendingEvents: [{ eventType: "session.started" }] } : {}), }; @@ -4007,7 +4023,8 @@ describe("stopped native conversation physical cleanup", () => { const receipt = { sourceEventId: `${identity.runnerInstanceId}:${identity.runId}:1`, sourcePayloadSha256: mode === "wrong_receipt" ? "wrong" : hash(normalizedEvent), payload: { prpEvent: normalizedEvent } }; const stop = { eventType: mode === "new_launch" ? "native.process_start_requested" : "native.local_process_stopped", - payload: { processPid: mode === "worker_alive" ? process.pid : 99_999_999, processGroupId: mode === "worker_alive" ? process.pid : 99_999_999 } }; + companyId: input.binding.companyId, runId: input.binding.runId, seq: 1, + processPid: mode === "worker_alive" ? process.pid : 99_999_999, processGroupId: mode === "worker_alive" ? process.pid : 99_999_999 }; let queryIndex = 0; const db = { select() { const rows = [ [{ provider: mode === "remote" ? "daytona" : "local", releasedAt: mode === "unreleased" ? null : new Date() }], @@ -4046,6 +4063,8 @@ describe("stopped native conversation physical cleanup", () => { describe("explicit failed native retry physical evidence", () => { it.each([ "suspended", + "indexed_runner", + "indexed_provider", "distinct_account", "null_account", "wrong_account", @@ -4117,6 +4136,8 @@ describe("explicit failed native retry physical evidence", () => { "suspended", "distinct_account", "null_account", + "indexed_runner", + "indexed_provider", ].includes(kind); let restoreRead: (() => void) | undefined; let restoreKill: (() => void) | undefined; @@ -4143,6 +4164,7 @@ describe("explicit failed native retry physical evidence", () => { }, kind === "ready" ? "ready" : "suspended", ), + ...(kind === "indexed_runner" ? { indexedGeneration: "1" } : {}), outbox: kind === "unacknowledged_output" ? [{}] : [], }; if (kind === "symlink") { @@ -4157,6 +4179,7 @@ describe("explicit failed native retry physical evidence", () => { JSON.stringify({ schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", + ...(kind === "indexed_provider" ? { indexedGeneration: "1" } : {}), threadId: "exact-thread", providerSessionId: providerAccount, activeProviderTurnId: @@ -4268,7 +4291,7 @@ describe("explicit failed native retry physical evidence", () => { receipt, }; expect((await nativePreProviderRetryAfterCleanupStateIsSafe(input))).toBe( - retryable, + retryable && !kind.startsWith("indexed_"), ); expect( (await nativePreProviderRetryAfterCleanupStateIsSafe({ @@ -4668,6 +4691,7 @@ type LeaseCoordinator = { issueId: string; phase: string; attempt: number; + controllerGeneration: number; leaseOwner: string | null; leaseExpiresAt: Date | null; resultId: string | null; @@ -4687,6 +4711,7 @@ function leaseDb( issueId: boundExecution.binding.issueId, phase: "observed", attempt: 0, + controllerGeneration: 0, leaseOwner: null, leaseExpiresAt: null, resultId: null, @@ -4725,6 +4750,9 @@ function leaseDb( runnerProfileJson, runtimeMode: "native", status: runStatus, + eventEpoch: "", + seq: 1, + nextEventSeq: 1, }, ] : table === issues @@ -4752,7 +4780,9 @@ function leaseDb( const insert = (table: unknown) => ({ values: (values: Record) => { updates.push({ table, values }); - return { returning: async () => [values] }; + const query = { returning: async () => [values], onConflictDoUpdate: () => query, + then: Promise.resolve([values]).then.bind(Promise.resolve([values])) }; + return query; }, }); const tx = { @@ -4803,10 +4833,9 @@ function cancellationDb(options?: { statusVersion: 3, lastStatusDecisionId: null, }; - const coordinator = - options && "coordinator" in options - ? options.coordinator - : { runId: execution.binding.runId, assessmentId: null }; + const coordinator = options && "coordinator" in options + ? options.coordinator && { controllerGeneration: 0, ...options.coordinator } + : { runId: execution.binding.runId, assessmentId: null, controllerGeneration: 0 }; let forUpdateCount = 0; let resultJsonUpdateCount = 0; const updates: Array<{ table: unknown; values: Record }> = @@ -8708,7 +8737,7 @@ describe("runnerd provider runtime wiring", () => { join(backupRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, backupLifecycle)), ); - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot, execution: priorExecution, runnerInstanceId: identity.runnerInstanceId, @@ -8826,10 +8855,10 @@ describe("runnerd provider runtime wiring", () => { // Accumulated history must not invalidate an otherwise identical session. const core = new DurablePrpControlPlane(options); for (let index = 0; index < commandCount; index += 1) { - core.queueCommand("turn.start", { text: "x".repeat(768 * 1024) }, `large-command-${index}`); + (await core.queueCommand("turn.start", { text: "x".repeat(768 * 1024) }, `large-command-${index}`)); } for (const command of core.store.state.commands) command.status = "completed"; - core.issueBootstrapTicket(); + (await core.issueBootstrapTicket()); expect(new DurablePrpControlPlane(options).store.state.commands).toHaveLength(commandCount); const controlPath = join(root, "control-plane", "control-plane-state.json"); const controlBytes = await readFile(controlPath); @@ -9087,6 +9116,7 @@ describe("runnerd provider runtime wiring", () => { "quarantined", "empty retry shell", "unsuspended current", + "indexed authority", "large control journal", "live runner", "live group", @@ -9255,6 +9285,7 @@ describe("runnerd provider runtime wiring", () => { join(root, "runner", "runner-state.json"), JSON.stringify({ ...durableRunnerState(identity, "ready"), + ...(scenario === "indexed authority" ? { indexedGeneration: "1" } : {}), outbox: scenario === "unacknowledged events" ? [{ sourceSeq: 10 }] : [], pendingTerminalDelivery: null, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c075a218a02..82a35db1373 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,14 @@ +import { normalizedEventId } from "../../vendor/paperclip-runner/index.js"; +import { snapshotCleanupProviderHome, copyCleanupProviderHomeTree, findCleanupRollout, readCleanupRolloutHeader } from "./native-cleanup-tree.js"; +import { startNativeMaintenanceLeaseRenewal } from "./native-maintenance-lease.js"; +import { checkpointExclusions, checkpointTar, downloadCheckpointDirectory, uploadCheckpointArchive } from "./native-checkpoint-transfer.js"; +import { digestNativeHarnessBackupDirectory, readNativeHarnessBackupManifestBytes, syncNativeHarnessDirectory } from "./native-harness-tree.js"; +import { useIndexedNativeAuthority } from "./native-indexed-activation.js"; +import { inspectRemoteIndexedState } from "./native-indexed-inspection.js"; +import { createRemoteIndexedSessionCheckpoint } from "./native-indexed-checkpoint.js"; +import { readNativeIndexedCheckpointProof, type NativeCheckpointBindings } from "./native-indexed-checkpoint-proof.js"; +import { createNativePostgresAuthorityStore, readPostgresAuthority, type AuthorityTransaction } from "./postgres-authority-store.js"; +import { readIndexedAuthorityProof, withNativeAuthorityReadContext } from "./native-authority-read-context.js"; import { readNativeJournalProjection, scanNativeStateFile, nativeStateFileIncludes, nativeStateBase64Fingerprint } from "./native-journal-projection-async.js"; import { isSupportedRemoteCodexVersion, @@ -13,7 +24,7 @@ import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js"; import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; -import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; +import { inspectStoppedCodexRollout } from "./stopped-codex-rollout.js"; import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js"; import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; @@ -45,6 +56,7 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; +import { open as openAsync } from "node:fs/promises"; import { DatabaseSync } from "node:sqlite"; import { basename, dirname, join, posix, resolve } from "node:path"; import type { @@ -176,6 +188,7 @@ import { } from "./native-runner-ownership.js"; import { currentNativeControllerIdentity, + nextNativeControllerGeneration, nextNativeProviderAttempt, type NativeControllerIdentity, type NativeRestartRecoveryClaim, @@ -275,9 +288,6 @@ export async function detachNativeSessionsForRestart( } return { detachedRunIds, inactiveRunIds, unsupportedRunIds }; } -const MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES = 64 * 1024 * 1024; -const MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES = 64 * 1024 * 1024; -const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000; const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024; const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024; const CODEX_HOME_NON_PERSISTENT_ENTRIES = [ @@ -516,6 +526,11 @@ function readBoundedNativeFile( } } +async function readCurrentNativeFile(path: string, maximum: number, errorCode: string): Promise { + const indexed = await readIndexedAuthorityProof(path); + return indexed?.bytes ?? readBoundedNativeFile(path, maximum, errorCode); +} + const NATIVE_PROVIDER_HOST_ENV_KEYS = [ "PATH", "HOME", @@ -729,6 +744,8 @@ export async function materializeRuntimeQuestionFallback(input: { fallback: RuntimeQuestionFallback; interaction: { id: string }; } | null> { + return withNativeAuthorityReadContext(input.db, async () => { + const fallback = runtimeQuestionFallbackFromEvent(input.event); if (!fallback) return null; const interaction = await issueThreadInteractionService(input.db).create( @@ -744,6 +761,8 @@ export async function materializeRuntimeQuestionFallback(input: { }, ); return { fallback, interaction }; + + }); } export function runtimeInputLifecycleMetric( @@ -1174,6 +1193,8 @@ export async function synchronizeCompletedProviderPlan(input: { payload: Record; }; }): Promise { + return withNativeAuthorityReadContext(input.db, async () => { + if ( input.event.eventType !== "plan.updated" || input.event.payload.complete !== true @@ -1411,6 +1432,8 @@ export async function synchronizeCompletedProviderPlan(input: { currentRevisionId: revision.id, confirmationId, }; + + }); } class SessionToolAuthorityEpoch { @@ -1652,10 +1675,10 @@ async function migrateLegacyRunnerdStateRoot(input: { exactRun && legacyIdentity && ["absent", "indeterminate"].includes( - runnerdAuthorityLifecycle( + (await runnerdAuthorityLifecycle( input.legacy, legacyIdentity as RunnerdDurableIdentity, - ), + )), ) ) { quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate"); @@ -1689,10 +1712,10 @@ async function migrateLegacyRunnerdStateRoot(input: { return input.scoped; } -function runnerdAuthorityLifecycle( +async function runnerdAuthorityLifecycle( root: string, identity: RunnerdDurableIdentity, -): "absent" | "suspended" | "not_suspended" | "indeterminate" { +): Promise<"absent" | "suspended" | "not_suspended" | "indeterminate"> { const runnerRoot = resolve(root, "runner"); if (!existsSync(runnerRoot)) return "absent"; if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate"; @@ -1703,7 +1726,7 @@ function runnerdAuthorityLifecycle( // though runner state was owned by the sandbox. It carries no authority, // so a verified failover backup may be consulted. Any non-empty direct // directory remains indeterminate and therefore blocks fallback. - return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate"; + return nativeDirectoryIsEmpty(runnerRoot) ? "absent" : "indeterminate"; } catch { return "indeterminate"; } @@ -1711,11 +1734,11 @@ function runnerdAuthorityLifecycle( try { const state = record( JSON.parse( - readBoundedNativeFile( + (await readCurrentNativeFile( statePath, NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large", - ).toString("utf8"), + )).toString("utf8"), ), ); if ( @@ -1731,6 +1754,9 @@ function runnerdAuthorityLifecycle( state.runId === identity.runId && state.normalizedSessionId === identity.normalizedSessionId ) { + // Suspension is committed by the runner's existing provider shutdown + // protocol. Reading that checkpoint is different from inferring a new + // suspension from PID absence in recoverQuiescentRunnerdState below. if (state.lifecycle === "suspended") return "suspended"; return "not_suspended"; } @@ -1740,25 +1766,25 @@ function runnerdAuthorityLifecycle( } } -function runnerdAuthorityLifecycleWithVerifiedBackup(input: { +async function runnerdAuthorityLifecycleWithVerifiedBackup(input: { root: string; identity: RunnerdDurableIdentity; execution: NativeExecutionInput; allowVerifiedBackup: boolean; -}): "suspended" | "not_suspended" | "indeterminate" { - const direct = runnerdAuthorityLifecycle(input.root, input.identity); +}): Promise<"suspended" | "not_suspended" | "indeterminate"> { + const direct = (await runnerdAuthorityLifecycle(input.root, input.identity)); if (direct !== "absent") return direct; if (!input.allowVerifiedBackup) return "indeterminate"; - const backup = verifyNativeHarnessBackup({ + const backup = await verifyNativeHarnessBackup({ root: input.root, execution: input.execution, runnerInstanceId: input.identity.runnerInstanceId, }); if (!backup) return "indeterminate"; - const backupLifecycle = runnerdAuthorityLifecycle( + const backupLifecycle = (await runnerdAuthorityLifecycle( backup.root, input.identity, - ); + )); return backupLifecycle === "absent" ? "indeterminate" : backupLifecycle; } @@ -1797,11 +1823,17 @@ async function cleanupStateSnapshot(root: string, providerFile = "codex-provider const files = [...CLEANUP_CANONICAL_FILES.slice(0, 2), `runner/${providerFile}`]; const controlProof = (await readNativeJournalProjection(resolve(root, files[0]!))); for (const directory of directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity); - const bytes = files.slice(1).map((file) => - readBoundedNativeFile(resolve(root, file), NATIVE_RUNNER_STATE_MAX_BYTES, "native_cleanup_maintenance_unproven"), - ); + const bytes = await Promise.all(files.slice(1).map((file) => + readCurrentNativeFile(resolve(root, file), NATIVE_RUNNER_STATE_MAX_BYTES, "native_cleanup_maintenance_unproven"), + )); const control = record(controlProof.value); const [runner, provider] = bytes.map(value => record(JSON.parse(value.toString("utf8")))); + // Materialized current projections are read evidence, not permission to + // retire an indexed lifetime. PID/group absence cannot prove that escaped + // provider descendants have stopped. Require the indexed recovery admission. + if ([control, runner, provider].some(value => typeof value?.indexedGeneration === "string")) { + throw new Error("native_cleanup_indexed_process_fence_required"); + } const fileSha256 = [controlProof.sha256, ...bytes.map(value => createHash("sha256").update(value).digest("hex"))] as [string, string, string]; return { control: control!, @@ -1831,10 +1863,19 @@ function cleanupProcessAbsent(pid: unknown): pid is number { }); } +function nativeDirectoryIsEmpty(path: string): boolean { + const directory = opendirSync(path); + try { + return directory.readSync() === null; + } finally { + directory.closeSync(); + } +} + function cleanupCanonicalVacancy(root: string) { const stat = lstatSync(root, { throwIfNoEntry: false }); if (!stat) return null; - if (!isSafeNativeStateDirectory(root) || readdirSync(root).length !== 0) { + if (!isSafeNativeStateDirectory(root) || !nativeDirectoryIsEmpty(root)) { throw new Error("native_cleanup_maintenance_unproven"); } return { @@ -1940,7 +1981,7 @@ export function retainedNativeCleanupJournalMatches(input: { identityEvent.sourceKind !== "runner" || identityEvent.eventType !== identities[0]!.eventType || identityEvent.sourceEventId !== - `${run.runnerInstanceId}:${run.id}:${identityEvent.sourceSeq}` + normalizedEventId(run.runnerInstanceId!, run.id, Number(identityEvent.sourceSeq), typeof identityEvent.sourceEpoch === "string" ? identityEvent.sourceEpoch : undefined) ) return false; const boundDigest = validPersisted.some((row) => { @@ -2102,6 +2143,8 @@ export async function appendRetainedNativeCleanupEvent( event: PrpEvent; }, ): Promise { + return withNativeAuthorityReadContext(db, async () => { + const parsed = validatePrpEvent(input.event); if ( !parsed.ok || @@ -2139,139 +2182,17 @@ export async function appendRetainedNativeCleanupEvent( canonicalPayload: receipt, }, }); + + }); } -function cleanupProviderHomeSnapshot(home: string, content: boolean) { - const entries: Array<{ - path: string; - directory: boolean; - dev: number; - ino: number; - size: number; - mtimeMs: number; - ctimeMs: number; - sha256?: string; - }> = []; - let bytes = 0; - const visit = (relative: string, depth: number) => { - if (depth > 32 || entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES) - throw new Error("native_cleanup_maintenance_unproven"); - const path = resolve(home, relative); - const metadata = lstatSync(path); - if ( - metadata.isSymbolicLink() || - (!metadata.isFile() && !metadata.isDirectory()) - ) - throw new Error("native_cleanup_maintenance_unproven"); - const entry = { - path: relative, - directory: metadata.isDirectory(), - dev: metadata.dev, - ino: metadata.ino, - size: metadata.isDirectory() ? 0 : metadata.size, - mtimeMs: metadata.mtimeMs, - ctimeMs: metadata.ctimeMs, - }; - entries.push(entry); - if (entry.directory) { - const directory = opendirSync(path); - const children: string[] = []; - try { - for ( - let child = directory.readSync(); - child !== null; - child = directory.readSync() - ) { - if ( - !relative && - (CODEX_HOME_NON_PERSISTENT_ENTRIES as readonly string[]).includes( - child.name, - ) - ) - continue; - if (children.length + entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES) - throw new Error("native_cleanup_maintenance_unproven"); - children.push(child.name); - } - } finally { - directory.closeSync(); - } - for (const name of children.sort()) - visit(relative ? `${relative}/${name}` : name, depth + 1); - } else { - bytes += metadata.size; - if (bytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES) - throw new Error("native_cleanup_maintenance_unproven"); - if (content) { - const value = readBoundedNativeFile( - path, - metadata.size, - "native_cleanup_maintenance_unproven", - ); - (entry as (typeof entries)[number]).sha256 = createHash("sha256") - .update(value) - .digest("hex"); - } - } - const after = lstatSync(path); - if ( - after.dev !== metadata.dev || - after.ino !== metadata.ino || - after.mtimeMs !== metadata.mtimeMs || - after.ctimeMs !== metadata.ctimeMs || - after.size !== metadata.size - ) - throw new Error("native_cleanup_maintenance_unproven"); - }; - visit("", 0); - return { - entries, - bytes, - metadataFingerprint: nativeSha256( - entries.map(({ sha256: _sha, ...entry }) => entry), - ), - fingerprint: content - ? nativeSha256( - entries.map(({ path, directory, size, sha256 }) => ({ - path, - directory, - size, - ...(sha256 ? { sha256 } : {}), - })), - ) - : null, - }; -} +const cleanupProviderHomeSnapshot = (home: string, content: boolean) => + snapshotCleanupProviderHome(home, content, CODEX_HOME_NON_PERSISTENT_ENTRIES); -function copyCleanupProviderHome( - source: string, - destination: string, - snapshot: ReturnType, -) { - for (const entry of snapshot.entries) { - const target = resolve(destination, entry.path); - if (entry.directory) mkdirSync(target, { mode: 0o700 }); - else { - const value = readBoundedNativeFile( - resolve(source, entry.path), - entry.size, - "native_cleanup_maintenance_unproven", - ); - if (createHash("sha256").update(value).digest("hex") !== entry.sha256) - throw new Error("native_cleanup_maintenance_unproven"); - writeFileSync(target, value, { flag: "wx", mode: 0o600 }); - } - } - if ( - cleanupProviderHomeSnapshot(source, false).metadataFingerprint !== - snapshot.metadataFingerprint || - cleanupProviderHomeSnapshot(destination, true).fingerprint !== - snapshot.fingerprint - ) - throw new Error("native_cleanup_maintenance_unproven"); -} +const copyCleanupProviderHome = (source: string, destination: string, snapshot: Awaited>) => + copyCleanupProviderHomeTree(source, destination, snapshot, CODEX_HOME_NON_PERSISTENT_ENTRIES); -export function rebaseRetainedNativeCleanupProviderHome( +export async function rebaseRetainedNativeCleanupProviderHome( home: string, canonicalHome: string, threadId: string, @@ -2279,25 +2200,8 @@ export function rebaseRetainedNativeCleanupProviderHome( ) { if (resolve(home) === resolve(canonicalHome)) throw new Error("native_cleanup_maintenance_unproven"); - const snapshot = cleanupProviderHomeSnapshot(home, true); - const rollouts = snapshot.entries.filter( - (entry) => - !entry.directory && - entry.path.startsWith("sessions/") && - entry.path.endsWith(`-${threadId}.jsonl`), - ); - if (rollouts.length !== 1) - throw new Error("native_cleanup_maintenance_unproven"); - const rollout = rollouts[0]!; - const bytes = readBoundedNativeFile( - resolve(home, rollout.path), - rollout.size, - "native_cleanup_maintenance_unproven", - ); - const newline = bytes.indexOf(10); - if (newline < 0 || newline > 64 * 1024) - throw new Error("native_cleanup_maintenance_unproven"); - const first = record(JSON.parse(bytes.subarray(0, newline).toString("utf8"))); + const rollout = await findCleanupRollout(home, threadId, CODEX_HOME_NON_PERSISTENT_ENTRIES); + const first = record(JSON.parse((await readCleanupRolloutHeader(home, rollout)).toString("utf8"))); if (first.type !== "session_meta" || record(first.payload).id !== threadId) throw new Error("native_cleanup_maintenance_unproven"); // Only open the NEW private copy. Codex 0.153.4 deliberately does not fall @@ -2305,14 +2209,6 @@ export function rebaseRetainedNativeCleanupProviderHome( // immutable rollout, which can differ after thread/revert. Relocate only // that already-proven path, never choose another rollout or change its mode. const sqlite = resolve(home, "state_5.sqlite"); - if ( - snapshot.entries.some( - (entry) => - /^state_\d+\.sqlite$/.test(entry.path) && - entry.path !== "state_5.sqlite", - ) - ) - throw new Error("native_cleanup_maintenance_unproven"); if (!existsSync(sqlite)) return; const database = new DatabaseSync(sqlite); try { @@ -2398,6 +2294,8 @@ export async function verifyStoppedNativeSessionForContinuation( db: Db, run: typeof heartbeatRuns.$inferSelect, ): Promise<{ evidence: Record; retire: () => Promise } | null> { + return withNativeAuthorityReadContext(db, async () => { + try { if (run.runtimeMode !== "native" || !["failed", "cancelled", "interrupted", "timed_out"].includes(run.status) || !run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null; @@ -2449,14 +2347,20 @@ export async function verifyStoppedNativeSessionForContinuation( !cleanupProcessAbsent(provider.processId) || provider.processId === stopped.processPid) return null; if (event.eventType === "session.reconciled" && (!providerPids.has(Number(provider.previousProcessId)) || !cleanupProcessAbsent(provider.previousProcessId))) return null; - const receipt = receipts.find(row => row.sourceEventId === `${run.runnerInstanceId}:${run.id}:${event.sourceSeq}`); + const matchingReceipts = receipts.filter(row => { + const normalized = record(record(row.payload).prpEvent), payload = record(normalized.payload); + return normalized.eventType === event.eventType && (payload.driverSessionId ?? payload.providerSessionId) === provider.providerSessionId + && (payload.processId === undefined || payload.processId === provider.processId); + }); + const receipt = matchingReceipts.length === 1 ? matchingReceipts[0] : undefined; const durable = record(record(receipt?.payload).prpEvent); const durableProvider = record(durable.payload); if (!receipt || !validatePrpEvent(durable).ok || durable.sourceInstanceId !== event.sourceInstanceId || durable.runId !== run.id || durable.normalizedSessionId !== run.nativeSessionId || // Normalized session-open receipts omit turn/item IDs. Those belong // to the durable runner identity checked above, not the open event. - durable.sourceSeq !== event.sourceSeq || durable.eventType !== event.eventType || + receipt.sourceEventId !== normalizedEventId(run.runnerInstanceId!, run.id, Number(durable.sourceSeq), typeof durable.sourceEpoch === "string" ? durable.sourceEpoch : undefined) + || durable.eventType !== event.eventType || receipt.sourcePayloadSha256 !== nativeSha256(durable) || (durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) || (durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null; @@ -2497,6 +2401,8 @@ export async function verifyStoppedNativeSessionForContinuation( }, }; } catch { return null; } + + }); } /** Prove that a crashed local Codex turn ended without external effects. No provider @@ -2506,6 +2412,8 @@ export async function verifyStoppedNativeSessionForReplacement( db: Db, run: typeof heartbeatRuns.$inferSelect, ): Promise<{ evidence: Record; retire: () => Promise } | null> { + return withNativeAuthorityReadContext(db, async () => { + try { if (run.runtimeMode !== "native" || run.status !== "failed" || !run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null; @@ -2526,6 +2434,7 @@ export async function verifyStoppedNativeSessionForReplacement( if (!stopped) return null; const root = scopedRunnerdStateRoot(execution); const snapshot = (await cleanupStateSnapshot(root)); + if (record(snapshot.control.indexedState).externalEffectEverAdmitted === true) return null; const identity = record(snapshot.control.identity); if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId || !["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key => @@ -2554,7 +2463,7 @@ export async function verifyStoppedNativeSessionForReplacement( const durableProvider = record(durableEvent.payload); if (!receipt || !validatePrpEvent(durableEvent).ok || durableEvent.runId !== run.id || durableEvent.sourceInstanceId !== run.runnerInstanceId || durableEvent.normalizedSessionId !== run.nativeSessionId || - receipt.sourceEventId !== `${run.runnerInstanceId}:${run.id}:${durableEvent.sourceSeq}` || + receipt.sourceEventId !== normalizedEventId(run.runnerInstanceId!, run.id, Number(durableEvent.sourceSeq), typeof durableEvent.sourceEpoch === "string" ? durableEvent.sourceEpoch : undefined) || receipt.sourcePayloadSha256 !== nativeSha256(durableEvent) || (durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) || (durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null; @@ -2596,18 +2505,12 @@ export async function verifyStoppedNativeSessionForReplacement( events.some(event => ["mcp_app.tool_input", "runtime.input.requested", "runtime_request.created"].includes(String(event.eventType)))) return null; if (snapshot.control.commands.map(record).some(command => command.status === "pending" && !["turn.stop", "runner.drain", "runner.suspend"].includes(String(command.type)))) return null; - const home = cleanupProviderHomeSnapshot(resolve(root, "codex-home"), false); - const rollouts = home.entries.filter(entry => !entry.directory && entry.path.startsWith("sessions/") && - basename(entry.path).endsWith(`-${provider.providerSessionId}.jsonl`)); - if (rollouts.length !== 1 || rollouts[0]!.size > 32 * 1024 * 1024) return null; - const rolloutPath = resolve(root, "codex-home", rollouts[0]!.path); - const bytes = readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven"); - // A partial final write is not a closed transcript. - if (!bytes.toString("utf8").endsWith("\n")) return null; - const rows = bytes.toString("utf8").trimEnd().split("\n").map(line => JSON.parse(line)); - if (!stoppedCodexTurnIsTextOnly({ rows, threadId: provider.providerSessionId, turnId, cwd: execution.workspace.cwd, completedTaskControlCalls })) return null; - const rolloutSha256 = nativeSha256(bytes.toString("utf8")); - const evidence = { schema: "paperclip.stopped_text_turn.v1", runId: run.id, nativeSessionId: run.nativeSessionId, + const rollout = await findCleanupRollout(resolve(root, "codex-home"), provider.providerSessionId, CODEX_HOME_NON_PERSISTENT_ENTRIES); + const rolloutPath = resolve(root, "codex-home", rollout.path); + const inventory = await inspectStoppedCodexRollout(rolloutPath, { threadId: provider.providerSessionId, turnId, cwd: execution.workspace.cwd, completedTaskControlCalls }); + if (!inventory.safe) return null; + const rolloutSha256 = inventory.sha256; + const evidence = { schema: "paperclip.stopped_text_turn.v2", runId: run.id, nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId, processPid: stopped.processPid, providerPid: provider.processId, providerSessionId: provider.providerSessionId, providerTurnId: turnId, stateFingerprint: snapshot.fingerprint, rolloutSha256, @@ -2616,10 +2519,12 @@ export async function verifyStoppedNativeSessionForReplacement( evidence, retire: async () => (await cleanupStateSnapshot(root)).fingerprint === snapshot.fingerprint && idle() && cleanupProcessAbsent(stopped.processPid) && cleanupProcessAbsent(provider.processId) && - nativeSha256(readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven").toString("utf8")) === rolloutSha256 && + (await inspectStoppedCodexRollout(rolloutPath)).sha256 === rolloutSha256 && completeTerminatedLocalNativeSessionCleanup({ companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId! }), }; } catch { return null; } + + }); } /** Exact local cleanup only: the accepted result and original quarantine are @@ -2635,6 +2540,8 @@ export async function reconcileRetainedNativeSessionCleanup( status: "settled" | "not_eligible" | "operator_required"; runId: string; }> { + return withNativeAuthorityReadContext(db, async () => { + const denied = () => new Error("native_cleanup_maintenance_unproven"); const leaseOwner = `native-cleanup:${randomUUID()}`; let reservedScope: string | null = null; @@ -2653,7 +2560,7 @@ export async function reconcileRetainedNativeSessionCleanup( root: string; emptyRoot: ReturnType; source: Awaited>; - providerHome: ReturnType; + providerHome: Awaited>; sourceArchive: { intent: Record; fromCanonical: boolean; @@ -2859,7 +2766,7 @@ export async function reconcileRetainedNativeSessionCleanup( fromCanonical: rootExists, completed: !!archived, }; - } else if (rootExists && readdirSync(root).length) { + } else if (rootExists && !nativeDirectoryIsEmpty(root)) { if (maintenanceHistory.length || candidates.length) return null; sourceDirectory = root; quarantine = resolve( @@ -2983,10 +2890,10 @@ export async function reconcileRetainedNativeSessionCleanup( snapshot: Awaited>; requestId: string; } | null = null; - const providerHome = cleanupProviderHomeSnapshot( + const providerHome = (await cleanupProviderHomeSnapshot( resolve(sourceDirectory, "codex-home"), true, - ); + )); if (sourceArchive) { if (archiveHistory.length) { if ( @@ -3152,8 +3059,24 @@ export async function reconcileRetainedNativeSessionCleanup( const archiveReference = owned.sourceArchive ? { sourceArchiveRequestId: owned.sourceArchive.intent.requestId } : {}; + let maintenanceDb: Db | AuthorityTransaction = db; + const renewMaintenanceLease = async () => { + if (!reservedScope || executingRunnerdSessionScopes.get(reservedScope) !== leaseOwner) throw denied(); + const renewed = await maintenanceDb.update(nativeRunFinalizations) + .set({ leaseExpiresAt: sql`clock_timestamp() + interval '60 seconds'`, updatedAt: new Date() }) + .where(and( + eq(nativeRunFinalizations.runId, owned.run.id), + eq(nativeRunFinalizations.companyId, owned.run.companyId), + eq(nativeRunFinalizations.phase, "committed"), + eq(nativeRunFinalizations.leaseOwner, leaseOwner), + gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`), + )).returning({ runId: nativeRunFinalizations.runId }); + if (renewed.length !== 1) throw denied(); + }; + let maintenanceLease = startNativeMaintenanceLeaseRenewal(renewMaintenanceLease); const assertLease = async () => { - const lease = await db + await maintenanceLease.assert(); + const lease = await maintenanceDb .select({ runId: nativeRunFinalizations.runId }) .from(nativeRunFinalizations) .where( @@ -3162,7 +3085,7 @@ export async function reconcileRetainedNativeSessionCleanup( eq(nativeRunFinalizations.companyId, owned.run.companyId), eq(nativeRunFinalizations.phase, "committed"), eq(nativeRunFinalizations.leaseOwner, leaseOwner), - gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), + gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`), ), ) .limit(1); @@ -3180,10 +3103,10 @@ export async function reconcileRetainedNativeSessionCleanup( if ( (await cleanupStateSnapshot(owned.quarantine)).fingerprint !== owned.source.fingerprint || - cleanupProviderHomeSnapshot( + (await cleanupProviderHomeSnapshot( resolve(owned.quarantine, "codex-home"), false, - ).metadataFingerprint !== owned.providerHome.metadataFingerprint || + )).metadataFingerprint !== owned.providerHome.metadataFingerprint || (owned.copySource && (!retainedRunnerdMaintenanceIsIdle(owned.copySource.directory) || (await cleanupStateSnapshot(owned.copySource.directory)).fingerprint !== @@ -3245,10 +3168,10 @@ export async function reconcileRetainedNativeSessionCleanup( canonicalJson(archive.intent.rootIdentity) || (await cleanupStateSnapshot(sourceDirectory)).fingerprint !== owned.source.fingerprint || - cleanupProviderHomeSnapshot( + (await cleanupProviderHomeSnapshot( resolve(sourceDirectory, "codex-home"), true, - ).fingerprint !== owned.providerHome.fingerprint || + )).fingerprint !== owned.providerHome.fingerprint || (archive.fromCanonical ? existsSync(owned.quarantine) : existsSync(owned.root)) @@ -3264,10 +3187,10 @@ export async function reconcileRetainedNativeSessionCleanup( renameSync(owned.root, owned.quarantine); archive.fromCanonical = false; } - const archivedHome = cleanupProviderHomeSnapshot( + const archivedHome = (await cleanupProviderHomeSnapshot( resolve(owned.quarantine, "codex-home"), true, - ); + )); if ( (await cleanupStateSnapshot(owned.quarantine)).fingerprint !== owned.source.fingerprint || @@ -3312,12 +3235,12 @@ export async function reconcileRetainedNativeSessionCleanup( ); chmodSync(resolve(copy, file), 0o600); } - copyCleanupProviderHome( + await copyCleanupProviderHome( resolve(owned.quarantine, "codex-home"), resolve(copy, "codex-home"), owned.providerHome, ); - rebaseRetainedNativeCleanupProviderHome( + await rebaseRetainedNativeCleanupProviderHome( resolve(copy, "codex-home"), resolve(owned.root, "codex-home"), owned.providerSessionId, @@ -3334,10 +3257,10 @@ export async function reconcileRetainedNativeSessionCleanup( stagingName: basename(copy), providerHomeFingerprint: owned.providerHome.fingerprint, providerHomeBytes: owned.providerHome.bytes, - stagedProviderHomeFingerprint: cleanupProviderHomeSnapshot( + stagedProviderHomeFingerprint: (await cleanupProviderHomeSnapshot( resolve(copy, "codex-home"), true, - ).fingerprint, + )).fingerprint, }); const proof = await settleRetainedRunnerdSession({ requestId: leaseOwner, @@ -3389,20 +3312,20 @@ export async function reconcileRetainedNativeSessionCleanup( // Commit intent before the filesystem handoff. A crash can then be // distinguished from an unattempted quarantine; never replay its source. if ( - cleanupProviderHomeSnapshot(resolve(owned.quarantine, "codex-home"), true) + (await cleanupProviderHomeSnapshot(resolve(owned.quarantine, "codex-home"), true)) .fingerprint !== owned.providerHome.fingerprint ) throw denied(); - rebaseRetainedNativeCleanupProviderHome( + await rebaseRetainedNativeCleanupProviderHome( resolve(copy, "codex-home"), resolve(owned.root, "codex-home"), owned.providerSessionId, "canonical", ); - const settledHome = cleanupProviderHomeSnapshot( + const settledHome = (await cleanupProviderHomeSnapshot( resolve(copy, "codex-home"), true, - ); + )); const emptyRootArchive = owned.emptyRoot ? `${basename(owned.root)}.empty-before-cleanup.${leaseOwner}` : null; @@ -3453,6 +3376,11 @@ export async function reconcileRetainedNativeSessionCleanup( .returning({ runId: nativeRunFinalizations.runId }); if (!prepared.length) throw denied(); owned.history = preparedHistory; + // Drain the outside connection before taking the row lock. Renewals during + // the final history check must use this transaction, or wait on our own + // row lock and deadlock the authority check. + await maintenanceLease.stop(); + await maintenanceLease.assert(); await db.transaction(async (tx) => { const current = await tx .select() @@ -3468,9 +3396,12 @@ export async function reconcileRetainedNativeSessionCleanup( .limit(1) .then((rows) => rows[0]); if (!current || current.phase !== "committed") throw denied(); + maintenanceDb = tx; + maintenanceLease = startNativeMaintenanceLeaseRenewal(renewMaintenanceLease); + try { await authorize(); if ( - cleanupProviderHomeSnapshot(resolve(copy, "codex-home"), false) + (await cleanupProviderHomeSnapshot(resolve(copy, "codex-home"), false)) .metadataFingerprint !== settledHome.metadataFingerprint ) throw denied(); @@ -3507,9 +3438,13 @@ export async function reconcileRetainedNativeSessionCleanup( updatedAt: new Date(), }) .where(eq(nativeRunFinalizations.runId, owned.run.id)); + } finally { + await maintenanceLease.stop(); + maintenanceDb = db; + } }); if ( - cleanupProviderHomeSnapshot(resolve(owned.root, "codex-home"), true) + (await cleanupProviderHomeSnapshot(resolve(owned.root, "codex-home"), true)) .fingerprint !== settledHome.fingerprint ) throw denied(); @@ -3570,8 +3505,11 @@ export async function reconcileRetainedNativeSessionCleanup( }); return { status: "operator_required", runId: input.runId }; } finally { + await maintenanceLease.stop(); releaseScope(); } + + }); } /** Read-only scoped admission fence; never grants provider or recovery authority. */ @@ -3579,6 +3517,8 @@ export async function assertRetainedNativeSourceArchiveSettled( db: Db, input: { companyId: string; issueId: string; stateKey: string }, ): Promise { + return withNativeAuthorityReadContext(db, async () => { + if (!/^[a-f0-9]{64}$/.test(input.stateKey)) throw new NativeSessionCleanupQuarantinedError(); const owners = await db @@ -3649,6 +3589,8 @@ export async function assertRetainedNativeSourceArchiveSettled( ) throw new NativeSessionCleanupQuarantinedError(); } + + }); } async function assertCleanupActivationCommitted( db: Db, @@ -3702,7 +3644,7 @@ async function assertCleanupActivationCommitted( !/^[0-9a-f]{64}$/.test(marker.settledProviderHomeFingerprint) || receipt?.settledProviderHomeFingerprint !== marker.settledProviderHomeFingerprint || - cleanupProviderHomeSnapshot(resolve(root, "codex-home"), true) + (await cleanupProviderHomeSnapshot(resolve(root, "codex-home"), true)) .fingerprint !== marker.settledProviderHomeFingerprint ) throw new NativeSessionCleanupQuarantinedError(); @@ -3760,6 +3702,7 @@ function failedNativeRetryOwnersStopped(execution: NativeExecutionInput, process * run. Never migrate/archive state, release an owner, or contact a provider. * Normal executor admission independently verifies the state again. */ export async function nativeFailedRunRetryStateIsSafe(input: { + db?: Db; execution: unknown; companyId: string; issueId: string; @@ -3774,6 +3717,10 @@ export async function nativeFailedRunRetryStateIsSafe(input: { recoveryMode: "bootstrap_retry" | "exact_checkpoint_resume"; allowVerifiedBackup: boolean; }): Promise { + if (input.db) { + const { db, ...bound } = input; + return withNativeAuthorityReadContext(db, () => nativeFailedRunRetryStateIsSafe(bound)); + } try { const execution = parseNativeExecutionInput(input.execution); if ( @@ -3834,9 +3781,9 @@ export async function nativeFailedRunRetryStateIsSafe(input: { ) return false; let stateRoot = root; - const direct = runnerdAuthorityLifecycle(root, identity); + const direct = (await runnerdAuthorityLifecycle(root, identity)); if (direct === "absent" && input.allowVerifiedBackup) { - const backup = verifyNativeHarnessBackup({ + const backup = await verifyNativeHarnessBackup({ root, execution, runnerInstanceId: input.runnerInstanceId, @@ -3844,15 +3791,15 @@ export async function nativeFailedRunRetryStateIsSafe(input: { if (!backup) return false; stateRoot = backup.root; } - if (runnerdAuthorityLifecycle(stateRoot, identity) !== "suspended") + if ((await runnerdAuthorityLifecycle(stateRoot, identity)) !== "suspended") return false; const runnerState = record( JSON.parse( - readBoundedNativeFile( + (await readCurrentNativeFile( resolve(stateRoot, "runner", "runner-state.json"), NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large", - ).toString("utf8"), + )).toString("utf8"), ), ); if ( @@ -3874,11 +3821,11 @@ export async function nativeFailedRunRetryStateIsSafe(input: { if (input.recoveryMode === "bootstrap_retry") return false; const providerState = record( JSON.parse( - readBoundedNativeFile( + (await readCurrentNativeFile( providerFile, NATIVE_RUNNER_STATE_MAX_BYTES, "runner_provider_state_too_large", - ).toString("utf8"), + )).toString("utf8"), ), ); if ( @@ -3910,6 +3857,7 @@ export async function nativeFailedRunRetryStateIsSafe(input: { * The caller separately proves the failed receipt/coordinator has no native * events or result and selects exactly one committed cleanup owner. */ export async function nativePreProviderRetryAfterCleanupStateIsSafe(input: { + db?: Db; failedExecution: unknown; retiredExecution: unknown; companyId: string; @@ -3925,6 +3873,10 @@ export async function nativePreProviderRetryAfterCleanupStateIsSafe(input: { processGroupId: number; receipt: Record; }): Promise { + if (input.db) { + const { db, ...bound } = input; + return withNativeAuthorityReadContext(db, () => nativePreProviderRetryAfterCleanupStateIsSafe(bound)); + } try { const failed = parseNativeExecutionInput(input.failedExecution); const retired = parseNativeExecutionInput(input.retiredExecution); @@ -4047,12 +3999,12 @@ async function verifyPriorRunnerdStateForSessionScope(input: { nativeSessionScopeKey(priorExecution) === nativeSessionScopeKey(input.execution); if (!sameScope) return "scope_mismatch"; - const lifecycle = runnerdAuthorityLifecycleWithVerifiedBackup(input); + const lifecycle = (await runnerdAuthorityLifecycleWithVerifiedBackup(input)); if (lifecycle === "suspended") return "verified"; - const directLifecycle = runnerdAuthorityLifecycle( + const directLifecycle = (await runnerdAuthorityLifecycle( input.root, input.identity, - ); + )); if ( input.allowRetainedWarmRunner && (lifecycle === "not_suspended" || @@ -4081,7 +4033,7 @@ async function hasRetainedWarmTransitionEvidence(root: string): Promise try { const state = record(directory === "control-plane" ? (await readNativeJournalProjection(path)).value - : JSON.parse(readBoundedNativeFile(path, maximum, "runner_state_too_large").toString("utf8"))); + : JSON.parse((await readCurrentNativeFile(path, maximum, "runner_state_too_large")).toString("utf8"))); if (Object.prototype.hasOwnProperty.call(state, "warmTransition") || state.schema === "paperclip.runner.durable.control-plane-state.warm-transition.v1" || state.schema === "paperclip.runner.durable.state.warm-transition.v1") return true; @@ -4121,7 +4073,7 @@ async function readWarmTransitionSnapshot(root: string) { const corePath = resolve(root, "control-plane", "control-plane-state.json"); const runnerPath = resolve(root, "runner", "runner-state.json"); const core = (await readNativeJournalProjection(corePath)); - const runner = readBoundedNativeFile(runnerPath, NATIVE_RUNNER_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven"); + const runner = (await readCurrentNativeFile(runnerPath, NATIVE_RUNNER_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven")); // The worker preserves the original canonical JSON [base64(core), base64(runner)] hash. const fingerprint = await nativeStateBase64Fingerprint([ { path: corePath, maximum: DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES }, @@ -4160,6 +4112,9 @@ async function verifyWarmTransitionRestart(input: { return deny(); const root = scopedRunnerdStateRoot(input.execution); const snapshot = (await readWarmTransitionSnapshot(root)); + if ([snapshot.controlPlaneState, snapshot.runnerState].some(value => typeof record(value).indexedGeneration === "string")) { + return deny(); + } const artifact = readRunnerdArtifactBinding(resolvePaperclipRunnerBinary()); const controller = await currentNativeControllerIdentity(); const binding = input.execution.binding; @@ -4574,12 +4529,12 @@ async function migrateRunnerdStateRootForExecution(input: { } if (durableIdentityMatchesExecution(identity, input.execution)) { if ( - runnerdAuthorityLifecycleWithVerifiedBackup({ + (await runnerdAuthorityLifecycleWithVerifiedBackup({ root: scoped, identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, - }) === "indeterminate" + })) === "indeterminate" ) { if (input.restartRecovery?.kind !== "reattach_existing_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); @@ -4681,7 +4636,7 @@ async function recoverQuiescentRunnerdState(input: { if (scopedExists && !isSafeNativeStateDirectory(input.scoped)) return; // Never hide corrupt or contradictory current authority behind an older // checkpoint. Only an absent/empty root may recover from quarantine. - const currentEmpty = !scopedExists || readdirSync(input.scoped).length === 0; + const currentEmpty = !scopedExists || nativeDirectoryIsEmpty(input.scoped); const candidates = currentEmpty ? isSafeNativeStateDirectory(quarantine) ? readdirSync(quarantine) @@ -4697,7 +4652,7 @@ async function recoverQuiescentRunnerdState(input: { currentEmpty && candidates.filter( (root) => - isSafeNativeStateDirectory(root) && readdirSync(root).length > 0, + isSafeNativeStateDirectory(root) && !nativeDirectoryIsEmpty(root), ).length > 1 ) { // Do not roll back to an older valid checkpoint when a newer quarantined @@ -4724,22 +4679,27 @@ async function recoverQuiescentRunnerdState(input: { try { const directoryIdentities = [root, resolve(root, "runner"), resolve(root, "control-plane")] .map(path => ({ path, identity: nativeStateDirectoryIdentity(path) })); - const readState = (directory: string, name: string) => { + const readState = async (directory: string, name: string) => { if (!isSafeNativeStateDirectory(resolve(root, directory))) throw new Error("unsafe_recovery_state"); - return readBoundedNativeFile( + return (await readCurrentNativeFile( resolve(root, directory, name), nativeStateFileMaxBytes(`${directory}/${name}`), "recovery_state_too_large", - ).toString("utf8"); + )).toString("utf8"); }; - const runnerBytes = readState("runner", "runner-state.json"); + const runnerBytes = (await readState("runner", "runner-state.json")); const controlProof = (await readNativeJournalProjection(resolve(root, "control-plane", "control-plane-state.json"))); const controlSha256 = controlProof.sha256; - const providerBytes = readState("runner", "codex-provider-state.json"); + const providerBytes = (await readState("runner", "codex-provider-state.json")); const runner = record(JSON.parse(runnerBytes)); const control = record(controlProof.value); const provider = record(JSON.parse(providerBytes)); + if ([runner, control, provider].some(value => typeof value.indexedGeneration === "string")) { + // This legacy recovery path proves individual PIDs only. It cannot + // authorize mutation or replacement of an indexed process lifetime. + continue; + } const commands = control.commands; const events = control.committedEvents; const terminalEnvelope = record( @@ -4859,9 +4819,9 @@ async function recoverQuiescentRunnerdState(input: { if (processIds.some((pid) => !localProcessDefinitelyGone(pid))) continue; // No mutation if any evidence changed while the database was read. if ( - runnerBytes !== readState("runner", "runner-state.json") || + runnerBytes !== (await readState("runner", "runner-state.json")) || controlSha256 !== (await scanNativeStateFile(resolve(root, "control-plane", "control-plane-state.json"), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES)).sha256 || - providerBytes !== readState("runner", "codex-provider-state.json") + providerBytes !== (await readState("runner", "codex-provider-state.json")) ) continue; for (const directory of directoryIdentities) assertNativeStateDirectoryIdentity(directory.path, directory.identity); @@ -4888,7 +4848,7 @@ async function recoverQuiescentRunnerdState(input: { currentEmpty && candidates.some( (root) => - isSafeNativeStateDirectory(root) && readdirSync(root).length > 0, + isSafeNativeStateDirectory(root) && !nativeDirectoryIsEmpty(root), ) ) { // Known provider history is not permission to start a replacement when @@ -4908,7 +4868,7 @@ async function recoverQuiescentRunnerdState(input: { if ( (relativePath === "control-plane/control-plane-state.json" ? (await scanNativeStateFile(resolve(candidate.root, relativePath), DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES)).sha256 - : readBoundedNativeFile(resolve(candidate.root, relativePath!), NATIVE_RUNNER_STATE_MAX_BYTES, "recovery_state_too_large").toString("utf8")) !== expected + : (await readCurrentNativeFile(resolve(candidate.root, relativePath!), NATIVE_RUNNER_STATE_MAX_BYTES, "recovery_state_too_large")).toString("utf8")) !== expected ) { throw new Error("runner_state_identity_mismatch"); } @@ -4925,7 +4885,7 @@ async function recoverQuiescentRunnerdState(input: { if (existsSync(input.scoped)) { if ( !isSafeNativeStateDirectory(input.scoped) || - readdirSync(input.scoped).length !== 0 + !nativeDirectoryIsEmpty(input.scoped) ) return; quarantineRunnerdStateRoot(input.scoped, "identity_indeterminate"); @@ -4937,11 +4897,8 @@ async function recoverQuiescentRunnerdState(input: { // next run, retaining the provider's history and exact thread identity. const statePath = resolve(input.scoped, "runner", "runner-state.json"); const temporary = `${statePath}.${randomUUID()}.tmp`; - writeFileSync( - temporary, - JSON.stringify({ ...candidate.runner, lifecycle: "suspended" }), - { encoding: "utf8", mode: 0o600, flag: "wx" }, - ); + writeFileSync(temporary, JSON.stringify({ ...candidate.runner, lifecycle: "suspended" }), + { encoding: "utf8", mode: 0o600, flag: "wx" }); renameSync(temporary, statePath); await input.onLog?.( "stdout", @@ -4983,6 +4940,8 @@ export async function runnerdStateProvesIncompleteBootstrap(root: string): Promi state.schema === RUNNERD_CONTROL_PLANE_STATE_SCHEMA && state.connectionCount === 0 && committedEvents.length === 0 && + record(state.indexedState).providerEverStarted !== true && + record(state.indexedState).externalEffectEverAdmitted !== true && onlyUnconsumedBootstrapCommands ); } catch (error) { @@ -4993,6 +4952,10 @@ export async function runnerdStateProvesIncompleteBootstrap(root: string): Promi function nativeJournalProofUnavailable(error: unknown): boolean { return error instanceof Error && (error.message.startsWith("native_journal_worker_") + || error.message.startsWith("indexed_state_") + || error.message.startsWith("native_indexed_") + || error.message.startsWith("native_legacy_migration_") + || error.name === "DurableAuthorityStoreError" || error.message === "native_state_file_changed" || error.message === "native_journal_projection_budget_exceeded"); } @@ -5238,6 +5201,7 @@ export interface NativeHarnessPersistenceProfile { export interface NativeHarnessBackupManifest { schema: "paperclip.native-harness-backup.v1"; + nativeState?: NativeCheckpointBindings; normalizedSessionId: string; runnerInstanceId: string; providerKind: NativeProviderKind; @@ -5523,43 +5487,6 @@ export function providerSessionIdentityTransitionIsAllowed(input: { ); } -function digestBackupDirectory(directory: string): { - sha256: string; - bytes: number; -} { - const hash = createHash("sha256"); - let bytes = 0; - const visit = (current: string, relative: string) => { - const entries = readdirSync(current, { withFileTypes: true }).sort( - (left, right) => left.name.localeCompare(right.name), - ); - if (entries.length === 0) hash.update(`directory:${relative}\0`); - for (const entry of entries) { - const entryPath = resolve(current, entry.name); - const entryRelative = relative ? `${relative}/${entry.name}` : entry.name; - const stats = lstatSync(entryPath); - if (entry.isDirectory()) { - hash.update(`directory:${entryRelative}:${stats.mode & 0o777}\0`); - visit(entryPath, entryRelative); - } else if (entry.isSymbolicLink()) { - hash.update(`symlink:${entryRelative}:${readlinkSync(entryPath)}\0`); - } else if (entry.isFile()) { - const contents = readFileSync(entryPath); - bytes += contents.byteLength; - hash.update( - `file:${entryRelative}:${stats.mode & 0o777}:${contents.byteLength}\0`, - ); - hash.update(contents); - } else { - throw new Error( - `runner_harness_backup_unsupported_entry:${entryRelative}`, - ); - } - } - }; - visit(directory, ""); - return { sha256: `sha256:${hash.digest("hex")}`, bytes }; -} function harnessBackupRoot(root: string): string { return resolve(root, "failover-backups"); @@ -5607,7 +5534,7 @@ function compatibleNativeHarnessBackupManifests(input: { let manifest: NativeHarnessBackupManifest; try { manifest = JSON.parse( - readFileSync(manifestPath, "utf8"), + readNativeHarnessBackupManifestBytes(manifestPath).toString("utf8"), ) as NativeHarnessBackupManifest; } catch { continue; @@ -5622,7 +5549,7 @@ function compatibleNativeHarnessBackupManifests(input: { nativeHarnessEnvironmentFingerprint(input.execution) || manifest.runnerContractVersion !== RUNNERD_BINARY_CONTRACT_VERSION || !providerSessionIdentityIsPresent(manifest.providerSessionIdentity) || - !Array.isArray(manifest.directories) + !Array.isArray(manifest.directories) || manifest.directories.length > 4 ) continue; const manifestNames = manifest.directories @@ -5634,25 +5561,32 @@ function compatibleNativeHarnessBackupManifests(input: { return compatible; } -export function buildNativeHarnessBackupManifest(input: { +export async function buildNativeHarnessBackupManifest(input: { backupRoot: string; execution: NativeExecutionInput; runnerInstanceId: string; providerSessionIdentity: unknown; sourceProviderLeaseId: string; completedAt?: string; -}): NativeHarnessBackupManifest { +}): Promise { if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) { throw new Error("runner_harness_state_mismatch"); } + const native = await readNativeIndexedCheckpointProof(resolve(input.backupRoot, "runner")); + if (native && (native.runner.state.runnerInstanceId !== input.runnerInstanceId || + native.runner.state.normalizedSessionId !== nativeSessionKey(input.execution) || native.bindings.runId !== input.execution.binding.runId || + canonicalJson(providerSessionIdentityFromDurableProviderState({ execution: input.execution, providerState: native.provider.state })) !== canonicalJson(input.providerSessionIdentity))) { + throw new Error("runner_harness_state_mismatch: indexed_snapshot_identity"); + } const profile = resolveNativeHarnessPersistenceProfile(input.execution); - const directories = profile.directories.map((directory) => { + const directories = await Promise.all(profile.directories.map(async (directory) => { const path = resolve(input.backupRoot, directory.name); if (!existsSync(path)) throw new Error("runner_harness_state_mismatch"); - return { name: directory.name, ...digestBackupDirectory(path) }; - }); + return { name: directory.name, ...await digestNativeHarnessBackupDirectory(path, { sync: true }) }; + })); return { schema: "paperclip.native-harness-backup.v1", + ...(native ? { nativeState: native.bindings } : {}), normalizedSessionId: nativeSessionKey(input.execution), runnerInstanceId: input.runnerInstanceId, providerKind: profile.providerKind, @@ -5668,15 +5602,21 @@ export function buildNativeHarnessBackupManifest(input: { }; } -export function verifyNativeHarnessBackup(input: { +export async function verifyNativeHarnessBackup(input: { root: string; execution: NativeExecutionInput; runnerInstanceId: string; -}): VerifiedHarnessBackup | null { +}): Promise { for (const { root: candidateRoot, manifest, } of compatibleNativeHarnessBackupManifests(input)) { + try { + const native = await readNativeIndexedCheckpointProof(resolve(candidateRoot, "runner")); + if (canonicalJson(native?.bindings ?? null) !== canonicalJson(manifest.nativeState ?? null) || + (native && (native.runner.state.runnerInstanceId !== input.runnerInstanceId || native.runner.state.normalizedSessionId !== nativeSessionKey(input.execution) || + canonicalJson(providerSessionIdentityFromDurableProviderState({ execution: input.execution, providerState: native.provider.state })) !== canonicalJson(manifest.providerSessionIdentity)))) continue; + } catch { continue; } let bytes = 0; let valid = true; for (const declared of manifest.directories) { @@ -5686,7 +5626,7 @@ export function verifyNativeHarnessBackup(input: { break; } try { - const digest = digestBackupDirectory(directoryPath); + const digest = await digestNativeHarnessBackupDirectory(directoryPath); if ( digest.sha256 !== declared.sha256 || digest.bytes !== declared.bytes @@ -5700,7 +5640,11 @@ export function verifyNativeHarnessBackup(input: { break; } } - if (valid) return { root: candidateRoot, manifest, bytes }; + try { + if (valid && canonicalJson(manifest) === canonicalJson(JSON.parse(readNativeHarnessBackupManifestBytes(resolve(candidateRoot, "manifest.json")).toString("utf8")))) return { root: candidateRoot, manifest, bytes }; + } catch { + // An asynchronously replaced manifest invalidates this candidate. + } } return null; } @@ -6089,6 +6033,8 @@ export async function nativeProviderRecoveryEvidence(input: { providerEventsExist: boolean; checkpointExists: boolean; }> { + return withNativeAuthorityReadContext(input.db, async () => { + const run = await input.db .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) .from(heartbeatRuns) @@ -6158,6 +6104,8 @@ export async function nativeProviderRecoveryEvidence(input: { providerEventsExist, checkpointExists, }; + + }); } function record(value: unknown): Record { @@ -7036,6 +6984,8 @@ export async function renewNativeSessionExecutionLease(input: { controller?: NativeControllerIdentity; leaseTtlMs?: number; }): Promise { + return withNativeAuthorityReadContext(input.db, async () => { + const controller = input.controller ?? (await currentNativeControllerIdentity()); const leaseTtlMs = input.leaseTtlMs ?? NATIVE_SESSION_EXECUTION_LEASE_TTL_MS; @@ -7070,6 +7020,8 @@ export async function renewNativeSessionExecutionLease(input: { ) .returning({ runId: nativeRunFinalizations.runId }); if (!updated) throw new Error("native_session_lease_lost"); + + }); } function startNativeSessionExecutionLeaseRenewal(input: { @@ -7173,6 +7125,8 @@ export async function executePaperclipNativeSession(input: { }, ) => Promise; }): Promise { + return withNativeAuthorityReadContext(input.db, async () => { + const runId = input.execution.binding.runId; if (nativeSessionStartups.has(runId)) { throw new Error("native_session_supervisor_busy"); @@ -7293,6 +7247,8 @@ export async function executePaperclipNativeSession(input: { .catch(() => undefined); } } + + }); } async function executePaperclipNativeSessionWithinScope( @@ -7558,7 +7514,7 @@ async function executePaperclipNativeSessionWithinScope( ? recovering.controllerGeneration : coordinator.controllerBootId === controller.bootId ? Math.max(1, coordinator.controllerGeneration) - : coordinator.controllerGeneration + 1; + : nextNativeControllerGeneration(coordinator.controllerGeneration); const claimed = await tx .update(nativeRunFinalizations) .set({ @@ -7613,6 +7569,7 @@ async function executePaperclipNativeSessionWithinScope( } const controlPlaneInstanceId = `${effectiveRunnerInstanceId}:control`; const planSynchronizations: PlanSynchronization[] = []; + let planSynchronizationsTruncated = false; const upsertPlanSynchronization = ( synchronization: PlanSynchronization, ): void => { @@ -7624,6 +7581,13 @@ async function executePaperclipNativeSessionWithinScope( return; } planSynchronizations.push(synchronization); + // Every synchronization already has an immutable activity/document receipt. + // The final run result carries a recent display excerpt, never lifetime + // history from a multi-day active turn. + if (planSynchronizations.length > 128) { + planSynchronizations.shift(); + planSynchronizationsTruncated = true; + } }; const recordPlanSynchronization = async (event: { sourceEventId: string; @@ -7690,6 +7654,7 @@ async function executePaperclipNativeSessionWithinScope( eventType: event.eventType, sourceInstanceId: event.sourceInstanceId, sourceRunId: event.runId, + sourceEpoch: event.sourceEpoch, sourceSeq: event.sourceSeq, payload: event.payload, }, @@ -9035,6 +9000,7 @@ async function executePaperclipNativeSessionWithinScope( nativeTerminal: native.terminal as unknown as Record, ...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}), planSynchronizations, + ...(planSynchronizationsTruncated ? { planSynchronizationsTruncated: true } : {}), }, summary: native.result.summary, sessionId: native.normalizedSessionId, @@ -9488,25 +9454,7 @@ async function stageRemoteRunnerFile(input: { } } -function archiveExcludeArgs(entries: readonly string[]): string[] { - for (const entry of entries) { - const segments = entry.split("/"); - if ( - entry.length === 0 || - entry.startsWith("/") || - segments.some( - (segment) => - segment === "" || - segment === "." || - segment === ".." || - !/^[A-Za-z0-9._-]+$/.test(segment), - ) - ) { - throw new Error("runner_remote_checkpoint_exclusion_invalid"); - } - } - return entries.map((entry) => `--exclude=./${entry}`); -} +const archiveExcludeArgs = checkpointExclusions; export async function stageRemoteRunnerDirectory(input: { target: Extract; @@ -9517,22 +9465,20 @@ export async function stageRemoteRunnerDirectory(input: { excludeEntries?: readonly string[]; }): Promise { const excludeArgs = archiveExcludeArgs(input.excludeEntries ?? []); - if (input.runner.syncIn) { + // Native bulk sync can impose a lifetime-sized archive cap or RPC deadline. + // The owned command channel keeps archive work separate from bounded chunks. + if (input.runner.syncIn && !input.runner.openDuplexChannel) { let stagingRoot: string | null = null; let sourcePath = input.sourcePath; try { if (excludeArgs.length > 0) { stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-restore-")); - const archive = execFileSync( - "tar", - [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, - ); - execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], { - input: archive, - maxBuffer: 64 * 1024 * 1024, - }); - sourcePath = stagingRoot; + const archive = join(stagingRoot, "payload.tar.gz"); + const payload = join(stagingRoot, "payload"); + mkdirSync(payload, { mode: input.mode }); + await checkpointTar([...excludeArgs, "-czf", archive, "-C", input.sourcePath, "."]); + await checkpointTar(["-xzf", archive, "-C", payload]); + sourcePath = payload; } await input.runner.syncIn([ { @@ -9552,25 +9498,12 @@ export async function stageRemoteRunnerDirectory(input: { } return; } - const archive = execFileSync( - "tar", - [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, - ); - const escapedTarget = input.targetPath.replaceAll("'", "'\\''"); - const script = - `umask 077; mkdir -p '${escapedTarget}' && ` + - `base64 -d | tar -xzf - -C '${escapedTarget}' && ` + - `chmod ${input.mode.toString(8)} '${escapedTarget}'`; - const result = await input.runner.execute({ - command: "sh", - args: ["-c", script], - stdin: archive.toString("base64"), - bypassSession: true, - }); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error("runner_remote_directory_staging_failed"); - } + const stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-upload-")); + try { + const archive = join(stagingRoot, "payload.tar.gz"); + await checkpointTar([...excludeArgs, "-czf", archive, "-C", input.sourcePath, "."]); + await uploadCheckpointArchive({ runner: input.runner, archive, targetPath: input.targetPath, mode: input.mode }); + } finally { rmSync(stagingRoot, { recursive: true, force: true }); } } async function remoteRunnerPathExists(input: { @@ -9588,98 +9521,6 @@ async function remoteRunnerPathExists(input: { return result.exitCode === 0 && !result.timedOut; } -function assertSafeRemoteCheckpointArchive(archive: Buffer): void { - if (archive.length > MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES) { - throw new Error("runner_remote_checkpoint_archive_too_large"); - } - let names: string[]; - let verboseEntries: string[]; - try { - names = execFileSync("tar", ["-tzf", "-"], { - input: archive, - encoding: "utf8", - maxBuffer: 8 * 1024 * 1024, - timeout: 30_000, - }) - .split("\n") - .filter((line) => line.length > 0); - verboseEntries = execFileSync("tar", ["-tvzf", "-"], { - input: archive, - encoding: "utf8", - maxBuffer: 8 * 1024 * 1024, - timeout: 30_000, - }) - .split("\n") - .filter((line) => line.length > 0); - } catch { - throw new Error("runner_remote_checkpoint_archive_invalid"); - } - if ( - names.length === 0 || - names.length > MAX_REMOTE_CHECKPOINT_ENTRIES || - verboseEntries.length !== names.length - ) { - throw new Error("runner_remote_checkpoint_archive_invalid"); - } - for (const name of names) { - const normalized = name.replace(/^\.\//, ""); - if ( - name.includes("\0") || - name.startsWith("/") || - normalized.split("/").some((part) => part === "..") - ) { - throw new Error("runner_remote_checkpoint_archive_unsafe_path"); - } - } - for (const entry of verboseEntries) { - const type = entry.trimStart()[0]; - if (type !== "-" && type !== "d") { - throw new Error("runner_remote_checkpoint_archive_unsafe_entry"); - } - } - try { - execFileSync("tar", ["-xOzf", "-"], { - input: archive, - stdio: ["pipe", "pipe", "pipe"], - maxBuffer: MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES, - timeout: 60_000, - }); - } catch { - throw new Error("runner_remote_checkpoint_archive_expanded_too_large"); - } -} - -function assertSafeExtractedCheckpoint(root: string): void { - const pending = [root]; - let entries = 0; - let totalBytes = 0; - while (pending.length > 0) { - const directory = pending.pop()!; - for (const entry of readdirSync(directory, { withFileTypes: true })) { - entries += 1; - if (entries > MAX_REMOTE_CHECKPOINT_ENTRIES) { - throw new Error("runner_remote_checkpoint_too_many_entries"); - } - const path = join(directory, entry.name); - const metadata = lstatSync(path); - if ( - metadata.isSymbolicLink() || - (!metadata.isDirectory() && !metadata.isFile()) - ) { - throw new Error("runner_remote_checkpoint_unsafe_entry"); - } - if (metadata.isDirectory()) { - pending.push(path); - } else { - totalBytes += metadata.size; - if (totalBytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES) { - throw new Error("runner_remote_checkpoint_expanded_too_large"); - } - } - } - } -} - /** * Copy a remote runner directory into durable local state. * @@ -9710,7 +9551,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { const excludeArgs = archiveExcludeArgs(excluded) .map((argument) => `'${argument}'`) .join(" "); - if (input.runner.syncOut) { + if (input.runner.syncOut && !input.runner.openDuplexChannel) { let syncSourcePath = input.sourcePath; let snapshotPath: string | null = null; if (excluded.length > 0) { @@ -9769,79 +9610,13 @@ export async function syncRemoteRunnerDirectoryOut(input: { } return; } - const escapedSource = input.sourcePath.replaceAll("'", "'\\''"); - const result = await input.runner.execute({ - command: "sh", - args: ["-c", `tar ${excludeArgs} -czf - -C '${escapedSource}' . | base64`], - bypassSession: true, - timeoutMs: 120_000, - }); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error("runner_remote_checkpoint_failed"); - } - const archive = Buffer.from(result.stdout.replace(/\s+/g, ""), "base64"); - assertSafeRemoteCheckpointArchive(archive); - const parent = resolve(input.targetPath, ".."); - const stagingRoot = mkdtempSync(join(parent, ".paperclip-checkpoint-")); - const stagedTarget = join(stagingRoot, "payload"); - const previousTarget = join( - parent, - `.paperclip-checkpoint-previous-${randomUUID()}`, - ); - let previousMoved = false; - let replacementInstalled = false; - try { - mkdirSync(stagedTarget, { recursive: true, mode: input.mode }); - execFileSync( - "tar", - [ - "--no-same-owner", - "--no-same-permissions", - "-xzf", - "-", - "-C", - stagedTarget, - ], - { input: archive, maxBuffer: 8 * 1024 * 1024, timeout: 60_000 }, - ); - assertSafeExtractedCheckpoint(stagedTarget); - chmodSync(stagedTarget, input.mode); - if (existsSync(input.targetPath)) { - renameSync(input.targetPath, previousTarget); - previousMoved = true; - } - renameSync(stagedTarget, input.targetPath); - replacementInstalled = true; - if (previousMoved) { - rmSync(previousTarget, { recursive: true, force: true }); - previousMoved = false; - } - } catch (error) { - if ( - previousMoved && - !replacementInstalled && - !existsSync(input.targetPath) - ) { - try { - renameSync(previousTarget, input.targetPath); - previousMoved = false; - } catch { - // Leave the last durable checkpoint at previousTarget. Deleting it in - // finally would turn a failed replacement into irreversible data loss. - } - } - throw error; - } finally { - if (previousMoved && replacementInstalled) { - rmSync(previousTarget, { recursive: true, force: true }); - } - rmSync(stagingRoot, { recursive: true, force: true }); - } + await downloadCheckpointDirectory(input); } async function readRemoteRunnerState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; + runnerBinary?: string; }): Promise> { const statePath = posix.join(input.stateDirectory, "runner-state.json"); const escapedPath = statePath.replaceAll("'", "'\\''"); @@ -9856,17 +9631,27 @@ async function readRemoteRunnerState(input: { `runner_remote_state_unavailable: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`, ); } - return record( + const state = record( JSON.parse( Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"), ), ); + return resolveRemoteIndexedLocator(state, input, statePath); +} + +async function resolveRemoteIndexedLocator(state: Record, input: { + runner: CommandManagedRuntimeRunner; runnerBinary?: string; +}, path: string): Promise> { + if (state.schema !== "paperclip.runner.durable.state.indexed.v1" && state.schema !== "paperclip.runner.codex-provider-state.indexed.v1") return state; + if (!input.runnerBinary) throw new Error("native_indexed_inspection_binary_unavailable"); + return (await inspectRemoteIndexedState({ runner: input.runner, runnerBinary: input.runnerBinary, path })).state; } async function readRemoteRunnerProviderState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; execution: NativeExecutionInput; + runnerBinary?: string; }): Promise> { const statePath = posix.join( input.stateDirectory, @@ -9882,11 +9667,12 @@ async function readRemoteRunnerProviderState(input: { if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_remote_provider_state_unavailable"); } - return record( + const state = record( JSON.parse( Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"), ), ); + return resolveRemoteIndexedLocator(state, input, statePath); } const REMOTE_RUNNER_PROCESS_IDENTITY_WAIT_MS = 20_000; @@ -9941,6 +9727,7 @@ export async function verifyRemoteRunnerReattachment(input: { identity: Record; runId: string; normalizedSessionId: string; + runnerBinary?: string; }) { const { claim, target, identity } = input; if ( @@ -9966,7 +9753,7 @@ export async function verifyRemoteRunnerReattachment(input: { createHash("sha256").update(input.normalizedSessionId).digest("hex"), "runner", ); - const runnerState = await readRemoteRunnerState({ runner, stateDirectory }); + const runnerState = await readRemoteRunnerState({ runner, stateDirectory, runnerBinary: input.runnerBinary }); for (const field of [ "runId", "normalizedSessionId", @@ -10319,6 +10106,7 @@ export function createRemoteRunnerProcessLauncher(input: { const durableState = await readRemoteRunnerState({ runner, stateDirectory: input.stateDirectory, + runnerBinary: input.remoteBinary, }).catch(() => null); const lifecycle = typeof durableState?.lifecycle === "string" @@ -10432,6 +10220,8 @@ export async function createRunnerdBackend(input: { }, ) => Promise; }): Promise { + return withNativeAuthorityReadContext(input.db, async () => { + const sessionScopeId = nativeSessionScopeKey(input.execution); const scopeOwner = executingRunnerdSessionScopes.get(sessionScopeId); if (scopeOwner && scopeOwner !== input.execution.binding.runId) { @@ -10474,6 +10264,8 @@ export async function createRunnerdBackend(input: { } finally { initializingSessionToolAuthorities.delete(sessionScopeId); } + + }); } async function createRunnerdBackendWithinSessionClaim( @@ -11261,6 +11053,7 @@ async function createRunnerdBackendWithinSessionClaim( const inspectRemoteHarnessState = async (): Promise<{ complete: boolean; + indexed?: boolean; runnerState: Record | null; providerSessionIdentity: Record | null; incompleteReason: "unavailable" | "not_suspended" | null; @@ -11305,6 +11098,7 @@ async function createRunnerdBackendWithinSessionClaim( }; } let runnerState: Record; + let indexed = false; try { runnerState = record( JSON.parse( @@ -11313,6 +11107,10 @@ async function createRunnerdBackendWithinSessionClaim( ), ), ); + indexed = runnerState.schema === "paperclip.runner.durable.state.indexed.v1"; + runnerState = await resolveRemoteIndexedLocator(runnerState, { + runner: remoteCommandRunner, runnerBinary: remoteBinary ?? undefined, + }, posix.join(remoteStateDirectory, "runner-state.json")); } catch { throw new Error("runner_harness_state_mismatch: runner_state_invalid_json"); } @@ -11336,6 +11134,7 @@ async function createRunnerdBackendWithinSessionClaim( runner: remoteCommandRunner, stateDirectory: remoteStateDirectory, execution: input.execution, + runnerBinary: remoteBinary ?? undefined, }); } catch { throw new Error("runner_harness_state_mismatch: provider_state_unreadable"); @@ -11365,6 +11164,7 @@ async function createRunnerdBackendWithinSessionClaim( } return { complete: true, + indexed, runnerState, providerSessionIdentity, incompleteReason: null, @@ -11476,7 +11276,7 @@ async function createRunnerdBackendWithinSessionClaim( if (!remoteTarget || !remoteCommandRunner) { throw new Error("runner_harness_backup_unavailable"); } - const backup = verifyNativeHarnessBackup({ + const backup = await verifyNativeHarnessBackup({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId, @@ -11906,9 +11706,14 @@ async function createRunnerdBackendWithinSessionClaim( `.pending-${randomUUID()}`, ); mkdirSync(pendingRoot, { recursive: true, mode: 0o700 }); + let nativeSnapshot: Awaited> | undefined; try { + if (verified.indexed) { + if (!remoteBinary) throw new Error("native_indexed_checkpoint_binary_missing"); + nativeSnapshot = await createRemoteIndexedSessionCheckpoint({ runner: remoteCommandRunner, runnerBinary: remoteBinary, directory: remoteStateDirectory }); + } for (const directory of persistenceProfile.directories) { - const sourcePath = remotePersistencePath(directory); + const sourcePath = directory.name === "runner" && nativeSnapshot ? nativeSnapshot.directory : remotePersistencePath(directory); if (!sourcePath) throw new Error("runner_harness_state_mismatch"); const targetPath = resolve(pendingRoot, directory.name); @@ -11923,7 +11728,7 @@ async function createRunnerdBackendWithinSessionClaim( throw new Error("runner_harness_state_mismatch"); } } - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: pendingRoot, execution: input.execution, runnerInstanceId: input.runnerInstanceId, @@ -11944,15 +11749,19 @@ async function createRunnerdBackendWithinSessionClaim( "manifest.json.tmp", ); const manifestPath = resolve(pendingRoot, "manifest.json"); - writeFileSync(temporaryManifest, JSON.stringify(manifest), { - encoding: "utf8", - mode: 0o600, - }); + const manifestFile = await openAsync(temporaryManifest, "wx", 0o600); + try { + await manifestFile.writeFile(JSON.stringify(manifest), "utf8"); + await manifestFile.sync(); + } finally { await manifestFile.close(); } renameSync(temporaryManifest, manifestPath); + await syncNativeHarnessDirectory(pendingRoot); + await syncNativeHarnessDirectory(backupRoot); + await syncNativeHarnessDirectory(root); const currentRoot = resolve(backupRoot, "current"); const previousRoot = resolve(backupRoot, "previous"); - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); let movedCurrent = false; if (existsSync(currentRoot)) { renameSync(currentRoot, previousRoot); @@ -11960,6 +11769,7 @@ async function createRunnerdBackendWithinSessionClaim( } try { renameSync(pendingRoot, currentRoot); + await syncNativeHarnessDirectory(backupRoot); if ( remoteTarget?.transport === "sandbox" && remoteTarget.leaseId @@ -11972,7 +11782,7 @@ async function createRunnerdBackendWithinSessionClaim( } } catch (error) { if (existsSync(currentRoot)) { - removeNativeHarnessBackup(currentRoot); + await removeNativeHarnessBackup(currentRoot); } if ( movedCurrent && @@ -11983,9 +11793,14 @@ async function createRunnerdBackendWithinSessionClaim( } throw error; } - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); + if (nativeSnapshot) { + await nativeSnapshot.dispose().catch(async () => { + await input.onLog?.("stderr", "[paperclip-runner] verified checkpoint published; remote snapshot cleanup will need a retry\n"); + }); + } } finally { - removeNativeHarnessBackup(pendingRoot); + await removeNativeHarnessBackup(pendingRoot); } }, { @@ -12179,6 +11994,7 @@ async function createRunnerdBackendWithinSessionClaim( identity: durableIdentity ?? {}, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), + runnerBinary: remoteBinary ?? undefined, }) : target.kind === "local" && localRecoveryProcess ? { @@ -12230,6 +12046,23 @@ async function createRunnerdBackendWithinSessionClaim( ), codexTransportFactory: (recoveryContext) => createRunnerdCodexTransport({ + // Indexed activation is explicit until the migration and all provider + // recovery lanes have completed qualification. Existing legacy state + // remains readable and is never silently reinterpreted as a locator. + ...(!remoteTarget && input.execution.provider.kind === "codex" ? { + authorityStoreFactory: async (identity, directory) => { + if (!useIndexedNativeAuthority(directory, process.env.PAPERCLIP_NATIVE_INDEXED_STATE === "1")) return undefined; + return createNativePostgresAuthorityStore(input.db, { + companyId: input.execution.binding.companyId, + issueId: input.execution.binding.issueId, + normalizedSessionId: identity.normalizedSessionId, + runId: identity.runId, + runnerInstanceId: identity.runnerInstanceId, + environmentLeaseId: identity.environmentLeaseId, + }, input.execution.binding.agentId); + }, + readAuthorityState: (location) => readPostgresAuthority(input.db, location), + } : {}), onSpawn: input.onSpawn, provider: input.execution.provider.kind === "codex" @@ -12334,6 +12167,7 @@ async function createRunnerdBackendWithinSessionClaim( readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory, + runnerBinary: remoteBinary ?? undefined, }) : undefined, prepareExternalRunnerState, diff --git a/server/src/services/native-runtime/native-tool-receipts.integration.test.ts b/server/src/services/native-runtime/native-tool-receipts.integration.test.ts new file mode 100644 index 00000000000..e2b7229bbf6 --- /dev/null +++ b/server/src/services/native-runtime/native-tool-receipts.integration.test.ts @@ -0,0 +1,286 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { + agents, + companies, + createDb, + heartbeatRuns, + issueComments, + issues, + nativeToolReceiptReferences, + nativeToolReceipts, + type Db, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { + findNativeToolReceiptReference, + insertNativeToolReceipt, + readNativeToolReceipt, + type NativeToolReceipt, + type NativeToolReceiptBinding, +} from "./native-tool-receipts.js"; + +const postgresSupport = await getEmbeddedPostgresTestSupport(); +const describePostgres = postgresSupport.supported ? describe.sequential : describe.skip; + +type Owner = NativeToolReceiptBinding & { agentId: string }; +const digest = (value: string) => createHash("sha256").update(value).digest("hex"); +const receipt = (operationId: string, input: unknown = {}, result: unknown = {}): NativeToolReceipt => ({ operationId, input, result }); + +describePostgres("native tool receipt persistence", () => { + let temporary: Awaited>; + let db: ReturnType; + let owner: Owner; + let other: Owner; + + async function createOwner(companyName: string): Promise { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: companyName, issuePrefix: `${companyName.slice(0, 2).toUpperCase()}${randomUUID().slice(0, 2).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: `${companyName} agent`, status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }); + await db.insert(issues).values({ id: issueId, companyId, title: `${companyName} issue`, status: "in_progress", workMode: "standard", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, + invocationSource: "assignment", triggerDetail: "system", resultJson: { stable: "unchanged" }, contextSnapshot: { issueId } }); + return { companyId, issueId, runId, agentId }; + } + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-tool-receipts-"); + db = createDb(temporary.connectionString); + owner = await createOwner("Receipt primary"); + other = await createOwner("Receipt other"); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + it("keeps thousands of receipts outside result_json and replays the oldest exact key", async () => { + const oldestKey = `oldest-${"legacy-key-".repeat(512)}`; + const oldest = receipt("write_document", { title: "exact" }, { disposition: "applied", documentId: randomUUID() }); + await insertNativeToolReceipt(db, owner, oldestKey, oldest); + const filler = Array.from({ length: 1_200 }, (_, index) => ({ + ...owner, + keySha256: digest(`late-${index}`), + idempotencyKey: `late-${index}`, + receipt: receipt("ordinary_read", { index }, { value: index }), + })); + for (let offset = 0; offset < filler.length; offset += 300) { + await db.insert(nativeToolReceipts).values(filler.slice(offset, offset + 300)); + } + + const replay = await readNativeToolReceipt(db, owner, oldestKey); + expect(replay).toEqual(oldest); + const [count] = await db.select({ count: sql`count(*)::int` }).from(nativeToolReceipts).where(and( + eq(nativeToolReceipts.companyId, owner.companyId), eq(nativeToolReceipts.runId, owner.runId), + )); + const [run] = await db.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, owner.runId)); + expect(count?.count).toBe(1_201); + expect(run?.resultJson).toEqual({ stable: "unchanged" }); + }); + + it("rejects mismatched company and issue ownership through the database foreign key", async () => { + const makeRow = (companyId: string, issueId: string, runId: string, key: string) => ({ + companyId, issueId, runId, keySha256: digest(key), idempotencyKey: key, + receipt: receipt("ordinary_read", {}, {}), + }); + await expect(db.insert(nativeToolReceipts).values(makeRow(other.companyId, owner.issueId, owner.runId, "wrong-company"))).rejects.toThrow(); + await expect(db.insert(nativeToolReceipts).values(makeRow(owner.companyId, other.issueId, owner.runId, "wrong-issue"))).rejects.toThrow(); + }); + + it("fails closed when the digest resolves to a different exact key or a malformed receipt", async () => { + const key = "digest-collision-corruption"; + await db.insert(nativeToolReceipts).values({ ...owner, keySha256: digest(key), idempotencyKey: `${key}-different`, receipt: receipt("ordinary_read") }); + await expect(readNativeToolReceipt(db, owner, key)).rejects.toThrow("native_tool_receipt_binding_mismatch"); + await expect(readNativeToolReceipt(db, owner, "legacy-null", { "legacy-null": null })).rejects.toThrow("native_tool_receipt_invalid"); + await expect(readNativeToolReceipt(db, owner, "legacy-malformed", { "legacy-malformed": { operationId: "missing-fields" } })).rejects.toThrow("native_tool_receipt_invalid"); + + const malformedKey = "persisted-null-jsonb"; + await db.execute(sql`INSERT INTO native_tool_receipts (company_id, issue_id, run_id, key_sha256, idempotency_key, receipt) + VALUES (${owner.companyId}::uuid, ${owner.issueId}::uuid, ${owner.runId}::uuid, ${digest(malformedKey)}, ${malformedKey}, 'null'::jsonb)`); + await expect(readNativeToolReceipt(db, owner, malformedKey)).rejects.toThrow("native_tool_receipt_invalid"); + const sourceCommentId = randomUUID(), attachmentId = randomUUID(); + await insertNativeToolReceipt(db, owner, "malformed-reuse", receipt("reuse_chat_attachment", { sourceCommentId, attachmentId }, null)); + await expect(findNativeToolReceiptReference(db, owner, "attachment-reuse", `${sourceCommentId}/${attachmentId}`)).rejects.toThrow("native_tool_receipt_invalid"); + }); + + it("rolls the business mutation and its receipt back in one transaction", async () => { + const key = "rollback-effect-and-receipt"; + const commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: owner.companyId, issueId: owner.issueId, + authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "rolled back generated comment" }); + const before = await db.select({ title: issues.title }).from(issues).where(eq(issues.id, owner.issueId)).then(rows => rows[0]?.title); + await expect(db.transaction(async tx => { + await tx.update(issues).set({ title: "must roll back" }).where(eq(issues.id, owner.issueId)); + await insertNativeToolReceipt(tx as unknown as Db, owner, key, receipt("report_progress", {}, { commentId })); + throw new Error("abort business transaction"); + })).rejects.toThrow("abort business transaction"); + const [issue] = await db.select({ title: issues.title }).from(issues).where(eq(issues.id, owner.issueId)); + const [count] = await db.select({ count: sql`count(*)::int` }).from(nativeToolReceipts).where(and( + eq(nativeToolReceipts.companyId, owner.companyId), eq(nativeToolReceipts.runId, owner.runId), eq(nativeToolReceipts.idempotencyKey, key), + )); + const [comment] = await db.select({ nativeToolGenerated: issueComments.nativeToolGenerated }).from(issueComments).where(eq(issueComments.id, commentId)); + expect(issue?.title).toBe(before); + expect(count?.count).toBe(0); + expect(comment?.nativeToolGenerated).toBe(false); + }); + + it("keeps indexed publication, reuse, and comment references available after later receipts without tenant leakage", async () => { + const publicationId = randomUUID(); + const reusePublicationId = randomUUID(); + const commentId = randomUUID(); + const progressCommentId = randomUUID(); + const crossRunCommentId = randomUUID(); + const malformedReceiptCommentId = randomUUID(); + const sourceCommentId = randomUUID(); + const attachmentId = randomUUID(); + const crossRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: crossRunId, companyId: owner.companyId, agentId: owner.agentId, + status: "succeeded", runtimeMode: "native", nativeIssueId: owner.issueId, invocationSource: "assignment", triggerDetail: "system" }); + await db.insert(issueComments).values([ + { id: commentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "generated comment" }, + { id: progressCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "progress comment" }, + { id: crossRunCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: crossRunId, body: "comment from another run" }, + { id: malformedReceiptCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "not generated by a valid receipt" }, + ]); + const publication = receipt("register_deliverable", {}, { + attachmentId: publicationId, commandId: `deliverable-prepared:${publicationId}`, disposition: "applied", entityRefs: [publicationId], + }); + const reuse = receipt("reuse_chat_attachment", { sourceCommentId, attachmentId }, { + commandId: `chat-attachment-reused:${reusePublicationId}`, disposition: "duplicate", + entityRefs: [reusePublicationId, commentId, crossRunCommentId], + prepared: { attachmentId: reusePublicationId, commentId }, + }); + const progress = receipt("report_progress", {}, { commentId: progressCommentId }); + const malformed = receipt("register_deliverable", {}, { + attachmentId: reusePublicationId, commandId: "wrong-command", disposition: "applied", + entityRefs: [reusePublicationId, malformedReceiptCommentId], + }); + await insertNativeToolReceipt(db, owner, "indexed-publication", publication); + await insertNativeToolReceipt(db, owner, "indexed-reuse", reuse); + await insertNativeToolReceipt(db, owner, "indexed-comment", progress); + await insertNativeToolReceipt(db, owner, "indexed-malformed-publication", malformed); + await insertNativeToolReceipt(db, other, "same-target-other-tenant", receipt("register_deliverable", {}, { + attachmentId: publicationId, commandId: `deliverable-prepared:${publicationId}`, disposition: "applied", entityRefs: [publicationId], + })); + + const filler = Array.from({ length: 800 }, (_, index) => ({ ...owner, keySha256: digest(`history-${index}`), idempotencyKey: `history-${index}`, receipt: receipt("ordinary_read", {}, { index }) })); + for (let offset = 0; offset < filler.length; offset += 250) await db.insert(nativeToolReceipts).values(filler.slice(offset, offset + 250)); + + expect(await findNativeToolReceiptReference(db, owner, "publication", publicationId)).toEqual(publication); + expect(await findNativeToolReceiptReference(db, owner, "publication", reusePublicationId)).toEqual(reuse); + expect(await findNativeToolReceiptReference(db, owner, "attachment-reuse", `${sourceCommentId}/${attachmentId}`)).toEqual(reuse); + expect(await findNativeToolReceiptReference(db, owner, "generated-comment", commentId)).toEqual(reuse); + expect(await findNativeToolReceiptReference(db, owner, "generated-comment", progressCommentId)).toEqual(progress); + expect(await findNativeToolReceiptReference(db, owner, "generated-comment", crossRunCommentId)).toBeNull(); + expect(await findNativeToolReceiptReference(db, owner, "generated-comment", malformedReceiptCommentId)).toBeNull(); + const [marked] = await db.select({ nativeToolGenerated: issueComments.nativeToolGenerated }).from(issueComments).where(eq(issueComments.id, commentId)); + const [markedProgress] = await db.select({ nativeToolGenerated: issueComments.nativeToolGenerated }).from(issueComments).where(eq(issueComments.id, progressCommentId)); + const [notMarkedCrossRun] = await db.select({ nativeToolGenerated: issueComments.nativeToolGenerated }).from(issueComments).where(eq(issueComments.id, crossRunCommentId)); + const [notMarkedMalformed] = await db.select({ nativeToolGenerated: issueComments.nativeToolGenerated }).from(issueComments).where(eq(issueComments.id, malformedReceiptCommentId)); + expect(marked?.nativeToolGenerated).toBe(true); + expect(markedProgress?.nativeToolGenerated).toBe(true); + expect(notMarkedCrossRun?.nativeToolGenerated).toBe(false); + expect(notMarkedMalformed?.nativeToolGenerated).toBe(false); + expect(await findNativeToolReceiptReference(db, other, "publication", publicationId)).toEqual(expect.objectContaining({ operationId: "register_deliverable" })); + expect(await findNativeToolReceiptReference(db, owner, "publication", randomUUID())).toBeNull(); + const [references] = await db.select({ count: sql`count(*)::int` }).from(nativeToolReceiptReferences).where(eq(nativeToolReceiptReferences.companyId, owner.companyId)); + expect(references?.count).toBeGreaterThanOrEqual(4); + + const generatedComments = Array.from({ length: 1_200 }, (_, index) => ({ companyId: owner.companyId, issueId: owner.issueId, + authorAgentId: owner.agentId, createdByRunId: owner.runId, nativeToolGenerated: true, body: `generated-${index}` })); + for (let offset = 0; offset < generatedComments.length; offset += 300) await db.insert(issueComments).values(generatedComments.slice(offset, offset + 300)); + await db.execute(sql`ANALYZE issue_comments`); + const plan = await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL enable_seqscan = off`); + return tx.execute(sql`EXPLAIN SELECT id FROM issue_comments WHERE company_id = ${owner.companyId}::uuid + AND created_by_run_id = ${owner.runId}::uuid AND issue_id = ${owner.issueId}::uuid AND native_tool_generated = false + ORDER BY created_at DESC, id DESC LIMIT 1`); + }); + expect(JSON.stringify(plan)).toContain("issue_comments_native_run_response_idx"); + }); + + it("backfills exact large and malformed legacy map values, removes only that map, and preserves the rest of result_json", async () => { + const key = `legacy-large-${"k".repeat(8_192)}`; + const targetId = randomUUID(); + const damagedReuseSource = randomUUID(), damagedReuseAttachment = randomUUID(); + const exact = receipt("register_deliverable", { title: "old publication" }, { + attachmentId: targetId, commandId: `deliverable-prepared:${targetId}`, disposition: "applied", entityRefs: [targetId], + }); + const generatedCommentId = randomUUID(); + const userCommentId = randomUUID(); + const wrongIssueCommentId = randomUUID(); + const wrongCompanyCommentId = randomUUID(); + const crossRunId = randomUUID(); + const crossRunCommentId = randomUUID(); + const malformedReceiptCommentId = randomUUID(); + const wrongIssueId = randomUUID(); + await db.insert(issues).values({ id: wrongIssueId, companyId: owner.companyId, title: "Different issue", status: "in_progress", workMode: "standard", assigneeAgentId: owner.agentId }); + await db.insert(heartbeatRuns).values({ id: crossRunId, companyId: owner.companyId, agentId: owner.agentId, + status: "succeeded", runtimeMode: "native", nativeIssueId: owner.issueId, invocationSource: "assignment", triggerDetail: "system" }); + await db.insert(issueComments).values([ + { id: generatedCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "generated by migrated receipt" }, + { id: userCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: null, body: "user or final response" }, + { id: wrongIssueCommentId, companyId: owner.companyId, issueId: wrongIssueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "generated on another issue" }, + { id: wrongCompanyCommentId, companyId: other.companyId, issueId: other.issueId, authorAgentId: other.agentId, createdByRunId: other.runId, body: "generated in another company" }, + { id: crossRunCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: crossRunId, body: "generated by another run" }, + { id: malformedReceiptCommentId, companyId: owner.companyId, issueId: owner.issueId, authorAgentId: owner.agentId, createdByRunId: owner.runId, body: "must remain a final response" }, + ]); + const semanticToolReceipts = { + [key]: exact, + "legacy-generated-comment": receipt("report_progress", {}, { commentId: generatedCommentId }), + "legacy-user-comment": receipt("report_progress", {}, { commentId: userCommentId }), + "legacy-wrong-issue-comment": receipt("report_progress", {}, { commentId: wrongIssueCommentId }), + "legacy-wrong-company-comment": receipt("report_progress", {}, { commentId: wrongCompanyCommentId }), + "legacy-cross-run-comment": receipt("report_progress", {}, { commentId: crossRunCommentId }), + "legacy-malformed-publication": receipt("register_deliverable", {}, { + attachmentId: targetId, commandId: "wrong-command", disposition: "applied", entityRefs: [targetId, malformedReceiptCommentId], + }), + "malformed-null": null, + "legacy-damaged-reuse": receipt("reuse_chat_attachment", { sourceCommentId: damagedReuseSource, attachmentId: damagedReuseAttachment }, null), + }; + await db.update(heartbeatRuns).set({ resultJson: { keep: { marker: "preserved" }, semanticToolReceipts } }).where(eq(heartbeatRuns.id, owner.runId)); + + await db.execute(sql.raw("DROP TABLE native_tool_receipt_references, native_tool_receipts CASCADE")); + const migration = await readFile(resolve(import.meta.dirname, "../../../../packages/db/src/migrations/0300_mushy_sunfire.sql"), "utf8"); + await db.transaction(async tx => { + for (const statement of migration.split("--> statement-breakpoint").map(part => part.trim()).filter(Boolean)) { + await tx.execute(sql.raw(statement)); + } + }); + + // Re-run the actual follow-up migration from its pre-0301 schema state. + await db.execute(sql.raw("DROP INDEX IF EXISTS issue_comments_native_run_response_idx")); + await db.execute(sql.raw("ALTER TABLE issue_comments DROP COLUMN native_tool_generated")); + const generatedCommentMigration = await readFile(resolve(import.meta.dirname, "../../../../packages/db/src/migrations/0301_petite_photon.sql"), "utf8"); + await db.transaction(async tx => { + for (const statement of generatedCommentMigration.split("--> statement-breakpoint").map(part => part.trim()).filter(Boolean)) { + await tx.execute(sql.raw(statement)); + } + }); + + expect(await readNativeToolReceipt(db, owner, key)).toEqual(exact); + await expect(readNativeToolReceipt(db, owner, "malformed-null")).rejects.toThrow("native_tool_receipt_invalid"); + await expect(findNativeToolReceiptReference(db, owner, "attachment-reuse", `${damagedReuseSource}/${damagedReuseAttachment}`)).rejects.toThrow("native_tool_receipt_invalid"); + expect(await findNativeToolReceiptReference(db, owner, "publication", targetId)).toEqual(exact); + const [run] = await db.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, owner.runId)); + expect(run?.resultJson).toEqual({ keep: { marker: "preserved" } }); + const flags = await db.select({ id: issueComments.id, generated: issueComments.nativeToolGenerated }).from(issueComments) + .where(sql`${issueComments.id} IN (${generatedCommentId}::uuid, ${userCommentId}::uuid, ${wrongIssueCommentId}::uuid, ${wrongCompanyCommentId}::uuid, ${crossRunCommentId}::uuid, ${malformedReceiptCommentId}::uuid)`); + expect(Object.fromEntries(flags.map(({ id, generated }) => [id, generated]))).toEqual({ + [generatedCommentId]: true, + [userCommentId]: false, + [wrongIssueCommentId]: false, + [wrongCompanyCommentId]: false, + [crossRunCommentId]: false, + [malformedReceiptCommentId]: false, + }); + const [index] = await db.select({ indexname: sql`indexname` }).from(sql`pg_indexes`).where(sql`indexname = 'issue_comments_native_run_response_idx'`); + expect(index?.indexname).toBe("issue_comments_native_run_response_idx"); + }); +}); diff --git a/server/src/services/native-runtime/native-tool-receipts.ts b/server/src/services/native-runtime/native-tool-receipts.ts new file mode 100644 index 00000000000..414e48c2e04 --- /dev/null +++ b/server/src/services/native-runtime/native-tool-receipts.ts @@ -0,0 +1,117 @@ +import { createHash } from "node:crypto"; +import { and, eq, inArray } from "drizzle-orm"; +import { issueComments, nativeToolReceipts, nativeToolReceiptReferences, type Db } from "@paperclipai/db"; + +export type NativeToolReceipt = { operationId: string; input: unknown; result: unknown }; +export type NativeToolReceiptBinding = { companyId: string; issueId: string; runId: string }; +export type NativeToolReceiptReferenceKind = "publication" | "attachment-reuse" | "generated-comment"; +const record = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const digestKey = (key: string) => createHash("sha256").update(key).digest("hex"); +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + +function validReuseResult(receipt: NativeToolReceipt): boolean { + const result = record(receipt.result); + const prepared = record(result.prepared); + return ["applied", "duplicate"].includes(String(result.disposition)) && + typeof prepared.attachmentId === "string" && UUID.test(prepared.attachmentId) && + Array.isArray(result.entityRefs) && result.entityRefs[0] === prepared.attachmentId && + result.commandId === `chat-attachment-reused:${prepared.attachmentId}`; +} + +function parseReceipt(value: unknown): NativeToolReceipt { + const receipt = record(value); + if (typeof receipt.operationId !== "string" || !receipt.operationId || + !Object.hasOwn(receipt, "input") || !Object.hasOwn(receipt, "result")) { + throw new Error("native_tool_receipt_invalid"); + } + return receipt as NativeToolReceipt; +} + +/** Caller holds the authenticated run lock. A hash collision/corrupt key is an + * integrity failure, never an absent receipt that permits repeating an effect. */ +export async function readNativeToolReceipt(db: Db, binding: NativeToolReceiptBinding, key: string, legacy?: unknown): Promise { + const [row] = await db.select().from(nativeToolReceipts).where(and( + eq(nativeToolReceipts.companyId, binding.companyId), eq(nativeToolReceipts.runId, binding.runId), + eq(nativeToolReceipts.keySha256, digestKey(key)), + )).limit(1); + if (!row) { + const receipts = record(legacy); + return Object.hasOwn(receipts, key) ? parseReceipt(receipts[key]) : null; + } + if (row.issueId !== binding.issueId || row.idempotencyKey !== key) throw new Error("native_tool_receipt_binding_mismatch"); + return parseReceipt(row.receipt); +} + +export function nativeToolReceiptReferencesFor(receipt: NativeToolReceipt): Array<{ kind: NativeToolReceiptReferenceKind; target: string }> { + const input = record(receipt.input); + const result = record(receipt.result); + const references: Array<{ kind: NativeToolReceiptReferenceKind; target: string }> = []; + const add = (kind: NativeToolReceiptReferenceKind, target: unknown) => { + if (typeof target === "string" && UUID.test(target)) references.push({ kind, target }); + }; + if (receipt.operationId === "report_progress") add("generated-comment", result.commentId); + // A known input tuple remains a duplicate-effect fence even when its old + // result is damaged. Lookup must report that damage, never repeat the effect. + if (receipt.operationId === "reuse_chat_attachment" && + typeof input.sourceCommentId === "string" && UUID.test(input.sourceCommentId) && + typeof input.attachmentId === "string" && UUID.test(input.attachmentId)) { + references.push({ kind: "attachment-reuse", target: `${input.sourceCommentId}/${input.attachmentId}` }); + } + if (!["applied", "duplicate"].includes(String(result.disposition))) return references; + const refs = Array.isArray(result.entityRefs) ? result.entityRefs : []; + if (receipt.operationId === "register_deliverable" && + typeof result.attachmentId === "string" && UUID.test(result.attachmentId) && + refs[0] === result.attachmentId && result.commandId === `deliverable-prepared:${result.attachmentId}`) { + add("publication", refs[0]); + // Register-deliverable has no dedicated commentId. Its exact server command + // and attachment binding authenticate the legacy comment-ID references. + for (const target of refs) add("generated-comment", target); + } + const prepared = record(result.prepared); + if (receipt.operationId === "reuse_chat_attachment" && + typeof prepared.attachmentId === "string" && UUID.test(prepared.attachmentId) && + refs[0] === prepared.attachmentId && result.commandId === `chat-attachment-reused:${prepared.attachmentId}`) { + add("publication", refs[0]); + if (typeof prepared.commentId === "string" && refs.includes(prepared.commentId)) add("generated-comment", prepared.commentId); + } + return references.filter((ref, index) => references.findIndex(prior => prior.kind === ref.kind && prior.target === ref.target) === index); +} + +/** Shares the caller's transaction with its actual business effect. */ +export async function insertNativeToolReceipt(db: Db, binding: NativeToolReceiptBinding, key: string, receipt: NativeToolReceipt): Promise { + parseReceipt(receipt); + const keySha256 = digestKey(key); + await db.insert(nativeToolReceipts).values({ ...binding, keySha256, idempotencyKey: key, receipt }); + const references = nativeToolReceiptReferencesFor(receipt); + if (references.length) await db.insert(nativeToolReceiptReferences).values(references.map(ref => ({ + companyId: binding.companyId, runId: binding.runId, keySha256, ...ref, + }))); + const comments = references.filter(ref => ref.kind === "generated-comment").map(ref => ref.target); + if (comments.length) await db.update(issueComments).set({ nativeToolGenerated: true }).where(and( + eq(issueComments.companyId, binding.companyId), eq(issueComments.issueId, binding.issueId), + eq(issueComments.createdByRunId, binding.runId), inArray(issueComments.id, comments), + )); +} + +export async function findNativeToolReceiptReference(db: Db, binding: NativeToolReceiptBinding, + kind: NativeToolReceiptReferenceKind, target: string): Promise { + const [row] = await db.select({ receipt: nativeToolReceipts.receipt, issueId: nativeToolReceipts.issueId, + keySha256: nativeToolReceipts.keySha256, idempotencyKey: nativeToolReceipts.idempotencyKey }) + .from(nativeToolReceiptReferences).innerJoin(nativeToolReceipts, and( + eq(nativeToolReceipts.companyId, nativeToolReceiptReferences.companyId), + eq(nativeToolReceipts.runId, nativeToolReceiptReferences.runId), + eq(nativeToolReceipts.keySha256, nativeToolReceiptReferences.keySha256), + )).where(and( + eq(nativeToolReceiptReferences.companyId, binding.companyId), eq(nativeToolReceiptReferences.runId, binding.runId), + eq(nativeToolReceiptReferences.kind, kind), eq(nativeToolReceiptReferences.target, target), + )).orderBy(nativeToolReceiptReferences.keySha256).limit(1); + if (!row) return null; + if (row.issueId !== binding.issueId || digestKey(row.idempotencyKey) !== row.keySha256) throw new Error("native_tool_receipt_binding_mismatch"); + const receipt = parseReceipt(row.receipt); + if (!nativeToolReceiptReferencesFor(receipt).some(ref => ref.kind === kind && ref.target === target)) { + throw new Error("native_tool_receipt_reference_mismatch"); + } + if (kind === "attachment-reuse" && !validReuseResult(receipt)) throw new Error("native_tool_receipt_invalid"); + return receipt; +} diff --git a/server/src/services/native-runtime/normalized-source-epochs.test.ts b/server/src/services/native-runtime/normalized-source-epochs.test.ts new file mode 100644 index 00000000000..5141dda0854 --- /dev/null +++ b/server/src/services/native-runtime/normalized-source-epochs.test.ts @@ -0,0 +1,76 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, nativeOutputBodyChunks, nativeSourceCursors, nativeSourceEpochs } from "@paperclipai/db"; +import { advanceSourceCursor, normalizedEventId, type PrpEvent, type SourceCursor } from "../../vendor/paperclip-runner/index.js"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; +import { readRunOutputBody } from "../run-output-body.js"; + +it("commits normalized epoch boundaries, rejects forged closes, replays from old cursors and downloads a body spanning epochs", async () => { + const objects = mkdtempSync(join(tmpdir(), "normalized-body-")); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk"); vi.stubEnv("PAPERCLIP_STORAGE_LOCAL_DIR", objects); + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-normalized-epochs-"); + const db = createDb(temporary.connectionString); + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), sourceInstanceId = randomUUID(), sessionId = randomUUID(); + const binding = { companyId, agentId, runId, issueId: randomUUID(), sessionId, sourceInstanceId, + controlPlaneSourceInstanceId: "control", completionContractId: randomUUID(), completionContractSha256: "test" }; + let port = new PaperclipControlPlanePort(db, binding); + const firstEpoch = randomUUID(), secondEpoch = randomUUID(); + const make = (sourceSeq: number, sourceEpoch?: string): PrpEvent => ({ schema: "paperclip.prp.event.v1", schemaVersion: 1, priority: 1, + runId, normalizedSessionId: sessionId, sourceKind: "runner", sourceInstanceId, sourceSeq, ...(sourceEpoch ? { sourceEpoch } : {}), + sourceEventId: normalizedEventId(sourceInstanceId, runId, sourceSeq, sourceEpoch), eventType: "harness.diagnostic", emittedAt: "2026-09-29T00:00:00.000Z", payload: { code: "epoch-test" } }); + const cross = (fromEpoch: string | null, nextEpoch: string, finalOrdinal = 4): PrpEvent => ({ ...make(1, nextEpoch), sourceEpochTransition: { + schema: "paperclip.prp.event-epoch.v1", runId, transitionId: randomUUID(), fromEpoch, nextEpoch, finalOrdinal } }); + const text = "body spanning source epochs ".repeat(500), hash = (v: string) => createHash("sha256").update(v).digest("hex"), bodyId = hash(text); + const body = { schema: "paperclip.output.body.v1", bodyId, sha256: bodyId, byteLength: String(Buffer.byteLength(text)), mediaType: "text/plain; charset=utf-8" }; + const chunk = (event: PrpEvent, offset: number, end: number): PrpEvent => ({ ...event, eventType: "output.body.chunk", payload: { + schema: "paperclip.output.body.chunk.v1", body, offset: String(offset), sha256: hash(text.slice(offset, end)), text: text.slice(offset, end) } }); + const accepted: PrpEvent[] = []; + try { + await db.insert(companies).values({ id: companyId, name: "Normalized epochs", issuePrefix: "NEP" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Epoch agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + // A committed gap is not a prefix and cannot authorize a close. + await port.appendEvent(make(1)); accepted.push(make(1)); + await port.appendEvent(make(3)); + await expect(port.appendEvent(cross(null, firstEpoch, 1))).rejects.toThrow("replay_conflict"); + await port.appendEvent(make(2)); accepted.push(make(2), make(3)); + const finalOld = chunk(make(4), 0, 8000); await port.appendEvent(finalOld); accepted.push(finalOld); + await expect(readRunOutputBody(db, companyId, runId, bodyId)).rejects.toThrow("not complete"); + const firstNew = chunk(cross(null, firstEpoch), 8000, text.length); + expect(await port.appendEvent(firstNew)).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1, highestContiguousSourceEpoch: firstEpoch }); + accepted.push(firstNew); + // Lost ACK: reopen the production port and deliver identical bytes again. + port = new PaperclipControlPlanePort(createDb(temporary.connectionString), binding); + expect(await port.appendEvent(firstNew)).toMatchObject({ disposition: "duplicate", highestContiguousSourceEpoch: firstEpoch }); + expect(await readRunOutputBody(db, companyId, runId, bodyId)).toBe(text); + await expect(port.appendEvent({ ...firstNew, sourceEpochTransition: { ...firstNew.sourceEpochTransition!, transitionId: randomUUID() } })).rejects.toThrow("replay_conflict"); + await expect(port.appendEvent(make(5))).rejects.toThrow("replay_conflict"); + for (let n = 2; n <= 4; n++) { const event = make(n, firstEpoch); await port.appendEvent(event); accepted.push(event); } + const second = cross(firstEpoch, secondEpoch); await port.appendEvent(second); accepted.push(second); + // Earlier identities remain exact after multiple rotations. Reused namespace rejected. + expect(await port.appendEvent(make(1))).toMatchObject({ disposition: "duplicate", highestContiguousSourceSeq: 1, highestContiguousSourceEpoch: secondEpoch }); + await expect(port.appendEvent(cross(secondEpoch, firstEpoch, 1))).rejects.toThrow("replay_conflict"); + let cursor: SourceCursor = { sourceSeq: 0 }; const replayed: PrpEvent[] = []; + for (;;) { + const page = await port.replayEvents({ runId, sourceInstanceId, sourceEpoch: cursor.sourceEpoch, afterSourceSeq: cursor.sourceSeq, limit: 2 }); + if (!page.events.length) break; + for (const event of page.events) { cursor = advanceSourceCursor(cursor, event); replayed.push(event); } + } + expect(replayed.map(event => event.sourceEventId)).toEqual(accepted.map(event => event.sourceEventId)); + expect(cursor).toEqual({ sourceEpoch: secondEpoch, sourceSeq: 1 }); + expect(await db.select().from(nativeSourceEpochs)).toHaveLength(2); + expect(await db.select().from(nativeSourceCursors)).toMatchObject([{ sourceEpoch: secondEpoch, cursor: 1 }]); + expect(await db.select().from(nativeOutputBodyChunks)).toHaveLength(2); + expect(await db.select().from(heartbeatRunEvents)).toHaveLength(9); + // Reading one body is independent of source ordinals, even at the former numeric ceiling. + await db.update(heartbeatRunEvents).set({ sourceSeq: Number.MAX_SAFE_INTEGER }).where(and(eq(heartbeatRunEvents.runId, runId), eq(heartbeatRunEvents.sourceEventId, finalOld.sourceEventId))); + expect(await readRunOutputBody(db, companyId, runId, bodyId)).toBe(text); + await expect(db.execute(sql`insert into native_source_epochs(company_id,run_id,source_instance_id,from_epoch,next_epoch,transition_id,final_ordinal,transition) + values (${randomUUID()},${runId},'other','',${randomUUID()},${randomUUID()},1,'{}')`)).rejects.toThrow(); + } finally { await temporary.cleanup(); vi.unstubAllEnvs(); rmSync(objects, { recursive: true, force: true }); } +}, 60_000); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index bb16da33b0c..bac9d1c6297 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -415,7 +415,9 @@ describe("PaperclipControlPlanePort conformance", () => { const observerDb = createDb(temporary!.connectionString); const observed: Array> = []; const committedCallbacks: string[] = []; - let visibilityCheck: Promise> | null = null; + const observation: { + visibilityCheck: Promise> | null; + } = { visibilityCheck: null }; const unsubscribe = subscribeAllCompanyLiveEvents((event) => { if ( event.companyId !== identity.companyId || @@ -423,8 +425,8 @@ describe("PaperclipControlPlanePort conformance", () => { event.payload.runId !== runId ) return; observed.push(event.payload); - visibilityCheck = observerDb - .select({ eventType: heartbeatRunEvents.eventType }) + observation.visibilityCheck = observerDb + .select({ id: heartbeatRunEvents.id, eventType: heartbeatRunEvents.eventType }) .from(heartbeatRunEvents) .where(eq(heartbeatRunEvents.runId, runId)); }); @@ -484,14 +486,19 @@ describe("PaperclipControlPlanePort conformance", () => { await expect(port.appendEvent(progressEvent)).resolves.toMatchObject({ disposition: "committed", }); - await expect(visibilityCheck).resolves.toEqual([ - { eventType: "tool.execution.started" }, + const visibleRows = await observation.visibilityCheck; + expect(visibleRows).toEqual([ + { id: expect.any(String), eventType: "tool.execution.started" }, ]); expect(observed).toEqual([{ runId, agentId: identity.agentId, issueId, + id: visibleRows![0].id, seq: 1, + eventEpoch: "", + cursor: "1", + commitHint: true, eventType: "tool.execution.started", }]); expect(JSON.stringify(observed)).not.toContain("must-not-cross-live-signal"); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index 328d6098e88..d02e1fc79b7 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -1,9 +1,12 @@ +import { runEventCursor } from "../run-event-history.js"; +import { readRunOutputChunk } from "../run-output-body.js"; import { and, asc, eq, gt, or } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns, issues, + nativeSourceEpochs, nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; @@ -19,6 +22,7 @@ import type { ReplayControlPlaneEventsInput, } from "../../vendor/paperclip-runner/index.js"; import { + advanceSourceCursor, normalizePrpResultSignals, validatePrpEvent, validatePrpStructuredRunResult, @@ -205,6 +209,8 @@ export class PaperclipControlPlanePort implements ControlPlanePort { sourceInstanceId: event.sourceInstanceId, sourceEventId: event.sourceEventId, sourceSeq: event.sourceSeq, + sourceEpoch: event.sourceEpoch, + sourceEpochTransition: event.sourceEpochTransition, protocolSchemaVersion: event.schemaVersion, canonicalPayload: event as unknown as Record, }, @@ -216,6 +222,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort { runId: this.#binding.runId, agentId: this.#binding.agentId, seq: persisted.row.seq, + id: persisted.row.id, + eventEpoch: persisted.row.eventEpoch, + cursor: runEventCursor(persisted.row), eventType: event.eventType, }); await this.#onCommittedEvent?.(event); @@ -227,8 +236,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort { await this.#onDuplicateEvent?.(event); } return { - cursor: persisted.row.seq, + cursor: persisted.row.eventEpoch ? runEventCursor(persisted.row) : persisted.row.seq, highestContiguousSourceSeq: persisted.highestContiguousSourceSeq, + ...(persisted.highestContiguousSourceEpoch ? { highestContiguousSourceEpoch: persisted.highestContiguousSourceEpoch } : {}), disposition: persisted.disposition, }; } @@ -240,27 +250,43 @@ export class PaperclipControlPlanePort implements ControlPlanePort { ) { throw new Error("native_replay_binding_mismatch"); } - const rows = await this.#db + let sourceEpoch = input.sourceEpoch, after = input.afterSourceSeq; + const read = () => this.#db .select({ payload: heartbeatRunEvents.payload, sourceSeq: heartbeatRunEvents.sourceSeq }) .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.runId, this.#binding.runId), - eq(heartbeatRunEvents.sourceInstanceId, input.sourceInstanceId), - gt(heartbeatRunEvents.sourceSeq, input.afterSourceSeq), - )) - .orderBy(asc(heartbeatRunEvents.sourceSeq)) - .limit(Math.max(1, Math.min(input.limit, 1_000))); - const events = rows.flatMap((row) => { - const candidate = row.payload?.prpEvent; - const parsed = validatePrpEvent(candidate); - return parsed.ok ? [parsed.event] : []; - }); - let cursor = input.afterSourceSeq; - for (const event of events) { - if (event.sourceSeq === cursor + 1) cursor += 1; - else if (event.sourceSeq > cursor + 1) break; + .where(and(eq(heartbeatRunEvents.runId, this.#binding.runId), eq(heartbeatRunEvents.companyId, this.#binding.companyId), + eq(heartbeatRunEvents.sourceInstanceId, input.sourceInstanceId), eq(heartbeatRunEvents.sourceEpoch, sourceEpoch ?? ""), gt(heartbeatRunEvents.sourceSeq, after))) + .orderBy(asc(heartbeatRunEvents.sourceSeq)).limit(Math.max(1, Math.min(input.limit, 128))); + let rows = await read(); + if (!rows.length) { + const [close] = await this.#db.select().from(nativeSourceEpochs).where(and(eq(nativeSourceEpochs.runId, input.runId), + eq(nativeSourceEpochs.companyId, this.#binding.companyId), eq(nativeSourceEpochs.sourceInstanceId, input.sourceInstanceId), eq(nativeSourceEpochs.fromEpoch, sourceEpoch ?? ""))).limit(1); + if (close) { + if (close.finalOrdinal !== after) throw new Error("native_replay_epoch_cursor_invalid"); + sourceEpoch = close.nextEpoch; after = 0; rows = await read(); + if (!rows.length) throw new Error("native_replay_epoch_first_event_missing"); + } } - return { events, highestContiguousSourceSeq: cursor }; + const events = await Promise.all(rows.map(async row => { + let candidate = row.payload?.prpEvent; + const envelope = record(candidate); + if (envelope.eventType === "output.body.chunk") { + const { textRef: _, ...payload } = await readRunOutputChunk({ companyId: this.#binding.companyId, runId: this.#binding.runId }, envelope.payload); + candidate = { ...envelope, payload }; + } + const parsed = validatePrpEvent(candidate); + if (!parsed.ok) throw new Error("native_replay_event_invalid"); + return parsed.event; + })); + let cursor = { sourceEpoch: input.sourceEpoch, sourceSeq: input.afterSourceSeq }; + for (const event of events) { + // Gap support is retained; it must not manufacture a contiguous ACK. + if (event.sourceEpoch === cursor.sourceEpoch && event.sourceSeq > cursor.sourceSeq + 1) break; + const next = advanceSourceCursor(cursor, event); + cursor = { sourceEpoch: next.sourceEpoch, sourceSeq: next.sourceSeq }; + } + return { events, highestContiguousSourceSeq: cursor.sourceSeq, + ...(cursor.sourceEpoch ? { highestContiguousSourceEpoch: cursor.sourceEpoch } : {}) }; } async completeRun(value: NativeRunResult | CompleteControlPlaneRunInput): Promise { diff --git a/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts b/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts index 622ed0da3e4..8030ba2db3f 100644 --- a/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts @@ -1,3 +1,4 @@ +import { resolveRunnerCargoTestBinary, resolveRunnerCargoTestBinaryOrDefault } from "./cargo-test-binary.js"; import { existsSync } from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; @@ -10,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createRunnerdCodexTransport, - defaultCapabilityRunnerdBinary, + defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary, } from "../../vendor/paperclip-runner/index.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { @@ -20,12 +21,20 @@ import { } from "../../realtime/runner-prp-ws.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; -const fakeCodexAppServer = resolve( - import.meta.dirname, - "../../../../packages/paperclip-runner/runner/target/debug/fake-codex-app-server", +const runnerWorkspace = resolve(import.meta.dirname, "../../../../packages/paperclip-runner/runner"); +const runnerBinary = () => resolveRunnerCargoTestBinaryOrDefault( + runnerWorkspace, + "debug", + "paperclip-runnerd", + qualifiedCapabilityRunnerdBinary, +); +const fakeCodexAppServer = resolveRunnerCargoTestBinary( + runnerWorkspace, + "debug", + "fake-codex-app-server", ); const runnerBinariesAvailable = - existsSync(defaultCapabilityRunnerdBinary()) && existsSync(fakeCodexAppServer); + existsSync(runnerBinary()) && existsSync(fakeCodexAppServer); const runnerBinaryIt = runnerBinariesAvailable ? it : it.skip; describe("paperclip-runner real server vertical slice", () => { @@ -92,7 +101,7 @@ describe("paperclip-runner real server vertical slice", () => { await writeFile(expectedContextFile, JSON.stringify({ companyId, actorId: agentId, taskId: issueId, runId, callId: "semantic-call-1" })); const fakeArgs = ["--state-file", resolve(stateDirectory, "fake-provider-state.json"), "--emit-tool-call", "--durable-turn-ids", "--durable-tool-ids", "--expected-canonical-task-context-file", expectedContextFile]; const bundle = createRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), + runnerBinary: runnerBinary(), codexCommand: fakeCodexAppServer, codexArgs: fakeArgs, stateDirectory, @@ -163,7 +172,7 @@ describe("paperclip-runner real server vertical slice", () => { }); await writeFile(expectedContextFile, JSON.stringify({ companyId, actorId: agentId, taskId: issueId, runId: resumedRunId, callId: "semantic-call-2" })); const restored = createRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), + runnerBinary: runnerBinary(), codexCommand: fakeCodexAppServer, codexArgs: fakeArgs, stateDirectory, diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 82093f10255..9a831a4dd42 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -17,6 +17,7 @@ import { normalizePaperclipWakePayload, } from "@paperclipai/adapter-utils/server-utils"; import { runnerApiToolsEnabled } from "./runner-api-rollout.js"; +import { findNativeToolReceiptReference, insertNativeToolReceipt, readNativeToolReceipt } from "./native-tool-receipts.js"; import { openRunnerApiWorkspaceFile } from "./runner-api-files.js"; import { basename } from "node:path"; import { createLocalAgentJwt } from "../../agent-auth-jwt.js"; @@ -1252,6 +1253,10 @@ export class PaperclipRunnerToolAuthority { throw new Error("paperclip_runner_tool_mode_denied"); } await authorize(tx, context.run.contextSnapshot); + const indexed = await findNativeToolReceiptReference(tx, this.binding, "attachment-reuse", `${sourceCommentId}/${attachmentId}`); + if (indexed) return { ...record(indexed.result), disposition: "duplicate" }; + // Compatibility for retained pre-migration rows only. New receipts + // never accumulate in the run's JSON checkpoint. const resultJson = record(context.run.resultJson); for (const receipt of Object.values(record(resultJson.semanticToolReceipts))) { const candidate = receipt as ToolReceipt | undefined; @@ -1457,8 +1462,8 @@ export class PaperclipRunnerToolAuthority { }; const resultJson = record(context.run.resultJson); const receipts = record(resultJson.semanticToolReceipts); - const prior = receipts[idempotencyKey] as ToolReceipt | undefined; - if (prior !== undefined) { + const prior = await readNativeToolReceipt(tx as unknown as Db, this.binding, idempotencyKey, receipts); + if (prior != null) { if ( prior.operationId !== operationId || canonicalJson(prior.input) !== canonicalJson(input) @@ -1473,15 +1478,14 @@ export class PaperclipRunnerToolAuthority { describeResult(await effect(tx as unknown as Db, context)), ), ) as unknown; - receipts[idempotencyKey] = { + await insertNativeToolReceipt(tx as unknown as Db, this.binding, idempotencyKey, { operationId, input, result, - } satisfies ToolReceipt; + }); await tx .update(heartbeatRuns) .set({ - resultJson: { ...resultJson, semanticToolReceipts: receipts }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, this.binding.runId)); diff --git a/server/src/services/native-runtime/postgres-authority-store.ts b/server/src/services/native-runtime/postgres-authority-store.ts new file mode 100644 index 00000000000..5de42e566f3 --- /dev/null +++ b/server/src/services/native-runtime/postgres-authority-store.ts @@ -0,0 +1,302 @@ +import { createHash, randomUUID } from "node:crypto"; +import { and, asc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; +import { heartbeatRuns, issues, nativeAuthorityRecords, nativeAuthorityWork, nativeSessionAuthorities, type Db } from "@paperclipai/db"; +import { + validatePrpEvent, authorityInteger, authorityGeneration, authorityJson, validateAuthorityCommit, validateAuthorityPage, DurableAuthorityStoreError, + type AuthorityCommit, type AuthorityPage, type AuthorityRecord, type AuthoritySnapshot, type DurableAuthorityStore, + type AuthorityWorkRecord, type AuthorityWorkPage, validateAuthorityWorkKey, validateAuthorityWorkPage, + materializeCurrentAuthority, +} from "../../vendor/paperclip-runner/index.js"; +import { createHistoryPayloadStore, type HistoryPayloadStore } from "./history-payload-store.js"; +import { classifyNativeAuthorityStorageError } from "./native-authority-storage-error.js"; + +export type AuthorityTransaction = Parameters[0]>[0]; +export interface PostgresAuthorityBinding { + companyId: string; + issueId: string; + normalizedSessionId: string; + runnerInstanceId: string; + environmentLeaseId: string; + runId: string; +} + +const digest = (body: string) => createHash("sha256").update(body).digest("hex"); + +async function retryRolledBackTransaction(operation: () => Promise): Promise { + for (let attempt = 0; ; attempt++) { + try { return await operation(); } + catch (error) { + let cause = error; + let aborted = false; + for (let depth = 0; depth < 4 && cause && typeof cause === "object"; depth++) { + const detail = cause as { code?: unknown; cause?: unknown }; + if (detail.code === "40P01" || detail.code === "40001") { aborted = true; break; } + cause = detail.cause; + } + // PostgreSQL explicitly rolls back these transactions. A connection + // loss/unknown commit result is different and must retain the write fence. + if (!aborted || attempt >= 3) throw error; + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt + Math.floor(Math.random() * 10))); + } + } +} + +/** All queries include company and normalized session. Database foreign keys + * additionally bind records to that company's issue and heartbeat run. */ +export class PostgresAuthorityStore implements DurableAuthorityStore { + readonly unorderedEffectReceipts = true; + readonly commandEpochs = true; + readonly eventEpochs = true; + readonly binding: string; + private epoch: string; + constructor( + private readonly db: Db, + private readonly owner: PostgresAuthorityBinding, + /** Additional transaction-local binding/ingestion checks for the embedding runtime. */ + private readonly commitEvents: (tx: AuthorityTransaction, events: readonly AuthorityRecord[]) => Promise, + /** A migration or maintenance owner must still hold its lease in the + * transaction that publishes authority, not merely before starting I/O. */ + private readonly assertWriteFence?: (tx: AuthorityTransaction) => Promise, + private readonly payloadStore: () => HistoryPayloadStore = createHistoryPayloadStore, + ) { + const { runId, ...session } = owner; + this.binding = authorityJson(session); + this.epoch = runId; + } + + get location(): import("../../vendor/paperclip-runner/index.js").AuthorityLocation { + return { kind: "postgres", binding: this.binding, epoch: this.epoch }; + } + + private authorityScope() { + return and(eq(nativeSessionAuthorities.companyId, this.owner.companyId), eq(nativeSessionAuthorities.normalizedSessionId, this.owner.normalizedSessionId), eq(nativeSessionAuthorities.runId, this.epoch)); + } + + private recordScope(epoch: string, kind: AuthorityRecord["kind"]) { + return and(eq(nativeAuthorityRecords.companyId, this.owner.companyId), eq(nativeAuthorityRecords.issueId, this.owner.issueId), eq(nativeAuthorityRecords.normalizedSessionId, this.owner.normalizedSessionId), eq(nativeAuthorityRecords.runId, epoch), eq(nativeAuthorityRecords.kind, kind)); + } + + private workScope(collection: string) { + return and(eq(nativeAuthorityWork.companyId, this.owner.companyId), eq(nativeAuthorityWork.issueId, this.owner.issueId), + eq(nativeAuthorityWork.normalizedSessionId, this.owner.normalizedSessionId), eq(nativeAuthorityWork.collection, collection)); + } + private decodeWork(row: typeof nativeAuthorityWork.$inferSelect): AuthorityWorkRecord { + if (row.bodySha256 !== digest(row.body)) throw new DurableAuthorityStoreError("invalid_authority", "outstanding-work digest mismatch"); + return { collection: row.collection as AuthorityWorkRecord["collection"], id: row.workId, body: JSON.parse(row.body), sha256: row.bodySha256 }; + } + async getWork(collection: AuthorityWorkRecord["collection"], id: string): Promise { + validateAuthorityWorkKey(collection, id); + const [row] = await this.db.select().from(nativeAuthorityWork).where(and(this.workScope(collection), eq(nativeAuthorityWork.workId, id))); + return row ? this.decodeWork(row) : null; + } + async readWorkPage(collection: AuthorityWorkRecord["collection"], after: string, limit: number, expectedGeneration: string): Promise { + validateAuthorityWorkPage(collection, after, limit, expectedGeneration); + return this.db.transaction(async tx => { + const [authority] = await tx.select().from(nativeSessionAuthorities).where(this.authorityScope()).for("share"); + if (!authority || authority.stateSha256 !== digest(authority.state) || authority.binding !== this.binding || authority.successorRunId !== null || String(authority.generation) !== expectedGeneration) throw new DurableAuthorityStoreError("stale_authority", "outstanding-work page generation changed"); + const rows = await tx.select().from(nativeAuthorityWork).where(and(this.workScope(collection), gt(nativeAuthorityWork.workId, after))) + .orderBy(asc(nativeAuthorityWork.workId)).limit(limit); + return { records: rows.map(row => this.decodeWork(row)), nextAfter: rows.at(-1)?.workId ?? null }; + }); + } + + async load(): Promise { + const [row] = await this.db.select().from(nativeSessionAuthorities).where(this.authorityScope()); + if (!row) return null; + if (row.successorRunId !== null) throw new DurableAuthorityStoreError("stale_authority", "authority epoch has a successor"); + if (row.binding !== this.binding || row.issueId !== this.owner.issueId || row.stateSha256 !== digest(row.state)) throw new DurableAuthorityStoreError("invalid_authority", "current authority binding/digest mismatch"); + authorityGeneration(row.generation); + if (row.committedFrom !== null) authorityGeneration(row.committedFrom); + if (row.generation.startsWith("r:") && row.committedFrom === null) throw new DurableAuthorityStoreError("invalid_authority", "opaque authority commit has no predecessor"); + return { generation: row.generation, ...(row.committedFrom === null ? {} : { committedFrom: row.committedFrom }), state: JSON.parse(row.state) }; + } + + async commit(input: AuthorityCommit): Promise { + try { return await this.commitOnce(input); } + catch (error) { throw classifyNativeAuthorityStorageError(error); } + } + + private async commitOnce(input: AuthorityCommit): Promise { + validateAuthorityCommit(input); + const expected = authorityGeneration(input.expectedGeneration), generation = `r:${randomUUID()}`; + if (generation === expected) throw new DurableAuthorityStoreError("storage_unavailable", "fresh authority revision unavailable"); + const state = authorityJson(input.state); + const identity = input.state.identity as { runId?: string } | undefined; + const nextEpoch = identity?.runId ?? this.epoch; + if (input.records.some((record) => record.epoch !== nextEpoch)) throw new DurableAuthorityStoreError("invalid_authority", "receipt epoch differs from current authority"); + const prepared = new Map(); + // Upload outside the authority transaction. No database lock is held across + // object-store I/O. The transaction below still validates every owner/fence. + for (const record of input.records) { + const body = authorityJson(record.body), bytes = Buffer.byteLength(body), bodySha256 = digest(body); + if (bytes <= 16 * 1024) prepared.set(record, { body, bodySha256, bodyEncoding: "json", bodyBytes: null }); + else { + const [run] = await this.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, this.owner.companyId), eq(heartbeatRuns.nativeIssueId, this.owner.issueId), + eq(heartbeatRuns.nativeSessionId, this.owner.normalizedSessionId), eq(heartbeatRuns.runnerInstanceId, this.owner.runnerInstanceId), + eq(heartbeatRuns.id, record.epoch), eq(heartbeatRuns.runtimeMode, "native"), + )); + if (!run) throw new DurableAuthorityStoreError("invalid_authority", "payload run is not authorized"); + const ref = await this.payloadStore().put({ companyId: this.owner.companyId, runId: record.epoch }, Buffer.from(body), "application/json"); + prepared.set(record, { body: authorityJson(ref), bodySha256, bodyEncoding: "object.v1", bodyBytes: bytes }); + } + } + const committed = await retryRolledBackTransaction(() => this.db.transaction(async (tx) => { + // Take parent locks before touching authority rows or inserting their + // foreign-key references. Otherwise a raw-event writer can hold an + // authority lock / run KEY SHARE while upgrading to the run UPDATE lock + // held by a concurrent event or lifecycle transaction. + const [issue] = await tx.select({ id: issues.id }).from(issues) + .where(and(eq(issues.companyId, this.owner.companyId), eq(issues.id, this.owner.issueId))).for("key share"); + if (!issue) throw new DurableAuthorityStoreError("invalid_authority", "authority issue is not authorized"); + const epochs = [...new Set([this.epoch, nextEpoch])].sort(); + const runs = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, this.owner.companyId), eq(heartbeatRuns.nativeIssueId, this.owner.issueId), + eq(heartbeatRuns.nativeSessionId, this.owner.normalizedSessionId), eq(heartbeatRuns.runnerInstanceId, this.owner.runnerInstanceId), + eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.id, epochs), + )).orderBy(asc(heartbeatRuns.id)).for("update"); + if (runs.length !== epochs.length) throw new DurableAuthorityStoreError("invalid_authority", "authority run is not authorized"); + await this.assertWriteFence?.(tx); + const patch = { generation, committedFrom: expected, state, stateSha256: digest(state), updatedAt: new Date() }; + if (nextEpoch !== this.epoch) { + const sealed = await tx.update(nativeSessionAuthorities).set({ generation, committedFrom: expected, successorRunId: nextEpoch, updatedAt: new Date() }) + .where(and(this.authorityScope(), eq(nativeSessionAuthorities.binding, this.binding), eq(nativeSessionAuthorities.generation, expected), isNull(nativeSessionAuthorities.successorRunId))).returning({ generation: nativeSessionAuthorities.generation }); + if (sealed.length !== 1) throw new DurableAuthorityStoreError("stale_authority", "authority generation or epoch changed"); + await tx.insert(nativeSessionAuthorities).values({ ...this.owner, runId: nextEpoch, binding: this.binding, ...patch }); + } else { + const changed = expected === "0" + ? await tx.insert(nativeSessionAuthorities).values({ ...this.owner, runId: this.epoch, binding: this.binding, ...patch }).onConflictDoNothing().returning({ generation: nativeSessionAuthorities.generation }) + : await tx.update(nativeSessionAuthorities).set(patch).where(and(this.authorityScope(), eq(nativeSessionAuthorities.issueId, this.owner.issueId), eq(nativeSessionAuthorities.binding, this.binding), eq(nativeSessionAuthorities.generation, expected), isNull(nativeSessionAuthorities.successorRunId))).returning({ generation: nativeSessionAuthorities.generation }); + if (changed.length !== 1) throw new DurableAuthorityStoreError("stale_authority", "authority generation or owner changed"); + } + for (const change of input.work ?? []) { + const scope = and(this.workScope(change.collection), eq(nativeAuthorityWork.workId, change.id)); + const [existing] = await tx.select().from(nativeAuthorityWork).where(scope).for("update"); + if (existing && existing.bodySha256 !== digest(existing.body)) throw new DurableAuthorityStoreError("invalid_authority", "outstanding-work digest mismatch"); + if ((existing?.bodySha256 ?? null) !== change.expectedSha256) throw new DurableAuthorityStoreError("stale_authority", "outstanding-work digest changed"); + if (change.body === null) await tx.delete(nativeAuthorityWork).where(scope); + else { + const body = authorityJson(change.body), bodySha256 = digest(body); + if (existing) await tx.update(nativeAuthorityWork).set({ body, bodySha256 }).where(scope); + else await tx.insert(nativeAuthorityWork).values({ companyId: this.owner.companyId, issueId: this.owner.issueId, + normalizedSessionId: this.owner.normalizedSessionId, runId: nextEpoch, collection: change.collection, workId: change.id, body, bodySha256 }); + } + } + for (const record of input.records) { + const stored = prepared.get(record)!; + const [existing] = await tx.select().from(nativeAuthorityRecords).where(and(this.recordScope(record.epoch, record.kind), eq(nativeAuthorityRecords.recordId, record.id))); + if (existing) { + // Retained inline receipts and object-backed receipts have the same + // canonical digest. A storage-format change cannot alter replay. + if (existing.sequenceEpoch !== (record.sequenceEpoch ?? "") || String(existing.sequence) !== record.sequence || existing.bodySha256 !== stored.bodySha256 || + (existing.bodyEncoding === "json" && existing.body !== authorityJson(record.body)) || + (existing.bodyEncoding === "object.v1" && (existing.body !== stored.body || existing.bodyBytes !== stored.bodyBytes)) || + !["json", "object.v1"].includes(existing.bodyEncoding)) throw new DurableAuthorityStoreError("receipt_conflict", "exact receipt differs"); + } else { + if (record.kind !== "effect") { + const [sequenceOwner] = await tx.select({ id: nativeAuthorityRecords.recordId }).from(nativeAuthorityRecords) + .where(and(this.recordScope(record.epoch, record.kind), eq(nativeAuthorityRecords.sequenceEpoch, record.sequenceEpoch ?? ""), eq(nativeAuthorityRecords.sequence, authorityInteger(record.sequence)))); + if (sequenceOwner) throw new DurableAuthorityStoreError("receipt_conflict", "receipt sequence already belongs to another identity"); + } + await tx.insert(nativeAuthorityRecords).values({ + companyId: this.owner.companyId, issueId: this.owner.issueId, normalizedSessionId: this.owner.normalizedSessionId, + runId: record.epoch, kind: record.kind, recordId: record.id, sequence: authorityInteger(record.sequence), sequenceEpoch: record.sequenceEpoch ?? "", ...stored, + }); + } + } + // The immutable raw event inbox and current cursor commit together. + // Paperclip's provider driver translates this stream into a separate + // run-log sequence. Original frames remain available for exact replay. + const events = input.records.filter((record) => record.kind === "event"); + if (events.length) await this.commitEvents(tx, events); + return String(generation); + })); + this.epoch = nextEpoch; + return committed; + } + + private async decode(row: typeof nativeAuthorityRecords.$inferSelect): Promise { + let body = row.body; + if (row.bodyEncoding === "object.v1") { + const ref = JSON.parse(body); + if (ref.mediaType !== "application/json" || ref.sha256 !== row.bodySha256 || ref.byteLength !== row.bodyBytes) throw new DurableAuthorityStoreError("invalid_authority", "receipt reference mismatch"); + body = (await this.payloadStore().read({ companyId: row.companyId, runId: row.runId }, ref, 1024 * 1024)).toString("utf8"); + } else if (row.bodyEncoding !== "json") throw new DurableAuthorityStoreError("invalid_authority", "receipt encoding is unsupported"); + if (row.bodySha256 !== digest(body)) throw new DurableAuthorityStoreError("invalid_authority", "receipt digest mismatch"); + return { epoch: row.runId, kind: row.kind as AuthorityRecord["kind"], id: row.recordId, sequence: String(row.sequence), ...(row.sequenceEpoch ? { sequenceEpoch: row.sequenceEpoch } : {}), body: JSON.parse(body) }; + } + + async getRecord(epoch: string, kind: AuthorityRecord["kind"], id: string): Promise { + const [row] = await this.db.select().from(nativeAuthorityRecords).where(and(this.recordScope(epoch, kind), eq(nativeAuthorityRecords.recordId, id))); + return row ? this.decode(row) : null; + } + + async getSessionEffect(id: string): Promise { + const [row] = await this.db.select().from(nativeAuthorityRecords).where(and( + eq(nativeAuthorityRecords.companyId, this.owner.companyId), eq(nativeAuthorityRecords.issueId, this.owner.issueId), + eq(nativeAuthorityRecords.normalizedSessionId, this.owner.normalizedSessionId), + eq(nativeAuthorityRecords.kind, "effect"), eq(nativeAuthorityRecords.recordId, id), + )); + return row ? this.decode(row) : null; + } + + async readEvents(epoch: string, after: string, limit: number, byteBudget: number, sequenceEpoch?: string): Promise { + validateAuthorityPage(after, limit, byteBudget, sequenceEpoch); + // Fetch bounded metadata first, so a page cannot allocate limit * maximum + // payload size before enforcing its byte budget. Immutable rows cannot + // change between these two queries. + const metadata = await this.db.select({ id: nativeAuthorityRecords.recordId, bytes: sql`coalesce(${nativeAuthorityRecords.bodyBytes}, octet_length(${nativeAuthorityRecords.body}))::integer` }).from(nativeAuthorityRecords) + .where(and(this.recordScope(epoch, "event"), eq(nativeAuthorityRecords.sequenceEpoch, sequenceEpoch ?? ""), gt(nativeAuthorityRecords.sequence, authorityInteger(after)))).orderBy(asc(nativeAuthorityRecords.sequence)).limit(limit); + const ids: string[] = []; + let bytes = 0; + for (const row of metadata) { bytes += row.bytes; if (bytes > byteBudget) break; ids.push(row.id); } + if (!ids.length) return { records: [], nextAfter: null }; + const rows = await this.db.select().from(nativeAuthorityRecords).where(and(this.recordScope(epoch, "event"), inArray(nativeAuthorityRecords.recordId, ids))).orderBy(asc(nativeAuthorityRecords.sequence)); + const records: AuthorityRecord[] = []; + for (const row of rows) records.push(await this.decode(row)); + return { records, nextAfter: records.at(-1)?.sequence ?? null }; + } + + async close(): Promise { /* The application owns the shared pool. */ } +} + +/** Read-only recovery can inspect a sealed epoch, but can never reopen it for writes. */ +export async function readPostgresAuthority(db: Db, location: import("../../vendor/paperclip-runner/index.js").AuthorityLocation): Promise { + if (location.kind !== "postgres" || !location.epoch) throw new DurableAuthorityStoreError("invalid_authority", "Postgres locator required"); + const owner = JSON.parse(location.binding) as PostgresAuthorityBinding; + if (![owner.companyId, owner.issueId, owner.normalizedSessionId, owner.runnerInstanceId, owner.environmentLeaseId].every((value) => typeof value === "string" && value.length > 0)) throw new DurableAuthorityStoreError("invalid_authority", "invalid owner locator"); + const [row] = await db.select().from(nativeSessionAuthorities).where(and( + eq(nativeSessionAuthorities.companyId, owner.companyId), eq(nativeSessionAuthorities.issueId, owner.issueId), + eq(nativeSessionAuthorities.normalizedSessionId, owner.normalizedSessionId), eq(nativeSessionAuthorities.runId, location.epoch), + eq(nativeSessionAuthorities.binding, location.binding), + )); + if (!row || row.stateSha256 !== digest(row.state)) throw new DurableAuthorityStoreError("storage_unavailable", "authority snapshot missing or changed"); + const store = new PostgresAuthorityStore(db, { ...owner, runId: location.epoch }, async () => {}); + const state = await materializeCurrentAuthority(JSON.parse(row.state), store.getRecord.bind(store)); + const [current] = await db.select({ generation: nativeSessionAuthorities.generation, stateSha256: nativeSessionAuthorities.stateSha256 }).from(nativeSessionAuthorities).where(and( + eq(nativeSessionAuthorities.companyId, owner.companyId), eq(nativeSessionAuthorities.normalizedSessionId, owner.normalizedSessionId), eq(nativeSessionAuthorities.runId, location.epoch), + )); + if (current?.generation !== row.generation || current.stateSha256 !== row.stateSha256) throw new DurableAuthorityStoreError("stale_authority", "authority changed while resolving current evidence"); + authorityGeneration(row.generation); + if (row.committedFrom !== null) authorityGeneration(row.committedFrom); + if (row.generation.startsWith("r:") && row.committedFrom === null) throw new DurableAuthorityStoreError("invalid_authority", "opaque authority commit has no predecessor"); + return { generation: row.generation, ...(row.committedFrom === null ? {} : { committedFrom: row.committedFrom }), state }; +} + +export function createNativePostgresAuthorityStore(db: Db, owner: PostgresAuthorityBinding, agentId: string, assertWriteFence?: (tx: AuthorityTransaction) => Promise): PostgresAuthorityStore { + return new PostgresAuthorityStore(db, owner, async (tx, records) => { + for (const record of records) { + const envelope = record.body.envelope as { payload?: unknown } | undefined; + const parsed = validatePrpEvent(envelope?.payload); + if (!parsed.ok) throw new DurableAuthorityStoreError("invalid_authority", "invalid native event record"); + const event = parsed.event; + if (event.sourceKind !== "runner" || event.sourceInstanceId !== owner.runnerInstanceId || event.normalizedSessionId !== owner.normalizedSessionId || event.runId !== record.epoch || event.sourceEventId !== record.id || String(event.sourceSeq) !== record.sequence) throw new DurableAuthorityStoreError("invalid_authority", "native event binding mismatch"); + const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, event.runId), eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.nativeIssueId, owner.issueId), + eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.nativeSessionId, owner.normalizedSessionId), + eq(heartbeatRuns.runnerInstanceId, owner.runnerInstanceId), eq(heartbeatRuns.runtimeMode, "native"), + )).for("update"); + if (!run) throw new DurableAuthorityStoreError("invalid_authority", "native event run is not authorized"); + } + }, assertWriteFence); +} diff --git a/server/src/services/native-runtime/postgres-capacity.qualification.test.ts b/server/src/services/native-runtime/postgres-capacity.qualification.test.ts new file mode 100644 index 00000000000..a4cc6fd1469 --- /dev/null +++ b/server/src/services/native-runtime/postgres-capacity.qualification.test.ts @@ -0,0 +1,135 @@ +import { execFile } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js"; +import { PostgresAuthorityStore } from "./postgres-authority-store.js"; + +const execute = promisify(execFile); + +// This lane consumes only a private container's 16 MiB tmpfs. The production +// disk, shared development database, WAL and other test containers are untouched. +it.skipIf(process.env.PAPERCLIP_HISTORY_PG_CAPACITY_QUALIFICATION !== "1")("retains exact authority and retries after physical PostgreSQL tablespace exhaustion", async () => { + const ownerId = randomUUID(), name = `paperclip-history-capacity-${ownerId}`; + const env = { ...process.env, POSTGRES_PASSWORD: randomBytes(32).toString("hex") }; + const docker = async (args: string[]) => (await execute("docker", args, { env, maxBuffer: 128 * 1024, timeout: 30_000 })).stdout.trim(); + const image = await docker(["image", "inspect", "postgres:17", "--format", "{{.Id}}"]); + if (!/^sha256:[a-f0-9]{64}$/.test(image)) throw new Error("capacity qualification requires an installed immutable image ID"); + let created = false, connectionString: string | undefined, root: string | undefined; + let core: DurablePrpControlPlane | undefined; + let evidence: Record | undefined; + const started = Date.now(); + try { + created = true; + await docker(["create", "--name", name, "--label", `paperclip.history-qualification=${ownerId}`, + "--publish", "127.0.0.1::5432", "--tmpfs", "/var/lib/postgresql/data:rw,size=1g", + "--tmpfs", "/history-capacity:rw,size=16m", "--env", "POSTGRES_PASSWORD", image]); + await docker(["start", name]); + const inspected = JSON.parse(await docker(["inspect", name]))[0]; + const port = inspected.NetworkSettings.Ports["5432/tcp"]?.[0]; + if (inspected.Config.Labels["paperclip.history-qualification"] !== ownerId || port?.HostIp !== "127.0.0.1" || !/^[0-9]+$/.test(port.HostPort)) throw new Error("capacity fixture ownership or port mismatch"); + const deadline = Date.now() + 20_000; + for (;;) { + if (await docker(["exec", name, "pg_isready", "-h", "127.0.0.1", "-U", "postgres"]).then(() => true, () => false)) break; + if (Date.now() >= deadline) throw new Error("capacity fixture readiness timeout"); + await new Promise(resolve => setTimeout(resolve, 100)); + } + connectionString = `postgres://postgres:${env.POSTGRES_PASSWORD}@127.0.0.1:${port.HostPort}/postgres`; + await applyPendingMigrations(connectionString); + const db = createDb(connectionString); + await docker(["exec", name, "mkdir", "/history-capacity/tablespace"]); + await docker(["exec", name, "chown", "postgres:postgres", "/history-capacity/tablespace"]); + await db.execute(sql`create tablespace history_capacity location '/history-capacity/tablespace'`); + await db.execute(sql`alter table native_authority_records set tablespace history_capacity`); + await db.execute(sql`create table capacity_probe (payload text not null) tablespace history_capacity`); + await db.execute(sql`alter table capacity_probe alter column payload set storage external`); + const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + const normalizedSessionId = randomUUID(), runnerInstanceId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Physical capacity qualification", issuePrefix: "CAP" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Capacity fixture", role: "engineer", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Capacity fixture", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, nativeSessionId: normalizedSessionId, runnerInstanceId, lastOutputSeq: 2_147_483_647 }); + const [outputProgress] = await db.update(heartbeatRuns).set({ lastOutputSeq: sql`${heartbeatRuns.lastOutputSeq} + 1` }) + .where(eq(heartbeatRuns.id, runId)).returning({ seq: heartbeatRuns.lastOutputSeq }); + expect(outputProgress?.seq).toBe(2_147_483_648); + const binding = { companyId, issueId, normalizedSessionId, runnerInstanceId, runId, environmentLeaseId: "capacity-fixture" }; + const authority = new PostgresAuthorityStore(db, binding, async () => {}); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 0 }, records: [] }); + // The first insert must extend the real receipt/TOAST relation after the + // current-state update. No SQL exception, mocked driver or error hook is used. + const record = { epoch: runId, kind: "event" as const, id: "capacity-event", sequence: "1", + body: { bytes: randomBytes(7 * 1024).toString("hex") } }; + const next = { expectedGeneration: gen1, state: { cursor: 1 }, records: [record] }; + const exhaust = async () => { + const result = await execute("docker", ["exec", name, "dd", "if=/dev/zero", "of=/history-capacity/ballast", "bs=1048576", "count=32"], { env, timeout: 30_000, maxBuffer: 4096 }).then(() => null, error => error as { code?: number; stderr?: string }); + expect(result?.code).toBe(1); + expect(result?.stderr).toContain("No space left on device"); + }; + const release = () => docker(["exec", name, "rm", "/history-capacity/ballast"]); + await exhaust(); + await expect(authority.commit(next)).rejects.toMatchObject({ code: "storage_pressure" }); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { cursor: 0 } }); + expect(await authority.getRecord(runId, "event", record.id)).toBeNull(); + await release(); + const gen2 = await authority.commit(next); + expect(await authority.load()).toEqual({ generation: gen2, committedFrom: gen1, state: { cursor: 1 } }); + expect(await authority.getRecord(runId, "event", record.id)).toEqual(record); + + // Exercise the actual in-memory control-plane rollback and same-owner + // retry too. Its write fence performs a real allocation in that tablespace; + // the preceding case covers rollback after authority rows have changed. + const controllerBinding = { ...binding, normalizedSessionId: randomUUID(), runId: randomUUID() }; + await db.insert(heartbeatRuns).values({ id: controllerBinding.runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, nativeSessionId: controllerBinding.normalizedSessionId, runnerInstanceId }); + let allocate = false; + const controllerAuthority = new PostgresAuthorityStore(db, controllerBinding, async () => {}, async tx => { + if (allocate) await tx.execute(sql`insert into capacity_probe(payload) values (${randomBytes(128 * 1024).toString("hex")})`); + }); + root = await mkdtemp(join(tmpdir(), "paperclip-history-pg-capacity-")); + core = await DurablePrpControlPlane.open({ stateDirectory: root, authorityStore: controllerAuthority, + identity: { runId: controllerBinding.runId, normalizedSessionId: controllerBinding.normalizedSessionId, + runnerInstanceId, environmentLeaseId: binding.environmentLeaseId, turnId: "turn-1", itemId: "item-1" }, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const before = await controllerAuthority.load(); + allocate = true; + await exhaust(); + await expect(core.issueBootstrapTicket()).rejects.toMatchObject({ code: "storage_pressure" }); + expect(await controllerAuthority.load()).toEqual(before); + expect(Object.keys(core.store.state.tickets)).toHaveLength(0); + await release(); + expect(await core.issueBootstrapTicket()).toBeTruthy(); + expect(Object.keys(core.store.state.tickets)).toHaveLength(1); + const after = (await controllerAuthority.load())!; + expect(after.committedFrom).toBe(before!.generation); + expect(after.generation).not.toBe(before!.generation); + expect(after.generation).toMatch(/^r:/); + const logs = await execute("docker", ["logs", name], { env, timeout: 30_000, maxBuffer: 128 * 1024 }); + expect(logs.stdout + logs.stderr).toContain("No space left on device"); + evidence = { image, backend: "PostgreSQL 17 in an isolated local Docker container", capacityBytes: 16 * 1024 * 1024, + fault: "physical tmpfs ENOSPC, PostgreSQL relation extension", rollbackAfterAuthorityWrite: true, + rejectedReceiptAbsent: true, exactReceiptRetried: true, sameControllerRetried: true, ticketCountAfterRetry: 1, + outputProgressBeyondSigned32Bit: outputProgress!.seq }; + } finally { + try { + await core?.stop(); + } finally { + if (connectionString) await closeRegisteredClients(connectionString); + if (root) await rm(root, { recursive: true, force: true }); + if (created) { + const found = await docker(["inspect", name]).then(value => JSON.parse(value)[0], () => null); + if (found) { + if (found.Config.Labels["paperclip.history-qualification"] !== ownerId) throw new Error("capacity fixture cleanup owner mismatch"); + await docker(["rm", "--force", "--volumes", found.Id]); + } + } + } + } + if (process.env.PAPERCLIP_HISTORY_REPORT) await writeFile(process.env.PAPERCLIP_HISTORY_REPORT, + JSON.stringify({ ...evidence, status: "passed", cleanupVerified: true, elapsedMs: Date.now() - started }, null, 2), { mode: 0o600 }); +}, 180_000); diff --git a/server/src/services/native-runtime/runner-prp-coordinator.test.ts b/server/src/services/native-runtime/runner-prp-coordinator.test.ts index 34b30505405..9b3473a20c7 100644 --- a/server/src/services/native-runtime/runner-prp-coordinator.test.ts +++ b/server/src/services/native-runtime/runner-prp-coordinator.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; @@ -15,6 +15,8 @@ import { heartbeatRunEvents, heartbeatRuns, issues, + nativeAuthorityRecords, + nativeSessionAuthorities, nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; @@ -23,7 +25,7 @@ import type { PrpStructuredRunResult, PrpTerminalState, } from "@paperclipai/paperclip-runner"; -import { NativeSessionProtocolIntegrityError } from "../../vendor/paperclip-runner/index.js"; +import { DurablePrpControlPlane, NativeSessionProtocolIntegrityError } from "../../vendor/paperclip-runner/index.js"; import { getEmbeddedPostgresTestSupport, @@ -33,10 +35,15 @@ import { runnerPrpWebSocketInternals, setupRunnerPrpWebSocketServer, } from "../../realtime/runner-prp-ws.js"; +import { createNativePostgresAuthorityStore, PostgresAuthorityStore } from "./postgres-authority-store.js"; import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js"; import { runnerPrpCoordinator } from "./runner-prp-coordinator.js"; import { PaperclipRunnerSemanticAuthority } from "./runner-semantic-authority.js"; import { nativeSha256 } from "./canonical.js"; +import { claimNativeRestartRecoveries, currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { startNativeLegacyMigrationLease } from "./native-legacy-migration.js"; +import { HistoryPayloadStore } from "./history-payload-store.js"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported @@ -118,6 +125,8 @@ describeEmbeddedPostgres("hidden runner PRP coordinator", () => { afterEach(async () => { runnerPrpWebSocketInternals.resetForTests(); + await db.delete(nativeAuthorityRecords); + await db.delete(nativeSessionAuthorities); await db.delete(nativeRunFinalizations); await db.delete(nativeRunResults); await db.delete(heartbeatRunEvents); @@ -136,6 +145,413 @@ describeEmbeddedPostgres("hidden runner PRP coordinator", () => { await tempDb?.cleanup(); }); + it("retains a migration beyond its original lease and cancels when its exact owner changes", async () => { + const seed = await seedNativeRun(); + const controller = await currentNativeControllerIdentity(); + const leaseOwner = randomUUID(); + await db.insert(nativeRunFinalizations).values({ companyId: seed.companyId, issueId: seed.issueId, runId: seed.runId, + phase: "observed", attempt: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() + 10_000), + controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt }); + const maintenance = await startNativeLegacyMigrationLease({ db, + owner: { companyId: seed.companyId, issueId: seed.issueId, runId: seed.runId, + normalizedSessionId: seed.sessionId, runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1" }, + lease: { owner: leaseOwner, attempt: 1 }, assertStoppedProcessTree: async () => {}, + }, { leaseMs: 2_000, intervalMs: 100, timeoutMs: 500 }); + try { + await new Promise(resolve => setTimeout(resolve, 4_500)); + await maintenance.assertLease(); + const [retained] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, seed.runId)); + expect(retained?.leaseOwner).toBe(leaseOwner); + expect(retained?.leaseExpiresAt?.getTime()).toBeGreaterThan(Date.now()); + await db.update(nativeRunFinalizations).set({ leaseOwner: "replacement-owner" }).where(eq(nativeRunFinalizations.runId, seed.runId)); + await new Promise((resolve, reject) => { + const deadline = setTimeout(() => reject(new Error("migration staging was not cancelled")), 3_000); + const cancelled = () => { clearTimeout(deadline); resolve(); }; + if (maintenance.signal.aborted) cancelled(); + else maintenance.signal.addEventListener("abort", cancelled, { once: true }); + }); + await expect(maintenance.assertLease()).rejects.toThrow("native_legacy_migration_lease_lost"); + const [replaced] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, seed.runId)); + expect(replaced?.leaseOwner).toBe("replacement-owner"); + } finally { await maintenance.stop(); } + }, 15_000); + + it("uses wall-clock migration expiry inside a transaction and never waits on its own renewal lock", async () => { + const seed = await seedNativeRun(); + const controller = await currentNativeControllerIdentity(); + const leaseOwner = randomUUID(); + await db.insert(nativeRunFinalizations).values({ companyId: seed.companyId, issueId: seed.issueId, runId: seed.runId, + phase: "observed", attempt: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() + 10_000), + controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt }); + const maintenance = await startNativeLegacyMigrationLease({ db, + owner: { companyId: seed.companyId, issueId: seed.issueId, runId: seed.runId, + normalizedSessionId: seed.sessionId, runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1" }, + lease: { owner: leaseOwner, attempt: 1 }, assertStoppedProcessTree: async () => {}, + }, { leaseMs: 3_000, intervalMs: 50, timeoutMs: 1_000 }); + try { + await db.transaction(async tx => { + await maintenance.assertLease(tx); + // The timer's other connection now waits for our row lock. Rechecking + // through this transaction must not await that blocked connection. + await tx.execute(sql`select pg_sleep(0.2)`); + await maintenance.assertLease(tx); + }); + await maintenance.assertLease(); + await maintenance.stop(); + await expect(db.transaction(async tx => { + await tx.update(nativeRunFinalizations).set({ leaseExpiresAt: sql`clock_timestamp() + interval '100 milliseconds'` }) + .where(eq(nativeRunFinalizations.runId, seed.runId)); + await tx.execute(sql`select pg_sleep(0.2)`); + await maintenance.assertLease(tx); + })).rejects.toThrow("native_legacy_migration_lease_lost"); + } finally { await maintenance.stop(); } + }, 15_000); + + it.each([0, 1, 260])("requires retirement of indexed owners without replaying historical provider events (owners: %s)", async (ownerCount) => { + const seed = await seedNativeRun(); + await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, seed.agentId)); + await db.insert(nativeRunFinalizations).values({ companyId: seed.companyId, issueId: seed.issueId, runId: seed.runId, phase: "observed" }); + const identity = { runId: seed.runId, normalizedSessionId: seed.sessionId, runnerInstanceId: seed.runnerInstanceId, + environmentLeaseId: "environment-1", turnId: "turn-1", itemId: "item-1" }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: { + identity: { companyId: seed.companyId, issueId: seed.issueId, agentId: seed.agentId, runId: seed.runId, sessionId: seed.sessionId }, + } } }).where(eq(heartbeatRuns.id, seed.runId)); + // A former transcript PID is deliberately live and has no start identity. + // It is history, not the new format's unresolved ownership authority. + await db.insert(heartbeatRunEvents).values({ companyId: seed.companyId, runId: seed.runId, agentId: seed.agentId, + seq: 1, eventType: "session.started", stream: "system", level: "info", payload: { prpEvent: { payload: { processId: process.pid } } } }); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + const authority = new PostgresAuthorityStore(db, owner, async () => {}); + const state = { identity, indexedState: { processOwnerIndexVersion: 1, providerEverStarted: true } }; + let generation = await authority.commit({ expectedGeneration: "0", state, records: [] }); + if (ownerCount) { + for (let offset = 0; offset < ownerCount; offset += 128) { + const work = Array.from({ length: Math.min(128, ownerCount - offset) }, (_, ordinal) => { + const index = offset + ordinal, launchId = `launch-${String(index).padStart(5, "0")}`; + return { collection: "process-owner" as const, id: launchId, expectedSha256: null, + body: { schema: "paperclip.process-owner.v1", identity, sourceSeq: index + 1, sourceEventId: `owner-${index}`, + // No recorded PID survives. This does not prove descendants + // stopped, so the launch still prevents replacement admission. + startup: { launchId, processTreeRetired: false, phase: "spawned", processId: 2147483647, processGroupId: 2147483647 } } }; + }); + generation = await authority.commit({ expectedGeneration: generation, state, records: [], work }); + } + } + const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [seed.runId] }); + expect(result).toMatchObject([ownerCount ? { kind: "blocked", reason: "provider_process_tree_retirement_unproven" } : { kind: "resume_dead_runner" }]); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seed.runId)); + expect(run.status).toBe(ownerCount ? "failed" : "running"); + }); + + it("commits current owner rows with authority and rolls them back with a failed event", async () => { + const seed = await seedNativeRun(); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + const authority = new PostgresAuthorityStore(db, owner, async () => { throw new Error("event commit failed"); }); + const start = { collection: "process-owner" as const, id: "launch-1", expectedSha256: null, body: { phase: "spawned", processId: 9001 } }; + const gen1 = await authority.commit({ expectedGeneration: "0", state: { owners: 1 }, records: [], work: [start] }); + const work = (await authority.getWork("process-owner", "launch-1"))!; + expect((await authority.readWorkPage("process-owner", "", 1, gen1)).records).toEqual([work]); + await expect(authority.commit({ expectedGeneration: gen1, state: { owners: 0 }, records: [{ epoch: seed.runId, kind: "event", id: "stop", sequence: "1", body: { stopped: true } }], + work: [{ ...start, expectedSha256: work.sha256, body: null }] })).rejects.toThrow("event commit failed"); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { owners: 1 } }); + expect(await authority.getWork("process-owner", "launch-1")).toEqual(work); + expect(await authority.getRecord(seed.runId, "event", "stop")).toBeNull(); + await expect(authority.commit({ expectedGeneration: gen1, state: {}, records: [], work: [{ ...start, expectedSha256: "0".repeat(64), body: null }] })).rejects.toThrow("stale_authority"); + const otherCompany = new PostgresAuthorityStore(db, { ...owner, companyId: randomUUID() }, async () => {}); + expect(await otherCompany.getWork("process-owner", "launch-1")).toBeNull(); + const gen2 = await authority.commit({ expectedGeneration: gen1, state: { owners: 0 }, records: [{ epoch: seed.runId, kind: "effect", id: "stop", sequence: "1", body: { stopped: true } }], work: [{ ...start, expectedSha256: work.sha256, body: null }] }); + await expect(authority.readWorkPage("process-owner", "", 1, gen1)).rejects.toThrow("stale_authority"); + expect((await authority.readWorkPage("process-owner", "", 128, gen2)).records).toEqual([]); + }); + + it("keeps the same controller writable after a PostgreSQL capacity rejection proves rollback", async () => { + const seed = await seedNativeRun(); + const root = mkdtempSync(resolve(tmpdir(), "paperclip-pg-capacity-")); + scratchDirectories.push(root); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + let full = false; + const authority = new PostgresAuthorityStore(db, owner, async () => {}, async tx => { + if (full) await tx.execute(sql`do $$begin raise exception 'qualified capacity fault' using errcode = '53100'; end$$`); + }); + const core = await DurablePrpControlPlane.open({ stateDirectory: root, authorityStore: authority, + identity: { runId: seed.runId, normalizedSessionId: seed.sessionId, runnerInstanceId: seed.runnerInstanceId, + environmentLeaseId: "environment-1", turnId: "turn-1", itemId: "item-1" }, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + const before = await authority.load(); + full = true; + await expect(core.issueBootstrapTicket()).rejects.toMatchObject({ code: "storage_pressure" }); + expect(await authority.load()).toEqual(before); + expect(Object.keys(core.store.state.tickets)).toHaveLength(0); + full = false; + expect(await core.issueBootstrapTicket()).toBeTruthy(); + expect(Object.keys(core.store.state.tickets)).toHaveLength(1); + const after = (await authority.load())!; + expect(after.committedFrom).toBe(before!.generation); + expect(after.generation).not.toBe(before!.generation); + expect(after.generation).toMatch(/^r:/); + } finally { await core.stop(); } + }); + + it("rolls back raw receipts and cursors on a PostgreSQL capacity error after their writes", async () => { + const seed = await seedNativeRun(); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + let full = true; + const authority = new PostgresAuthorityStore(db, owner, async tx => { + if (full) await tx.execute(sql`do $$begin raise exception 'qualified capacity fault' using errcode = '53100'; end$$`); + }); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 0 }, records: [] }); + const next = { expectedGeneration: gen1, state: { cursor: 1 }, records: [{ epoch: seed.runId, kind: "event" as const, + id: "event-1", sequence: "1", body: { original: true } }] }; + await expect(authority.commit(next)).rejects.toMatchObject({ code: "storage_pressure" }); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { cursor: 0 } }); + expect(await authority.getRecord(seed.runId, "event", "event-1")).toBeNull(); + full = false; + const gen2 = await authority.commit(next); + expect(await authority.load()).toEqual({ generation: gen2, committedFrom: gen1, state: { cursor: 1 } }); + expect(await authority.getRecord(seed.runId, "event", "event-1")).toEqual(next.records[0]); + }); + + it("commits indexed authority, exact receipts and native events in one Postgres transaction", async () => { + const seed = await seedNativeRun(); + const native = store(seed); + const owner = { + companyId: seed.companyId, issueId: seed.issueId, + normalizedSessionId: seed.sessionId, runnerInstanceId: seed.runnerInstanceId, + environmentLeaseId: "environment-1", runId: seed.runId, + }; + let rejectCommit = false; + const authority = new PostgresAuthorityStore(db, owner, async (tx, records) => { + for (const record of records) await native.appendEventInTransaction(tx, record.body as unknown as PrpEvent); + if (rejectCommit) throw new Error("injected failure before transaction commit"); + }); + const record = (seq: number) => ({ epoch: seed.runId, kind: "event" as const, id: `event-${seq}`, sequence: String(seq), body: runnerEvent(seed, seq) as unknown as Record }); + expect(await authority.load()).toBeNull(); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 1 }, records: [record(1)] }); + rejectCommit = true; + await expect(authority.commit({ expectedGeneration: gen1, state: { cursor: 2 }, records: [record(2)] })).rejects.toThrow("injected failure"); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { cursor: 1 } }); + expect(await authority.getRecord(seed.runId, "event", "event-2")).toBeNull(); + expect((await db.select().from(heartbeatRunEvents)).length).toBe(1); + rejectCommit = false; + await expect(authority.commit({ expectedGeneration: "0", state: {}, records: [] })).rejects.toThrow("stale_authority"); + await expect(authority.commit({ expectedGeneration: gen1, state: {}, records: [{ ...record(1), body: { ...record(1).body, payload: { changed: true } } }] })).rejects.toThrow("receipt_conflict"); + await expect(authority.commit({ expectedGeneration: gen1, state: {}, records: [{ ...record(1), id: "another-event" }] })).rejects.toThrow("receipt_conflict"); + const gen2 = await authority.commit({ expectedGeneration: gen1, state: { cursor: 2 }, records: [record(2)] }); + expect(await authority.load()).toEqual({ generation: gen2, committedFrom: gen1, state: { cursor: 2 } }); + const reopened = new PostgresAuthorityStore(db, owner, async () => {}); + expect(await reopened.getRecord(seed.runId, "event", "event-1")).toEqual(record(1)); + expect((await reopened.readEvents(seed.runId, "1", 128, 1048576)).records).toEqual([record(2)]); + await expect(new PostgresAuthorityStore(db, { ...owner, issueId: randomUUID() }, async () => {}).load()).rejects.toThrow("binding/digest mismatch"); + expect(await new PostgresAuthorityStore(db, { ...owner, companyId: randomUUID() }, async () => {}).load()).toBeNull(); + }); + + it("externalizes large immutable receipts without advancing authority on upload or transaction failure", async () => { + const seed = await seedNativeRun(); + const root = mkdtempSync(resolve(tmpdir(), "paperclip-authority-payloads-")); + scratchDirectories.push(root); + const provider = createLocalDiskStorageProvider(root), payloads = new HistoryPayloadStore(provider); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + let failCommit = false; + const authority = new PostgresAuthorityStore(db, owner, async () => { if (failCommit) throw new Error("commit interrupted"); }, undefined, () => payloads); + const record = (n: number) => ({ epoch: seed.runId, kind: "event" as const, id: `large-${n}`, sequence: String(n), body: { ordinal: n, text: "λ".repeat(300_000) } }); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 0 }, records: [] }); + failCommit = true; + await expect(authority.commit({ expectedGeneration: gen1, state: { cursor: 1 }, records: [record(1)] })).rejects.toThrow("commit interrupted"); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { cursor: 0 } }); + expect(await authority.getRecord(seed.runId, "event", "large-1")).toBeNull(); + failCommit = false; + const gen2 = await authority.commit({ expectedGeneration: gen1, state: { cursor: 2 }, records: [record(1), record(2)] }); + const rows = await db.select().from(nativeAuthorityRecords).orderBy(nativeAuthorityRecords.sequence); + expect(rows).toHaveLength(2); + expect(rows.every(row => row.bodyEncoding === "object.v1" && row.body.length < 1024 && row.bodyBytes! > 600_000)).toBe(true); + const reopened = new PostgresAuthorityStore(db, owner, async () => {}, undefined, () => new HistoryPayloadStore(provider)); + expect(await reopened.getRecord(seed.runId, "event", "large-1")).toEqual(record(1)); + const first = await reopened.readEvents(seed.runId, "0", 128, 1024 * 1024); + expect(first.records).toEqual([record(1)]); + expect((await reopened.readEvents(seed.runId, first.nextAfter!, 128, 1024 * 1024)).records).toEqual([record(2)]); + const gen3 = await authority.commit({ expectedGeneration: gen2, state: { cursor: 2 }, records: [record(1)] }); + await expect(authority.commit({ expectedGeneration: gen3, state: {}, records: [{ ...record(1), body: record(2).body }] })).rejects.toThrow("receipt_conflict"); + const ref = JSON.parse(rows[0]!.body); + writeFileSync(resolve(root, ref.objectKey), "corrupt"); + await expect(reopened.getRecord(seed.runId, "event", "large-1")).rejects.toThrow("history_payload_length_mismatch"); + // A failed immutable upload does not publish the next cursor or its receipt. + const failed = new HistoryPayloadStore({ ...provider, putObject: async () => { throw new Error("storage pressure"); } }); + const pressured = new PostgresAuthorityStore(db, owner, async () => {}, undefined, () => failed); + await expect(pressured.commit({ expectedGeneration: gen3, state: { cursor: 3 }, records: [record(3)] })).rejects.toThrow("storage pressure"); + expect(await authority.load()).toEqual({ generation: gen3, committedFrom: gen2, state: { cursor: 2 } }); + expect(await authority.getRecord(seed.runId, "event", "large-3")).toBeNull(); + }); + + it("keeps raw transport receipts separate from translated run-log sequences", async () => { + const seed = await seedNativeRun(); + const authority = createNativePostgresAuthorityStore(db, { + companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId, + }, seed.agentId); + const event = runnerEvent(seed); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 1 }, records: [{ epoch: seed.runId, kind: "event", id: event.sourceEventId, sequence: "1", body: { envelope: { payload: event } } }] }); + expect(await db.select().from(heartbeatRunEvents)).toHaveLength(0); + const translated = { ...event, eventType: "session.started", payload: { providerSessionId: "translated-session" } } as PrpEvent; + expect((await store(seed).appendEvent(translated)).disposition).toBe("committed"); + expect((await authority.getRecord(seed.runId, "event", event.sourceEventId))?.body).toEqual({ envelope: { payload: event } }); + expect((await store(seed).appendEvent(translated)).disposition).toBe("duplicate"); + }); + + it("checks migration ownership inside the authority transaction and preserves receipts after lease loss", async () => { + const seed = await seedNativeRun(); + await db.insert(nativeRunFinalizations).values({ runId: seed.runId, companyId: seed.companyId, issueId: seed.issueId, phase: "observed", leaseOwner: "migration-owner" }); + const authority = createNativePostgresAuthorityStore(db, { + companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId, + }, seed.agentId, async tx => { + const [row] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, seed.runId)).for("update"); + if (row?.leaseOwner !== "migration-owner") throw new Error("migration lease lost"); + }); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { phase: "prepared" }, records: [] }); + let stale: Promise | undefined; + try { + await db.transaction(async tx => { + await tx.update(nativeRunFinalizations).set({ leaseOwner: "successor-owner" }).where(eq(nativeRunFinalizations.runId, seed.runId)); + // The new lease is still uncommitted when this old writer starts. + // Its transaction must wait and reject, rather than trusting an + // earlier application-level lease check. + const event = runnerEvent(seed); + stale = authority.commit({ expectedGeneration: gen1, state: { phase: "active" }, records: [{ epoch: seed.runId, kind: "event", id: event.sourceEventId, sequence: "1", body: { envelope: { payload: event } } }] }) + .then(() => { throw new Error("stale authority was published"); }, error => { expect(error.message).toContain("migration lease lost"); }); + }); + await stale; + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { phase: "prepared" } }); + expect(await authority.getRecord(seed.runId, "event", runnerEvent(seed).sourceEventId)).toBeNull(); + } finally { await stale; } + }); + + it.each(["40P01", "40001"])("retries a PostgreSQL-aborted %s transaction without exposing partial receipts", async (code) => { + const seed = await seedNativeRun(); + const native = store(seed); + let calls = 0; + const authority = new PostgresAuthorityStore(db, { + companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId, + }, async (tx, records) => { + for (const record of records) await native.appendEventInTransaction(tx, record.body as unknown as PrpEvent); + if (++calls === 1) await tx.execute(sql.raw(`DO $$ BEGIN RAISE EXCEPTION 'qualification rollback' USING ERRCODE = '${code}'; END $$;`)); + }); + const event = runnerEvent(seed); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 1 }, records: [{ epoch: seed.runId, kind: "event", id: event.sourceEventId, sequence: "1", body: event as unknown as Record }] }); + expect(calls).toBe(2); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: "0", state: { cursor: 1 } }); + expect(await db.select().from(nativeAuthorityRecords)).toHaveLength(1); + expect(await db.select().from(heartbeatRunEvents)).toHaveLength(1); + }); + + it("waits for the run lock before acquiring current authority during concurrent ingestion", async () => { + const seed = await seedNativeRun(); + const authority = createNativePostgresAuthorityStore(db, { + companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId, + }, seed.agentId); + const gen1 = await authority.commit({ expectedGeneration: "0", state: { cursor: 0 }, records: [] }); + let writer: Promise | undefined; + try { + await db.transaction(async (tx) => { + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seed.runId)).for("update"); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + const event = runnerEvent(seed); + writer = authority.commit({ expectedGeneration: gen1, state: { cursor: 1 }, records: [{ epoch: seed.runId, kind: "event", id: event.sourceEventId, sequence: "1", body: { envelope: { payload: event } } }] }); + // Observe the actual lock wait, rather than assuming a sleep allowed + // the writer to reach it. The old ordering held current authority here. + const deadline = Date.now() + 5_000; + let blocked = false; + while (!blocked && Date.now() < deadline) { + const rows = await tx.execute(sql`select pid from pg_locks where not granted and ${Number(pid)} = any(pg_blocking_pids(pid))`); + blocked = rows.length > 0; + if (!blocked) await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(blocked).toBe(true); + const rows = await tx.select().from(nativeSessionAuthorities).where(eq(nativeSessionAuthorities.runId, seed.runId)).for("update", { noWait: true }); + expect(rows[0]?.generation).toBe(gen1); + }); + const gen2 = await writer; + expect(await authority.load()).toEqual({ generation: gen2, committedFrom: gen1, state: { cursor: 1 } }); + } finally { + await writer?.catch(() => undefined); + } + }); + + it("crosses the old maximum revision without replacing the run or losing exact receipts", async () => { + const seed = await seedNativeRun(); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + const authority = new PostgresAuthorityStore(db, owner, async () => {}); + const original = { epoch: seed.runId, kind: "event" as const, id: "original", sequence: "1", body: { exact: "original receipt" } }; + await authority.commit({ expectedGeneration: "0", state: { cursor: 1 }, records: [original] }); + // Seed a retained v1 revision at its actual database boundary. Migration + // preserves these decimals, and the first v2 write must not increment it. + const maximum = "9223372036854775807"; + await db.update(nativeSessionAuthorities).set({ generation: maximum, committedFrom: null }).where(eq(nativeSessionAuthorities.runId, seed.runId)); + expect(await authority.load()).toEqual({ generation: maximum, state: { cursor: 1 } }); + const gen1 = await authority.commit({ expectedGeneration: maximum, state: { cursor: 2 }, records: [] }); + expect(gen1).toMatch(/^r:[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/); + expect(await authority.load()).toEqual({ generation: gen1, committedFrom: maximum, state: { cursor: 2 } }); + const gen2 = await authority.commit({ expectedGeneration: gen1, state: { cursor: 3 }, records: [original] }); + expect(gen2).not.toBe(gen1); + await expect(authority.commit({ expectedGeneration: maximum, state: { stale: true }, records: [] })).rejects.toThrow("stale_authority"); + await expect(authority.commit({ expectedGeneration: gen1, state: { stale: true }, records: [] })).rejects.toThrow("stale_authority"); + const reopened = new PostgresAuthorityStore(db, owner, async () => {}); + expect(await reopened.load()).toEqual({ generation: gen2, committedFrom: gen1, state: { cursor: 3 } }); + expect(await reopened.getRecord(seed.runId, "event", "original")).toEqual(original); + const rows = await db.select().from(nativeSessionAuthorities).where(eq(nativeSessionAuthorities.runId, seed.runId)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ runId: seed.runId, normalizedSessionId: seed.sessionId, successorRunId: null }); + }); + + it("stores multiple unordered effects while preserving ordered receipts and old effect identities", async () => { + const seed = await seedNativeRun(); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + const authority = new PostgresAuthorityStore(db, owner, async () => {}); + expect(authority.unorderedEffectReceipts).toBe(true); + const old = { epoch: seed.runId, kind: "effect" as const, id: "old", sequence: "9223372036854775807", body: { exact: "old" } }; + const first = { ...old, id: "first", sequence: "0", body: { exact: "first" } }; + const second = { ...first, id: "second", body: { exact: "second" } }; + const event = { ...first, kind: "event" as const, id: "event", sequence: "1" }; + const generation = await authority.commit({ expectedGeneration: "0", state: { settled: true }, records: [old, first, second, event] }); + const reopened = new PostgresAuthorityStore(db, owner, async () => {}); + for (const receipt of [old, first, second]) expect(await reopened.getSessionEffect(receipt.id)).toEqual(receipt); + for (const conflicting of [{ ...first, body: { exact: "changed" } }, { ...old, sequence: "0" }, { ...event, id: "event-alias" }]) { + await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [conflicting] })).rejects.toThrow("receipt_conflict"); + } + expect(await reopened.load()).toMatchObject({ generation, state: { settled: true } }); + expect(await reopened.getRecord(seed.runId, "event", "event-alias")).toBeNull(); + }); + + it.each(["command", "event"] as const)("indexes %s ordinals within renewable epochs while keeping exact identity", async kind => { + const seed = await seedNativeRun(); + const owner = { companyId: seed.companyId, issueId: seed.issueId, normalizedSessionId: seed.sessionId, + runnerInstanceId: seed.runnerInstanceId, environmentLeaseId: "environment-1", runId: seed.runId }; + const authority = new PostgresAuthorityStore(db, owner, async () => {}); + expect(authority.commandEpochs).toBe(true); + const original = { epoch: seed.runId, kind, id: "original", sequence: "1", body: { exact: true } }; + const next = { ...original, id: "next", sequenceEpoch: randomUUID() }; + const third = { ...next, id: "third", sequenceEpoch: randomUUID() }; + const generation = await authority.commit({ expectedGeneration: "0", state: {}, records: [original, next, third] }); + const reopened = new PostgresAuthorityStore(db, owner, async () => {}); + for (const record of [original, next, third]) expect(await reopened.getRecord(seed.runId, kind, record.id)).toEqual(record); + for (const record of [{ ...original, sequenceEpoch: randomUUID() }, { ...next, id: "alias" }]) { + await expect(reopened.commit({ expectedGeneration: generation, state: {}, records: [record] })).rejects.toThrow("receipt_conflict"); + } + if (kind === "event") { + for (const record of [original, next, third]) expect((await reopened.readEvents(seed.runId, "0", 128, 1024 * 1024, "sequenceEpoch" in record && typeof record.sequenceEpoch === "string" ? record.sequenceEpoch : undefined)).records).toEqual([record]); + } + expect((await reopened.load())?.generation).toBe(generation); + }); + async function seedNativeRun(): Promise { const companyId = randomUUID(); const issueId = randomUUID(); @@ -393,6 +809,29 @@ describeEmbeddedPostgres("hidden runner PRP coordinator", () => { }); }); + it("settles the exact earlier turn across source/public epochs and rejects conflicting old replay", async () => { + const seed = await seedNativeRun(); + const nativeStore = store(seed); + const first = runnerEvent(seed); + await nativeStore.appendEvent(first); + const proposed: PrpEvent = { ...runnerEvent(seed, 2), eventType: "run.result.proposed", payload: result }; + await nativeStore.appendEvent(proposed); + const sourceEpoch = randomUUID(); + const terminalEvent: PrpEvent = { ...runnerEvent(seed, 1), sourceEpoch, sourceEventId: `terminal-${sourceEpoch}`, + sourceEpochTransition: { schema: "paperclip.prp.event-epoch.v1", runId: seed.runId, transitionId: randomUUID(), fromEpoch: null, nextEpoch: sourceEpoch, finalOrdinal: 2 }, + eventType: "run.terminal", payload: terminal }; + await db.update(heartbeatRuns).set({ nextEventSeq: 1_000_001 }).where(eq(heartbeatRuns.id, seed.runId)); + const accepted = await nativeStore.appendEvent(terminalEvent); + expect(accepted).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1, highestContiguousSourceEpoch: sourceEpoch }); + expect(accepted.cursor).toMatch(/^e:/); + expect(await store(seed).appendEvent(first)).toMatchObject({ disposition: "duplicate", cursor: 1, highestContiguousSourceSeq: 1, highestContiguousSourceEpoch: sourceEpoch }); + await expect(nativeStore.appendEvent({ ...terminalEvent, priority: 2 })).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError); + // A later proposal must not replace the result selected by the old terminal. + await nativeStore.appendEvent({ ...runnerEvent(seed, 2), sourceEpoch, sourceEventId: `later-${sourceEpoch}`, eventType: "run.result.proposed", payload: { ...result, summary: "later" } }); + expect(await store(seed).reconcileTerminalEvent(terminalEvent)).toMatchObject({ disposition: "committed" }); + expect((await store(seed).readCompletedRun())?.result.summary).toBe(result.summary); + }); + it("persists events and results idempotently and leases finalization", async () => { const seed = await seedNativeRun(); const nativeStore = store(seed); diff --git a/server/src/services/native-runtime/runner-prp-coordinator.ts b/server/src/services/native-runtime/runner-prp-coordinator.ts index b71c9f715a3..522f748010d 100644 --- a/server/src/services/native-runtime/runner-prp-coordinator.ts +++ b/server/src/services/native-runtime/runner-prp-coordinator.ts @@ -55,7 +55,7 @@ export interface PreparedRunnerPrpSession { type: string, payload?: Record, commandId?: string, - ): { readonly commandId: string; readonly controllerSeq: number }; + ): Promise<{ readonly commandId: string; readonly controllerSeq: number }>; completeRun(input: { readonly result: PrpStructuredRunResult; readonly terminal: PrpTerminalState; @@ -321,8 +321,8 @@ export function runnerPrpCoordinator( try { releaseQuestionTarget = registerNativeQuestionCommandTarget({ binding: storeBinding, - queueCommand: (type, payload = {}, commandId) => { - const command = authority.queueCommand(type, payload, commandId, true); + queueCommand: async (type, payload = {}, commandId) => { + const command = (await authority.queueCommand(type, payload, commandId, true)); return { commandId: command.commandId, controllerSeq: command.controllerSeq }; }, }); @@ -335,7 +335,7 @@ export function runnerPrpCoordinator( } let bootstrapTicket: string; try { - bootstrapTicket = authority.issueBootstrapTicket(bootstrapTtlMs); + bootstrapTicket = (await authority.issueBootstrapTicket(bootstrapTtlMs)); } catch (error) { releaseQuestionTarget(); authority.disconnectActiveRunner(); @@ -347,14 +347,14 @@ export function runnerPrpCoordinator( connectUrl: registration.connectUrl, bootstrapTicket, semanticTools, - queueCommand: (type, payload = {}, commandId) => { + queueCommand: async (type, payload = {}, commandId) => { if (released) throw new Error("runner_prp_session_released"); - const command = authority.queueCommand( + const command = (await authority.queueCommand( type, payload, commandId, true, - ); + )); return { commandId: command.commandId, controllerSeq: command.controllerSeq, @@ -399,7 +399,7 @@ export function runnerPrpCoordinator( if (released) throw new Error("runner_prp_session_released"); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - const outcome = authority.commandOutcome(commandId); + const outcome = (await authority.commandOutcome(commandId)); if (!outcome) throw new Error(`runner_prp_command_missing:${commandId}`); if (outcome.status === "completed") return outcome.result; if (outcome.status === "failed" || outcome.status === "rejected") { diff --git a/server/src/services/native-runtime/runtime-request-resolution-authority.test.ts b/server/src/services/native-runtime/runtime-request-resolution-authority.test.ts index ed9922154d7..c35e325d5b6 100644 --- a/server/src/services/native-runtime/runtime-request-resolution-authority.test.ts +++ b/server/src/services/native-runtime/runtime-request-resolution-authority.test.ts @@ -1,6 +1,12 @@ import type { Db } from "@paperclipai/db"; import { describe, expect, it, vi } from "vitest"; +const { readRunEventLane, runEventLane } = vi.hoisted(() => ({ + readRunEventLane: vi.fn(), + runEventLane: vi.fn((_kind: string, key: string) => `request:${key}`), +})); +vi.mock("../run-event-history.js", () => ({ readRunEventLane, runEventLane })); + import { assertNativeRuntimeRequestResolverAuthorized, NativeRuntimeRequestResolutionAuthorizationError, @@ -8,16 +14,7 @@ import { type PendingNativeRuntimeRequest, } from "./runtime-request-resolution-authority.js"; -function dbReturning(row: Record | null): Db { - const limit = vi.fn(async () => row ? [row] : []); - const query = { - from: vi.fn().mockReturnThis(), - where: vi.fn().mockReturnThis(), - orderBy: vi.fn().mockReturnThis(), - limit, - }; - return { select: vi.fn(() => query) } as unknown as Db; -} +function dbReturning(_row: Record | null): Db { return {} as Db; } const binding = { companyId: "company-1", @@ -49,10 +46,20 @@ function createdEvent(requestKind: string) { } describe("native runtime request resolution authority", () => { + it("reads only the request lane and enforces company binding", async () => { + readRunEventLane.mockResolvedValueOnce([{ ...createdEvent("command_approval"), companyId: binding.companyId }]); + await expect(readPendingNativeRuntimeRequest(dbReturning(null), binding)).resolves.toMatchObject({ requestKind: "command_approval" }); + expect(readRunEventLane).toHaveBeenLastCalledWith(expect.anything(), binding.runId, expect.stringMatching(/^request:/), 1); + + readRunEventLane.mockResolvedValueOnce([{ ...createdEvent("command_approval"), companyId: "other-company" }]); + await expect(readPendingNativeRuntimeRequest(dbReturning(null), binding)).resolves.toBeNull(); + }); + it("derives privileged approval policy from the durable canonical request", async () => { + readRunEventLane.mockResolvedValueOnce([{ ...createdEvent("command_approval"), companyId: binding.companyId }]); await expect( readPendingNativeRuntimeRequest( - dbReturning(createdEvent("command_approval")), + dbReturning(null), binding, ), ).resolves.toEqual({ @@ -64,11 +71,13 @@ describe("native runtime request resolution authority", () => { }); it("treats a terminal latest event as no longer pending", async () => { + readRunEventLane.mockResolvedValueOnce([{ + companyId: binding.companyId, + eventType: "runtime_request.resolved", + payload: { prpEvent: { payload: { requestId: binding.requestId } } }, + }]); await expect( - readPendingNativeRuntimeRequest(dbReturning({ - eventType: "runtime_request.resolved", - payload: { prpEvent: { payload: { requestId: binding.requestId } } }, - }), binding), + readPendingNativeRuntimeRequest(dbReturning(null), binding), ).resolves.toBeNull(); }); diff --git a/server/src/services/native-runtime/runtime-request-resolution-authority.ts b/server/src/services/native-runtime/runtime-request-resolution-authority.ts index 6477ffcdcd0..697f4fd4c27 100644 --- a/server/src/services/native-runtime/runtime-request-resolution-authority.ts +++ b/server/src/services/native-runtime/runtime-request-resolution-authority.ts @@ -1,8 +1,6 @@ -import { and, desc, eq, inArray, sql } from "drizzle-orm"; - import type { Db } from "@paperclipai/db"; -import { heartbeatRunEvents } from "@paperclipai/db"; import type { HarnessRuntimeRequestKind } from "../../vendor/paperclip-runner/index.js"; +import { readRunEventLane, runEventLane } from "../run-event-history.js"; const TERMINAL_RUNTIME_REQUEST_EVENTS = [ "runtime_request.resolved", @@ -120,24 +118,18 @@ export async function readPendingNativeRuntimeRequest( }, ): Promise { if (!REQUEST_ID_PATTERN.test(input.requestId)) return null; - const [latest] = await db - .select({ - eventType: heartbeatRunEvents.eventType, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.companyId, input.companyId), - eq(heartbeatRunEvents.runId, input.runId), - inArray(heartbeatRunEvents.eventType, [...RUNTIME_REQUEST_EVENTS]), - sql`coalesce( - ${heartbeatRunEvents.payload} #>> '{prpEvent,payload,request,requestId}', - ${heartbeatRunEvents.payload} #>> '{prpEvent,payload,requestId}' - ) = ${input.requestId}`, - )) - .orderBy(desc(heartbeatRunEvents.seq)) - .limit(1); - if (!latest || latest.eventType !== "runtime_request.created") return null; + const [latest] = await readRunEventLane( + db, + input.runId, + runEventLane("request", input.requestId), + 1, + ); + if ( + !latest + || latest.companyId !== input.companyId + || !RUNTIME_REQUEST_EVENTS.includes(latest.eventType as typeof RUNTIME_REQUEST_EVENTS[number]) + || latest.eventType !== "runtime_request.created" + ) return null; return canonicalPendingRequest({ ...input, payload: latest.payload }); } diff --git a/server/src/services/native-runtime/stopped-codex-rollout.test.ts b/server/src/services/native-runtime/stopped-codex-rollout.test.ts new file mode 100644 index 00000000000..9b1e7d74b15 --- /dev/null +++ b/server/src/services/native-runtime/stopped-codex-rollout.test.ts @@ -0,0 +1,42 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, open, rm, appendFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { inspectStoppedCodexRollout } from "./stopped-codex-rollout.js"; + +const binding = { threadId: "thread", turnId: "turn", cwd: "/workspace" }; +const row = (type: string, payload: unknown) => JSON.stringify({ type, payload }) + "\n"; +const meta = row("session_meta", { id: "thread", cwd: "/workspace" }); +const start = row("event_msg", { type: "task_started", turn_id: "turn" }); +const context = row("turn_context", { turn_id: "turn", cwd: "/workspace" }); +const stop = row("event_msg", { type: "turn_aborted", turn_id: "turn", reason: "interrupted" }); + +it("inventories a transcript beyond the old 32 MiB limit with an exact raw-byte digest", async () => { + const root = await mkdtemp(join(tmpdir(), "stopped-rollout-")), path = join(root, "rollout.jsonl"); + try { + const file = await open(path, "wx", 0o600), hash = createHash("sha256"); + const block = row("response_item", { type: "message", text: "x".repeat(1024) }).repeat(32); + try { + await file.writeFile(meta); hash.update(meta); + // Old transcript bytes do not become current replay authority. + for (let index = 0; index < 1100; index++) { await file.writeFile(block); hash.update(block); } + const tail = start + context + block + stop; await file.writeFile(tail); hash.update(tail); + } finally { await file.close(); } + const expected = hash.digest("hex"); + expect(await inspectStoppedCodexRollout(path, binding)).toEqual({ safe: true, sha256: expected }); + expect((await inspectStoppedCodexRollout(path)).sha256).toBe(expected); + await appendFile(path, row("response_item", { type: "function_call" })); + expect((await inspectStoppedCodexRollout(path, binding)).safe).toBe(false); + } finally { await rm(root, { recursive: true, force: true }); } +}, 30_000); + +it("refuses partial tails and unknown current actions", async () => { + const root = await mkdtemp(join(tmpdir(), "stopped-rollout-invalid-")), path = join(root, "rollout.jsonl"); + try { + await writeFile(path, meta + start + context + stop.trimEnd()); + expect((await inspectStoppedCodexRollout(path, binding)).safe).toBe(false); + await writeFile(path, meta + start + context + row("response_item", { type: "function_call" }) + stop); + expect((await inspectStoppedCodexRollout(path, binding)).safe).toBe(false); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/server/src/services/native-runtime/stopped-codex-rollout.ts b/server/src/services/native-runtime/stopped-codex-rollout.ts new file mode 100644 index 00000000000..21035ee8396 --- /dev/null +++ b/server/src/services/native-runtime/stopped-codex-rollout.ts @@ -0,0 +1,36 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open, lstat } from "node:fs/promises"; +import { createStoppedCodexTurnInventory } from "./stopped-codex-turn.js"; + +const MAX_RECORD_BYTES = 4 * 1024 * 1024; +const unchanged = (a: Awaited>, b: Awaited>) => + a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs; + +/** Closed JSONL inventory and raw-byte digest with no total transcript limit. + * The record bound is per provider frame; it never counts historical bytes. */ +export async function inspectStoppedCodexRollout(path: string, input?: Parameters[0]): Promise<{ safe: boolean; sha256: string }> { + const before = await lstat(path); + if (!before.isFile() || before.isSymbolicLink()) throw new Error("native_crash_inventory_unproven"); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK), hash = createHash("sha256"); + const inventory = input ? createStoppedCodexTurnInventory(input) : null; + let pending: Buffer = Buffer.alloc(0), valid = true; + try { + if (!unchanged(before, await file.stat())) throw new Error("native_crash_inventory_unproven"); + for await (const raw of file.createReadStream({ highWaterMark: 64 * 1024, autoClose: false })) { + const chunk = raw as Buffer; hash.update(chunk); + if (!inventory) continue; + pending = pending.length ? Buffer.concat([pending, chunk]) : chunk; + let offset = 0, newline: number; + while ((newline = pending.indexOf(10, offset)) >= 0) { + if (newline - offset > MAX_RECORD_BYTES || !inventory.push(JSON.parse(pending.subarray(offset, newline).toString("utf8")))) valid = false; + offset = newline + 1; + if (!valid) return { safe: false, sha256: "" }; + } + pending = Buffer.from(pending.subarray(offset)); + if (pending.length > MAX_RECORD_BYTES) return { safe: false, sha256: "" }; + } + if (!unchanged(before, await file.stat()) || !unchanged(before, await lstat(path))) throw new Error("native_crash_inventory_unproven"); + return { safe: !inventory || (valid && pending.length === 0 && inventory.finish()), sha256: hash.digest("hex") }; + } finally { await file.close(); } +} diff --git a/server/src/services/native-runtime/stopped-codex-turn.ts b/server/src/services/native-runtime/stopped-codex-turn.ts index 32c498fd16e..165068d7e1c 100644 --- a/server/src/services/native-runtime/stopped-codex-turn.ts +++ b/server/src/services/native-runtime/stopped-codex-turn.ts @@ -59,32 +59,40 @@ function completionScriptInput(source: unknown): unknown { * This is deliberately a closed inventory, not a search for known bad tools. * The caller must independently authenticate the session and contain its processes. */ -export function stoppedCodexTurnIsTextOnly(input: { - rows: unknown[]; +interface StoppedTurnInput { threadId: string; turnId: string; cwd: string; completedTaskControlCalls?: CompletedTaskControlCall[]; -}): boolean { - const rows = input.rows.map(record); - const meta = rows[0]; - if (!input.threadId || !input.turnId || meta?.type !== "session_meta" || - record(meta.payload).id !== input.threadId || record(meta.payload).cwd !== input.cwd) - return false; - const starts = rows.flatMap((row, index) => row.type === "event_msg" && - record(row.payload).type === "task_started" && record(row.payload).turn_id === input.turnId - ? [index] : []); - if (starts.length !== 1) return false; - const turn = rows.slice(starts[0]!); +} + +export function stoppedCodexTurnIsTextOnly(input: StoppedTurnInput & { rows: unknown[] }): boolean { + const inventory = createStoppedCodexTurnInventory(input); + for (const row of input.rows) if (!inventory.push(row)) return false; + return inventory.finish(); +} + +/** Streaming equivalent of the closed turn inventory. Historical rows before + * the exact turn are discarded; accepted completion IDs bound every set. */ +export function createStoppedCodexTurnInventory(input: StoppedTurnInput) { + let metaSeen = false, started = false, valid = true; let contextSeen = false; let aborted = false; const completedCalls = input.completedTaskControlCalls ?? []; const scripts = new Set(); const seenCalls = new Set(); const outputs = new Set(); - for (let index = 0; index < turn.length; index++) { - const row = turn[index]!; - const payload = record(row.payload); + const accept = (value: unknown): boolean => { + const row = record(value), payload = record(row.payload); + if (!metaSeen) { + metaSeen = true; + return Boolean(input.threadId && input.turnId && row.type === "session_meta" && payload.id === input.threadId && payload.cwd === input.cwd); + } + let first = false; + if (!started) { + if (row.type !== "event_msg" || payload.type !== "task_started" || payload.turn_id !== input.turnId) return true; + started = true; first = true; + } if (aborted) return false; switch (row.type) { case "turn_context": @@ -94,7 +102,7 @@ export function stoppedCodexTurnIsTextOnly(input: { case "response_item": if (payload.type === "custom_tool_call") { try { - if (payload.name !== "exec" || typeof payload.call_id !== "string" || scripts.has(payload.call_id) || + if (payload.name !== "exec" || typeof payload.call_id !== "string" || scripts.has(payload.call_id) || scripts.size >= completedCalls.length || !completedCalls.some(call => canonicalNativeJson(call.input) === canonicalNativeJson(completionScriptInput(payload.input)))) return false; } catch { return false; } scripts.add(payload.call_id as string); @@ -113,7 +121,7 @@ export function stoppedCodexTurnIsTextOnly(input: { case "event_msg": switch (payload.type) { case "task_started": - if (index !== 0 || payload.turn_id !== input.turnId) return false; + if (!first || payload.turn_id !== input.turnId) return false; break; case "turn_aborted": if (payload.turn_id !== input.turnId || payload.reason !== "interrupted") return false; @@ -136,6 +144,10 @@ export function stoppedCodexTurnIsTextOnly(input: { break; default: return false; } - } - return contextSeen && aborted && scripts.size === outputs.size && scripts.size === completedCalls.length && seenCalls.size === completedCalls.length; + return true; + }; + return { + push(value: unknown): boolean { valid = valid && accept(value); return valid; }, + finish(): boolean { return valid && metaSeen && started && contextSeen && aborted && scripts.size === outputs.size && scripts.size === completedCalls.length && seenCalls.size === completedCalls.length; }, + }; } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 30130537938..bda6f4fc4bb 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -10,6 +10,16 @@ type RunnerModule = typeof import("@paperclipai/paperclip-runner"); export type { + EventEpochTransition, + SourceCursor, + AuthorityCommit, + AuthorityPage, + AuthorityRecord, + AuthoritySnapshot, + AuthorityWorkRecord, + AuthorityWorkChange, + AuthorityWorkPage, + DurableAuthorityStore, PaperclipJsonValue, PaperclipQuestionResponse, PaperclipSemanticActionBinding, @@ -67,6 +77,16 @@ const sourceUrl = new URL( ); const runner = (await import(sourceUrl.href)) as RunnerModule; +export const authorityInteger = runner.authorityInteger; +export const authorityGeneration = runner.authorityGeneration; +export const authorityDirectlyFollows = runner.authorityDirectlyFollows; +export const authorityJson = runner.authorityJson; +export const validateAuthorityCommit = runner.validateAuthorityCommit; +export const validateAuthorityPage = runner.validateAuthorityPage; +export const validateAuthorityWorkKey = runner.validateAuthorityWorkKey; +export const validateAuthorityWorkPage = runner.validateAuthorityWorkPage; +export const DurableAuthorityStoreError = runner.DurableAuthorityStoreError; + export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES = runner.DURABLE_PRP_CONTROL_PLANE_MAX_STATE_BYTES; @@ -128,3 +148,18 @@ export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailur export const completeTerminatedRemoteNativeSessionCleanup = runner.completeTerminatedRemoteNativeSessionCleanup; export const completeTerminatedLocalNativeSessionCleanup = runner.completeTerminatedLocalNativeSessionCleanup; + +export { readDurableControlPlaneState, parseAuthorityLocator, type AuthorityLocation, type AuthorityLocator, type ExternalAuthorityReader } from "@paperclipai/paperclip-runner"; +export { materializeCurrentAuthority } from "@paperclipai/paperclip-runner"; + +export { readIndexedLocalState, suspendIndexedRunnerState } from "@paperclipai/paperclip-runner"; +export type { IndexedLocalSnapshot } from "@paperclipai/paperclip-runner"; +export { IndexedInspectionDecoder } from "@paperclipai/paperclip-runner"; +export { migrateLegacySessionAuthority, assertNoPendingLegacyMigration, type LegacySessionMigrationOptions, type PreparedLocalSession } from "@paperclipai/paperclip-runner"; +export type { DurableRecoveryIdentity } from "@paperclipai/paperclip-runner"; + +export const isEventEpochTransition = runner.isEventEpochTransition; +export const normalizedEventId = runner.normalizedEventId; +export const advanceSourceCursor = runner.advanceSourceCursor; + +export const isProviderProcessGeneration = runner.isProviderProcessGeneration;