diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7b87a374fa2..cc1e23d53e1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2632,6 +2632,15 @@ An upgrade readiness report contains preliminary native evidence, not permission to remove a deployment volume. Keep the checkpoint pool bounded and measure all pools when qualifying database cost; the current size of two is a default. +For retained Codex event tails, keep source drain/accounting and target admission +separate. `native-staged-attachment-recovery.ts` validates private source/target +header evidence; `native-recovery-usage-delivery.ts` binds the source's accounting +receiver. The transport persists its staged target before source bootstrap, then +uses the existing authenticated attachment protocol after durable drain. Do not +rotate the identity first, acknowledge before receipt persistence, or weaken the +Runner's settled-provider check. A staged target does not yet certify a resumable +session. Its eventual snapshot still uses the existing checkpoint transaction. + Custom source-mode HTTP launchers should create their listener with `createPaperclipHttpServer` from `server/src/http/server.ts`, passing the selected API origin as `apiUrl`. This registers the native Runner WebSocket transport diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index e8ea4441426..9409d0d608e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2724,6 +2724,47 @@ attachment IDs. A healthy warm handoff can create new attachment IDs within the retained Runner. Replacing that run's controller must preserve those IDs; deriving them again from the business run ID would request an invalid authority rotation. +Cold Codex continuation can restore a completed provider with a retained event +suffix. Native remote controllers first persist a private staged attachment under +the target run's existing checkpoint owner, then restore and drain the exact +source authority before attaching the target. The source's run, turn and item +identity stays on every old event. Receipt persistence precedes acknowledgement; +unknown semantic work holds the handoff and never reaches the target's tool +authority. The provider's settled-session attachment guard remains mandatory. + +The staged controller format is +`paperclip.runner.durable.control-plane-state.staged-attachment.v1`. It contains +the source and target attachment tuples and one bound target attachment template +inside the existing encrypted PRP authority, subject to its existing size limits. +The header stores a bounded descriptor and digest. Before command delivery, the +template is a separate changed-record part; after queueing, recovery uses the +existing command or receipt. Control checkpoints do not rewrite another copy of +instructions or tool context in the header. +It is separate from the 4 KiB run-profile recovery envelope. Startup discovers +this target obligation through bounded encrypted-header reads before a target +session snapshot exists. It verifies both frozen runs, the accepted and released +source, its original Runner artifact and the exact source header. Remote process +verification then chooses authenticated source adoption or verified suspended +source restoration. An in-memory rebound snapshot supplies normal admission; +an existing target snapshot always takes precedence over that source baseline. +The intent remains through command delivery and authenticated handoff, including +an uncertain commit response. Source and target receipt phases require their +exact physical authority; receipt replay cannot start a second Runner or attach +again under a different endpoint. After the ordinary target snapshot commits, +an in-memory notification permits intent retirement in the next existing command +write. There is no separate metadata/snapshot transaction for that retirement. + +Source accounting uses the source's immutable producer receipts and billing +classification. Opening this receiver does not reset accounting ownership or +reopen a finalized run. Equal finalized usage retains the existing audit policy; +changed, incomplete or conflicting usage remains unacknowledged for +reconciliation. The target's usage receiver never receives source events. Staging +adds recovery control work only: text streaming and settled idle operation add no +queries, snapshot transactions, pools or recurring scans. The first rollback +release supporting this continuation must also read the staged controller format; +older readers fail closed. Prepared-launch and other provider paths retain their +existing admission and recovery contracts. + Accepted-result retirement still stops the old provider turn before draining and suspending its Runner. The checkpoint codec classifies `turn.stop` as lifecycle maintenance. Existing exact ownership, generation and accepted-result fences diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index 84ec0f5c73e..639d1b27e3a 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -41,6 +41,22 @@ retains the 32-candidate and newer-progress barrier checks. Snapshot transaction counts and stale-generation rejection are exercised by the control-plane port tests; incomplete-publication retention is exercised by the expiration tests. +The cold Codex transport regressions additionally retain a real provider event +after completion, hold its persistence callback and require the original source +tuple and Runner outbox to survive until acceptance. A separate case deletes the +controller cache after its staged target is durably recorded and resumes through +the same source before target attachment. Unknown semantic tails must stay held +without target tool dispatch or a fabricated response. The staged restart helper +and classifier tests cover target discovery before its first session snapshot, +exact encrypted source/header/artifact bindings and the existing compatibility +rebinder. Source accounting tests use real PostgreSQL to distinguish immutable +receipt replay, equal finalized usage and late usage requiring reconciliation. +Handoff crash tests must restore the exact queued command or authenticated +receipt, reuse the verified physical Runner and preserve endpoint binding. Test +both direct callbacks and the same guest listener when that ingress is enabled; +changing an endpoint or losing physical proof must hold recovery. +These component cases do not replace the normal three-controller Daytona gate. + Per-turn plugin continuation uses a stopped checkpoint on a replacement lease. The executor tests cover Claude and Codex, including reopening blocked work. The lease-authority integration test uses PostgreSQL to check that checkpoint diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 36b38caba82..c4f29bdbb6a 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -32,6 +32,8 @@ import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, + inspectStagedRunAttachment, + inspectStagedRunAttachmentHeader, type DurablePrpCheckpoint, durableRecoveryInternals, SemanticToolNotDispatchedError, @@ -53,6 +55,66 @@ const identity: DurableRecoveryIdentity = { const expectedRunnerVersion = "0.3.0"; const expectedRunnerDigest = `sha256:${"a".repeat(64)}`; +it("persists an undispatched source-to-target attachment before bootstrap and preserves it after cache loss", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-staged-attachment-")); + const target = { ...identity, runId: "00000000-0000-4000-8000-000000000002", turnId: "next-turn", itemId: "next-item" }; + const template = { provider: { provider: "codex", command: "fixture", instructions: "assigned instructions ".repeat(1_500) }, authorizedTools: {} }; + const core = new DurablePrpControlPlane({ stateDirectory: resolve(root, "first"), identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint, templateWrites = 0; + core.setCheckpointPersistence(async value => { checkpoint = value; if (value.changedRecords.has("staged-attachment-template")) templateWrites++; }); + let restored: DurablePrpControlPlane | undefined; + let client: AuthenticatedClient | null = null; + try { + core.stageRunAttachment(target, template); + await core.flushCheckpoint(); + expect(checkpoint.runId).toBe(target.runId); + expect(checkpoint.predecessor).toEqual({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId }); + expect(checkpoint.workCommandIds).toEqual([]); + const proof = inspectStagedRunAttachmentHeader(checkpoint.records.get("header")!); + expect(proof).toMatchObject({ attachment: { sourceIdentity: identity, targetIdentity: target, commandId: null } }); + expect(proof?.attachment).not.toHaveProperty("runAttachTemplate"); + expect(checkpoint.records.get("header")!.length).toBeLessThan(JSON.stringify(template).length / 2); + expect(JSON.parse(checkpoint.records.get("staged-attachment-template")!)).toEqual(template); + expect(proof?.runnerVersion).toBe(expectedRunnerVersion); + expect(proof?.runnerDigest).toBe(expectedRunnerDigest); + checkpoint.restoreSessionDirectory(resolve(root, "replacement")); + restored = new DurablePrpControlPlane({ stateDirectory: resolve(root, "replacement", "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }); + expect(inspectStagedRunAttachment(restored.store.state, target)).toMatchObject({ ...proof?.attachment, runAttachTemplate: template }); + expect(restored.store.state.commands).toEqual([]); + expect(() => restored!.stageRunAttachment({ ...target, itemId: "foreign" }, template)).toThrow("conflicts"); + expect(() => restored!.queueCommand("turn.start", {})).toThrow("maintenance only"); + expect(() => restored!.queueCommand("run.attach", { ...template, paperclipNextAuthority: { identity: { ...target, itemId: "foreign" } } })).toThrow("conflicts"); + const corruptedRecords = new Map(checkpoint.records); + corruptedRecords.set("staged-attachment-template", JSON.stringify({ provider: { command: "changed" } })); + expect(() => decodeDurablePrpCheckpoint(corruptedRecords)).toThrow("native_runner_checkpoint_invalid"); + expect(inspectStagedRunAttachment(restored.store.state, { ...target, turnId: "foreign" })).toBeNull(); + for (const field of ["runnerInstanceId", "environmentLeaseId", "normalizedSessionId"] as const) { + const changed = JSON.parse(checkpoint.records.get("header")!); + changed.state.stagedRunAttachment.targetIdentity[field] = "foreign"; + expect(inspectStagedRunAttachmentHeader(JSON.stringify(changed))).toBeNull(); + } + const downgraded = JSON.parse(checkpoint.records.get("header")!); + downgraded.state.schema = "paperclip.runner.durable.control-plane-state.v1"; + expect(inspectStagedRunAttachmentHeader(JSON.stringify(downgraded))).toBeNull(); + const beforeRetry = checkpoint; + core.stageRunAttachment(target, template); await core.flushCheckpoint(); expect(checkpoint).toBe(beforeRetry); + core.queueCommand("session.snapshot", {}); await core.flushCheckpoint(); + expect(checkpoint.changedRecords.has("staged-attachment-template")).toBe(false); + await core.start(); client = await authenticate(core, core.issueBootstrapTicket()); + expect(client).not.toBeNull(); + core.queueCommand("run.attach", { ...template, paperclipNextAuthority: { identity: target, + connection: { mode: "connect", connectUrl: core.connectUrl } } }, "staged-next-attach"); + await core.flushCheckpoint(); + expect(templateWrites).toBe(1); + expect(checkpoint.records.has("staged-attachment-template")).toBe(false); + expect(checkpoint.removedRecords).toContain("staged-attachment-template"); + expect(() => decodeDurablePrpCheckpoint(checkpoint.records)).not.toThrow(); + } finally { + client?.socket.destroy(); + await restored?.stop(); await core.stop(); rmSync(root, { recursive: true, force: true }); + } +}); + it("continues externally persisted commands after losing the local controller directory", async () => { const root = mkdtempSync(resolve(tmpdir(), "runner-external-store-")); const options = { stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 4b7ae503420..ea82cd674e0 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -54,6 +54,7 @@ const protocolVersion = RUNNER_PRP_PROTOCOL_MAX; const secureFrameSchema = "paperclip.runner.secure-frame.v1"; const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; +const stagedCoreStateSchema = "paperclip.runner.durable.control-plane-state.staged-attachment.v1"; const externalCoreStateSchema = "paperclip.runner.external-control-plane-state.v1"; const transitionCoreStateSchema = "paperclip.runner.durable.control-plane-state.warm-transition.v1"; @@ -140,8 +141,17 @@ interface ConnectionLeaseRecord { revokedAt: string | null; } +export interface StagedRunAttachment { + readonly sourceIdentity: DurableRecoveryIdentity; + readonly targetIdentity: DurableRecoveryIdentity; + readonly runAttachTemplate: Record; + readonly templateDigest: string; + readonly commandId: string | null; +} + interface StoredCoreState { - schema: typeof coreStateSchema | typeof transitionCoreStateSchema; + schema: typeof coreStateSchema | typeof transitionCoreStateSchema | typeof stagedCoreStateSchema; + stagedRunAttachment?: Omit & { runAttachTemplate?: Record }; identity: DurableRecoveryIdentity; /** External checkpoints contain control state, not a duplicate output log. */ checkpointEventPolicy?: "control-v1"; @@ -311,10 +321,12 @@ type CheckpointHeader = { schema: "paperclip.runner.controller-checkpoint.v2"; runnerVersion: string; runnerDigest: string; - state: Omit; + state: Omit + & { stagedRunAttachment?: Omit }; }; function checkpointRunId(state: StoredCoreState): string { + if (state.stagedRunAttachment) return state.stagedRunAttachment.targetIdentity.runId; if (state.warmTransition) return state.warmTransition.receipt.newIdentity.runId; const attachment = state.commands.findLast(command => command.type === "run.attach"); const next = attachment?.payload.paperclipNextAuthority as { identity?: { runId?: string } } | undefined; @@ -347,13 +359,15 @@ function checkpointChanges(records: ReadonlyMap, runId: string, function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runnerDigest: string): DurablePrpCheckpoint { const runId = checkpointRunId(state); const records = new Map(); - const { commands, committedEvents, ackedSourceSeq: _cursor, replayDeliveries: _replays, ...metadata } = state; + const { commands, committedEvents, ackedSourceSeq: _cursor, replayDeliveries: _replays, stagedRunAttachment, ...metadata } = state; + const stagedDescriptor = stagedRunAttachment ? (({ runAttachTemplate: _template, ...descriptor }) => descriptor)(stagedRunAttachment) : undefined; + if (stagedRunAttachment?.commandId === null) records.set("staged-attachment-template", JSON.stringify(stagedRunAttachment.runAttachTemplate)); const latestUsage = committedEvents.findLast(event => event.eventType === "usage.reported"); const recoveryEvents = committedEvents.filter(event => !isStreamingEvent(event) || event === latestUsage); for (const command of commands) records.set(`command:${command.controllerSeq}`, JSON.stringify(command)); for (const event of recoveryEvents) records.set(`event:${event.sourceSeq}`, JSON.stringify(event)); const header: CheckpointHeader = { schema: "paperclip.runner.controller-checkpoint.v2", - runnerVersion, runnerDigest, state: { ...metadata, checkpointEventPolicy: "control-v1" } }; + runnerVersion, runnerDigest, state: { ...metadata, ...(stagedDescriptor ? { stagedRunAttachment: stagedDescriptor } : {}), checkpointEventPolicy: "control-v1" } }; records.set("header", JSON.stringify(header)); records.set("cursor", JSON.stringify({ ackedSourceSeq: state.ackedSourceSeq, replayDeliveries: state.replayDeliveries, commandCount: commands.length, eventCount: recoveryEvents.length })); @@ -397,11 +411,43 @@ export function inspectDurablePrpCheckpointHeader(contents: string): DurableReco const header = parseCheckpointHeader(contents); if (header.schema !== "paperclip.runner.controller-checkpoint.v2" || header.state.checkpointEventPolicy !== "control-v1" - || ![coreStateSchema, transitionCoreStateSchema].includes(String(header.state.schema))) return null; + || ![coreStateSchema, transitionCoreStateSchema, stagedCoreStateSchema].includes(String(header.state.schema)) + || (header.state.schema === stagedCoreStateSchema && !validStagedRunAttachment(header.state as unknown as StoredCoreState, true))) return null; return { ...header.state.identity }; } catch { return null; } } +/** Bounded checkpoint metadata inspection; never itself grants recovery authority. */ +export function inspectStagedRunAttachmentHeader(contents: string): { + attachment: Omit; runnerVersion: string; runnerDigest: string; + protocolIdentity: DurableRecoveryIdentity; + phase: "source_bootstrap" | "attachment_queued" | "awaiting_result" | "prepared" | "activated" | "completed"; + receipt: DurableWarmRunTransition | null; +} | null { + try { + if (Buffer.byteLength(contents) > maxStateBytes) return null; + const header = parseCheckpointHeader(contents), state = header.state as unknown as StoredCoreState; + if (header.schema !== "paperclip.runner.controller-checkpoint.v2" + || state.checkpointEventPolicy !== "control-v1" || !validStagedRunAttachment(state, true) + || !state.stagedRunAttachment) return null; + const transition = state.warmTransition; + const completed = stagedTargetActivated(state) ? state.completedWarmTransition : undefined; + const bound = transition ?? completed; + if (bound) { + const receipt = bound.receipt; + const result = transition?.phase === "awaiting_result" ? transition.expectedResult : bound.command.result; + const expected = warmTransitionReceipt(receipt.oldIdentity, bound.command, result!, receipt.oldAckedSourceSeq, + { leaseId: receipt.leaseId, expiresAtUnixMs: receipt.leaseExpiresAtUnixMs, revocationEpoch: receipt.leaseRevocationEpoch }, + header.runnerVersion, header.runnerDigest); + if (canonicalJson(expected) !== canonicalJson(receipt)) return null; + } + return { attachment: structuredClone(state.stagedRunAttachment), protocolIdentity: structuredClone(state.identity), + phase: transition?.phase ?? (completed ? "completed" : state.stagedRunAttachment.commandId ? "attachment_queued" : "source_bootstrap"), + receipt: bound ? structuredClone(bound.receipt) : null, + runnerVersion: header.runnerVersion, runnerDigest: header.runnerDigest }; + } catch { return null; } +} + function checkpointState(records: ReadonlyMap) { const header = parseCheckpointHeader(records.get("header") ?? "null"); const read = (key: string): unknown => { @@ -423,9 +469,14 @@ function checkpointState(records: ReadonlyMap) { } const cursor = v2 ? read("cursor") : header.state; if (!isRecord(cursor) || (v2 && (cursor.commandCount !== commandKeys.length || cursor.eventCount !== eventKeys.length - || records.size !== commandKeys.length + eventKeys.length + 2))) throw new Error("native_runner_checkpoint_incomplete"); + || records.size !== commandKeys.length + eventKeys.length + 2 + (isRecord(header.state.stagedRunAttachment) && header.state.stagedRunAttachment.commandId === null ? 1 : 0)))) throw new Error("native_runner_checkpoint_incomplete"); const state = { ...header.state, ackedSourceSeq: cursor.ackedSourceSeq, replayDeliveries: cursor.replayDeliveries, - commands: commandKeys.map(read), committedEvents: eventKeys.map(read) }; + commands: commandKeys.map(read), committedEvents: eventKeys.map(read) } as unknown as StoredCoreState; + if (state.stagedRunAttachment?.commandId === null) { + const template = read("staged-attachment-template"); + if (!isRecord(template)) throw new Error("native_runner_checkpoint_invalid"); + state.stagedRunAttachment = { ...state.stagedRunAttachment, runAttachTemplate: template }; + } if (!isStoredCoreState(state, header.state.identity)) throw new Error("native_runner_checkpoint_invalid"); if (state.commands.some((command, index) => commandKeys[index] !== `command:${command.controllerSeq}`) || state.committedEvents.some((event, index) => eventKeys[index] !== `event:${event.sourceSeq}`)) { @@ -683,9 +734,9 @@ function warmTransitionReceipt( function validStoredWarmTransition(state: StoredCoreState): boolean { const transition = state.warmTransition; - if (!transition) return state.schema === coreStateSchema; + if (!transition) return state.schema === coreStateSchema || (state.schema === stagedCoreStateSchema && validStagedRunAttachment(state)); if ( - state.schema !== transitionCoreStateSchema || + (state.schema !== transitionCoreStateSchema && state.schema !== stagedCoreStateSchema) || !["awaiting_result", "prepared", "activated"].includes(transition.phase) || !isRecord(transition.receipt) || !isRecord(transition.command) || @@ -802,6 +853,76 @@ function unsettledSemanticInput( } } +function stagedCommandMatches(staged: Pick, command: DurableRecoveryCoreCommand): boolean { + if (command.type !== "run.attach" || command.commandId !== staged.commandId) return false; + const { paperclipNextAuthority, ...template } = command.payload; + return isRecord(paperclipNextAuthority) + && canonicalJson(paperclipNextAuthority.identity) === canonicalJson(staged.targetIdentity) + && `sha256:${canonicalDigest(template)}` === staged.templateDigest; +} + +function stagedAttachmentTemplate(state: StoredCoreState): Record | null { + const staged = state.stagedRunAttachment; + if (!staged) return null; + if (staged.commandId === null) return staged.runAttachTemplate ?? null; + const command = state.commands?.find(command => command.commandId === staged.commandId) + ?? state.warmTransition?.command ?? state.completedWarmTransition?.command; + if (!command || !stagedCommandMatches(staged, command)) return null; + const { paperclipNextAuthority: _authority, ...template } = command.payload; + return template; +} + +function stagedTargetActivated(state: StoredCoreState): boolean { + const staged = state.stagedRunAttachment, completed = state.completedWarmTransition; + return !!staged && !state.warmTransition && !!completed + && canonicalJson(state.identity) === canonicalJson(staged.targetIdentity) + && completed.receipt.commandId === staged.commandId + && canonicalJson(completed.receipt.oldIdentity) === canonicalJson(staged.sourceIdentity) + && canonicalJson(completed.receipt.newIdentity) === canonicalJson(staged.targetIdentity) + && stagedCommandMatches(staged, completed.command); +} + +function validStagedRunAttachment(state: StoredCoreState, headerOnly = false): boolean { + const staged = state.stagedRunAttachment; + if (staged === undefined) return state.schema !== stagedCoreStateSchema; + if (!(state.schema === stagedCoreStateSchema && isRecord(staged) + && Object.keys(staged).sort().join(",") === (headerOnly || staged.commandId !== null ? "commandId,sourceIdentity,targetIdentity,templateDigest" : "commandId,runAttachTemplate,sourceIdentity,targetIdentity,templateDigest") + && exactIdentity(staged.sourceIdentity) && exactIdentity(staged.targetIdentity) + && staged.sourceIdentity.runId !== staged.targetIdentity.runId + && ["runnerInstanceId", "environmentLeaseId", "normalizedSessionId"].every(key => + staged.sourceIdentity[key as keyof DurableRecoveryIdentity] === staged.targetIdentity[key as keyof DurableRecoveryIdentity]) + && /^sha256:[0-9a-f]{64}$/.test(staged.templateDigest) + && (headerOnly || staged.commandId !== null || (isRecord(staged.runAttachTemplate) && isRecord(staged.runAttachTemplate.provider) + && staged.runAttachTemplate.paperclipNextAuthority === undefined + && `sha256:${canonicalDigest(staged.runAttachTemplate)}` === staged.templateDigest + && Buffer.byteLength(JSON.stringify(staged.runAttachTemplate)) <= maxCommandBytes)) + && (staged.commandId === null || (typeof staged.commandId === "string" && stableIdPattern.test(staged.commandId))))) return false; + const sourceCurrent = canonicalJson(state.identity) === canonicalJson(staged.sourceIdentity); + if (!sourceCurrent && canonicalJson(state.identity) !== canonicalJson(staged.targetIdentity)) return false; + if (state.warmTransition) { + const transition = state.warmTransition; + return staged.commandId !== null && stagedCommandMatches(staged, transition.command) + && canonicalJson(transition.receipt.oldIdentity) === canonicalJson(staged.sourceIdentity) + && canonicalJson(transition.receipt.newIdentity) === canonicalJson(staged.targetIdentity) + && (sourceCurrent ? transition.phase !== "activated" : transition.phase === "activated"); + } + if (!sourceCurrent) return stagedTargetActivated(state); + if (staged.commandId === null) return true; + // Header-only inspection cannot certify the separate command record. Full + // restoration below requires its exact tuple, payload and command ID. + return !Array.isArray(state.commands) || state.commands.some(command => stagedCommandMatches(staged, command)); +} + +/** Inspection is evidence only; the caller still owns recovery authorization. */ +export function inspectStagedRunAttachment(state: unknown, expectedTarget?: DurableRecoveryIdentity): StagedRunAttachment | null { + try { + if (!isRecord(state) || !exactIdentity(state.identity) || !isStoredCoreState(state, state.identity) + || !state.stagedRunAttachment || (expectedTarget && canonicalJson(state.stagedRunAttachment.targetIdentity) !== canonicalJson(expectedTarget))) return null; + const template = stagedAttachmentTemplate(state); + return template ? { ...structuredClone(state.stagedRunAttachment), runAttachTemplate: structuredClone(template) } : null; + } catch { return null; } +} + function isStoredCoreState( value: unknown, identity: DurableRecoveryIdentity, @@ -811,7 +932,7 @@ function isStoredCoreState( const events = value.committedEvents; if ( (value.schema !== coreStateSchema && - value.schema !== transitionCoreStateSchema) || + value.schema !== transitionCoreStateSchema && value.schema !== stagedCoreStateSchema) || canonicalJson(value.identity) !== canonicalJson(identity) || (value.checkpointEventPolicy !== undefined && value.checkpointEventPolicy !== "control-v1") || !isRecord(value.tickets) || @@ -828,6 +949,7 @@ function isStoredCoreState( ) { return false; } + if (!validStagedRunAttachment(value as unknown as StoredCoreState)) return false; if (!validStoredWarmTransition(value as unknown as StoredCoreState)) return false; if (value.completedWarmTransition !== undefined) { @@ -1758,6 +1880,7 @@ class AuthorityConnection { /** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */ export class DurablePrpControlPlane { #identity: DurableRecoveryIdentity; + #stagedTargetCheckpointPublished: string | null = null; readonly #store: DurableCoreStore; #expectedRunnerVersion: string; #expectedRunnerDigest: string; @@ -1854,7 +1977,12 @@ export class DurablePrpControlPlane { cached.set(part, prior?.record === record ? prior : { record, command: JSON.parse(record) }); } this.#publicationCommands = cached; - return { ...header.state, ...cursor, commands: [...cached.values()].map(value => value.command), + const descriptor = header.state.stagedRunAttachment; + const workingStage = this.#store.state.stagedRunAttachment; + const staged = descriptor?.commandId === null ? { ...descriptor, + runAttachTemplate: workingStage?.templateDigest === descriptor.templateDigest && workingStage.runAttachTemplate + ? workingStage.runAttachTemplate : JSON.parse(snapshot.records.get("staged-attachment-template")!) as Record } : descriptor; + return { ...header.state, ...(staged ? { stagedRunAttachment: staged } : {}), ...cursor, commands: [...cached.values()].map(value => value.command), // Event records are immutable after insertion. Retain their array, not // another deep copy of the streaming payload window. committedEvents: this.#store.state.committedEvents }; @@ -2323,6 +2451,23 @@ export class DurablePrpControlPlane { return ticket; } + /** Import exact durable handoff evidence for a caller-verified surviving participant; issues no credential. */ + prepareWarmTransitionAdoption(input: { transitionId: string; runnerState: Record }): void { + this.#store.assertWritable(); + const pending = input.runnerState.warmTransition; + const proof = warmTransitionRecoveryProof({ controlPlaneState: this.#store.state, runnerState: input.runnerState, + expectedNewIdentity: (isRecord(pending) && isRecord(pending.receipt) ? pending.receipt.newIdentity : null) as DurableRecoveryIdentity, + expectedRunnerVersion: this.#expectedRunnerVersion, expectedRunnerDigest: this.#expectedRunnerDigest }); + if (!proof || proof.transition.receipt.transitionId !== input.transitionId || !this.#store.state.stagedRunAttachment) { + throw new Error("Warm transition adoption snapshot proof is not authorized."); + } + if (this.#store.state.warmTransition && canonicalJson(this.#store.state.warmTransition) === canonicalJson(proof.transition)) return; + const candidate = structuredClone(this.#store.state); + candidate.schema = stagedCoreStateSchema; + candidate.warmTransition = structuredClone(proof.transition); + this.#store.commit(candidate); + } + /** Caller-owned recovery admission is required; a receipt is not a credential. */ issueWarmTransitionBootstrapTicket( input: { @@ -2361,7 +2506,7 @@ export class DurablePrpControlPlane { original.expiresAtUnixMs, ); const candidate = structuredClone(this.#store.state); - candidate.schema = transitionCoreStateSchema; + candidate.schema = candidate.stagedRunAttachment ? stagedCoreStateSchema : transitionCoreStateSchema; candidate.warmTransition = structuredClone(transition); candidate.tickets[material.credentialId] = { recordId: `bootstrap_ticket_${randomUUID()}`, @@ -2380,6 +2525,36 @@ export class DurablePrpControlPlane { return ticket; } + /** Caller confirms its existing session-checkpoint transaction committed. No journal write. */ + markStagedTargetCheckpointPublished(identity: DurableRecoveryIdentity): boolean { + if (!stagedTargetActivated(this.#store.state) || canonicalJson(identity) !== canonicalJson(this.#store.state.identity)) return false; + this.#stagedTargetCheckpointPublished = canonicalJson(identity); + return true; + } + + /** Durable intent precedes source restoration; it is never a dispatched command. */ + stageRunAttachment(targetIdentity: DurableRecoveryIdentity, runAttachTemplate: Record): void { + this.#store.assertWritable(); + const existing = this.#store.state.stagedRunAttachment; + if (existing && !(stagedTargetActivated(this.#store.state) + && this.#stagedTargetCheckpointPublished === canonicalJson(this.#store.state.identity) + && targetIdentity.runId !== existing.targetIdentity.runId)) { + if (!validStagedRunAttachment(this.#store.state) + || canonicalJson(existing.targetIdentity) !== canonicalJson(targetIdentity) + || canonicalJson(stagedAttachmentTemplate(this.#store.state)) !== canonicalJson(runAttachTemplate)) { + throw new Error("Staged run attachment conflicts with its durable intent."); + } + return; + } + const candidate = structuredClone(this.#store.state); + candidate.schema = stagedCoreStateSchema; + candidate.stagedRunAttachment = { sourceIdentity: structuredClone(candidate.identity), + targetIdentity: structuredClone(targetIdentity), runAttachTemplate: structuredClone(runAttachTemplate), + templateDigest: `sha256:${canonicalDigest(runAttachTemplate)}`, commandId: null }; + if (!validStagedRunAttachment(candidate)) throw new Error("Staged run attachment binding is invalid."); + this.#store.commit(candidate); + } + queueCommand( type: string, payload: Record = {}, @@ -2387,6 +2562,21 @@ export class DurablePrpControlPlane { deliverImmediately = false, ): DurableRecoveryCoreCommand { this.#store.assertWritable(); + const staged = this.#store.state.stagedRunAttachment; + const retireStaged = stagedTargetActivated(this.#store.state) && (type === "turn.start" + || (this.#stagedTargetCheckpointPublished === canonicalJson(this.#store.state.identity) + && !["runner.drain", "session.snapshot", "turn.stop", "runner.suspend", "session.close"].includes(type))); + if (staged && !retireStaged) { + if (type === "run.attach") { + const { paperclipNextAuthority, ...template } = payload; + if (!isRecord(paperclipNextAuthority) + || canonicalJson(paperclipNextAuthority.identity) !== canonicalJson(staged.targetIdentity) + || `sha256:${canonicalDigest(template)}` !== staged.templateDigest + || (staged.commandId !== null && staged.commandId !== commandId)) throw new Error("Staged run attachment command conflicts."); + } else if (!stagedTargetActivated(this.#store.state) && !["runner.drain", "session.open", "session.snapshot", "turn.stop", "runner.suspend", "session.close", "semantic_tool.result"].includes(type)) { + throw new Error("Staged run attachment permits recovery maintenance only."); + } + } const transition = this.#store.state.warmTransition; if (transition && transition.phase !== "activated") { if ( @@ -2461,6 +2651,16 @@ export class DurablePrpControlPlane { if (Buffer.byteLength(JSON.stringify(command)) > maxCommandBytes) throw new CommandJournalLimitError("command_payload_too_large"); this.#store.state.commands.push(command); + if (staged && !retireStaged && type === "run.attach") { + const { runAttachTemplate: _template, ...descriptor } = staged; + this.#store.state.stagedRunAttachment = { ...descriptor, commandId: command.commandId }; + } else if (retireStaged) { + // Native runtime persists the recovered session before its first turn. + // Read/goal controls during opening must not retire startup evidence. + delete this.#store.state.stagedRunAttachment; + this.#store.state.schema = coreStateSchema; + this.#stagedTargetCheckpointPublished = null; + } this.#store.save(); void this.flushCheckpoint().catch(() => undefined); if (deliverImmediately) { @@ -2677,7 +2877,7 @@ export class DurablePrpControlPlane { } if (transition) { const candidate = structuredClone(this.#store.state); - candidate.schema = coreStateSchema; + candidate.schema = candidate.stagedRunAttachment ? stagedCoreStateSchema : coreStateSchema; candidate.leases[transition.credentialId]!.identity = structuredClone( receipt.newIdentity, ); @@ -3205,8 +3405,9 @@ export class DurablePrpControlPlane { } if (transition.phase === "prepared") { const candidate = initialCoreState(transition.receipt.newIdentity); + candidate.stagedRunAttachment = structuredClone(this.#store.state.stagedRunAttachment); candidate.preparedLaunchConfigurations = structuredClone(this.#store.state.preparedLaunchConfigurations); - candidate.schema = transitionCoreStateSchema; + candidate.schema = candidate.stagedRunAttachment ? stagedCoreStateSchema : transitionCoreStateSchema; candidate.warmTransition = { ...structuredClone(transition), phase: "activated", @@ -3507,7 +3708,7 @@ export class DurablePrpControlPlane { )!; completed.status = "completed"; completed.result = structuredClone(result); - candidate.schema = transitionCoreStateSchema; + candidate.schema = candidate.stagedRunAttachment ? stagedCoreStateSchema : transitionCoreStateSchema; candidate.warmTransition = { receipt, phase: "prepared", diff --git a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts index f94fa464082..526d628d9f7 100644 --- a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts +++ b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts @@ -36,6 +36,10 @@ export interface RunnerExecutionDriver { checkpoint?: (settlement: "settled" | "unsettled") => Promise | void; release: (reason?: "detach") => Promise | void; }>; + /** Register source and target checkpoint ownership before staging changes the stored run. */ + onStagedRunAttachment?: (authority: DurablePrpControlPlane, proof: { + sourceIdentity: DurableRecoveryIdentity; targetIdentity: DurableRecoveryIdentity; templateDigest: string; + }) => Promise; /** Override process ownership; omission uses the SDK's local process owner. */ runnerProcessLauncher?: ( spec: RunnerProcessLaunchSpec, diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 6b42469e678..18056c43708 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -37,6 +37,9 @@ export { RUNNER_PRP_PROTOCOL_MIN, RUNNER_PRP_PROTOCOL_MAX, inspectWarmRunTransition, + inspectStagedRunAttachment, + inspectStagedRunAttachmentHeader, + type StagedRunAttachment, type DurablePrpControlPlaneOptions, type DurablePrpCheckpoint, decodeDurablePrpCheckpoint, diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 37c6011ffe2..b1749f4653a 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -8239,7 +8239,7 @@ it("still fails closed when a real close grace period cannot fit a durable suspe } }, 30_000); -it("cold-restores a suspended provider session under its durable run binding", async () => { +it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); const tracePath = join(stateDirectory, "provider-trace.ndjson"); const skillRoot = join(stateDirectory, "runtime-skill"); @@ -8332,14 +8332,57 @@ it("cold-restores a suspended provider session under its durable run binding", a throw new Error("cold attach fixture did not return a provider thread"); } + // Model a retained provider-owned suffix: rotation must deliver and durably + // accept it before attempting run.attach, rather than dropping or ACKing it. + const retainedDelta = "cold-recovery-retained-tail"; + if (pendingTail) { + const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); + const provider = JSON.parse(await readFile(providerFile, "utf8")); + expect(provider.pendingEvents).toEqual([]); + provider.pendingEvents.push({ + executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, + eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", + priority: "p0", + payload: pendingTail === "unknown-semantic" ? { semantic_tool: { + schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", + callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, + correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, + turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, + content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, + } } : { provider: "codex", kind: "usage", delta: retainedDelta, + providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, + runDeltaAvailable: true, runDeltaBasis: "run" }, + }); + await writeFile(providerFile, JSON.stringify(provider)); + } + const retainedEvents: PrpEvent[] = []; + let releaseRetained!: () => void, retainedEntered!: () => void; + const retainedAccepted = new Promise(done => { releaseRetained = done; }); + const retainedAdmission = new Promise(done => { retainedEntered = done; }); const secondIdentity = { ...baseIdentity, runId: "run-cold-second", turnId: "turn-cold-second", itemId: "item-cold-second", }; - const rotated = createCapabilityRunnerdCodexTransport({ + let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; + const rotatedOptions = { ...options, + onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { + authority.setCheckpointPersistence(async checkpoint => { + stagedCheckpoint = checkpoint; + if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); + }); + } : undefined, + onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { + if (event.payload.delta === retainedDelta) { + retainedEvents.push(structuredClone(event)); + retainedEntered(); + if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); + await retainedAccepted; + } + } : undefined, environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability" }, codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), resumeDynamicTools: dynamicTools, @@ -8348,14 +8391,81 @@ it("cold-restores a suspended provider session under its durable run binding", a criterionIds: ["criterion-second"], }, prpIdentity: secondIdentity, - }); - rotated.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); + }; + let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + if (pendingTail === "restart-before-bootstrap") { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); + expect(rotated.evidence().runnerPid).toBeNull(); + expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); + expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); + await rotated.detachControllerForRestart().catch(() => undefined); + rejectStagedCommit = false; + await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); + stagedCheckpoint!.restoreSessionDirectory(stateDirectory); + rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + } + const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); + rotated.transport.setServerRequestHandler(currentHandler); + if (pendingTail === "accounting-rejected") { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + try { + await Promise.race([retainedAdmission, reading]); + const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(source.identity).toEqual(baseIdentity); + expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + expect(currentHandler).not.toHaveBeenCalled(); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await expect(reading).rejects.toThrow(); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + if (pendingTail === "unknown-semantic") { + try { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); + expect(currentHandler).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retained.identity).toEqual(baseIdentity); + expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); + expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } try { - const read = await rotated.transport.request("thread/read", {}); + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + if (pendingTail) { + await Promise.race([retainedAdmission, reading]); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + const retainedCore = stagedCheckpoint ? { + identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, + commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), + } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retainedCore.identity).toEqual(baseIdentity); + expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + } + releaseRetained(); + const read = await reading; + expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); + if (pendingTail) expect(retainedEvents[0]).toMatchObject({ + runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, + }); const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(persistedProvider.pendingEvents).toEqual([]); expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); expect(read.thread).toMatchObject({ @@ -8372,8 +8482,14 @@ it("cold-restores a suspended provider session under its durable run binding", a expect(rotated.evidence().diagnostics).toContain( "runnerd attached the durable provider session to a fresh PRP run authority", ); - expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); + if (pendingTail) { + const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state + : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(target.stagedRunAttachment).toBeUndefined(); + expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); + } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); } finally { + releaseRetained(); await rotated.transport.close(); } const requestFrames = (await readFile(tracePath, "utf8")) @@ -8410,6 +8526,12 @@ it("cold-restores a suspended provider session under its durable run binding", a ); } + if (pendingTail) { + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + return; + } + // A remote process owner keeps runner-state outside the controller's local // session root. Resume must defer to its explicit state reader instead of // rejecting recovery before the remote checkpoint can be made available. diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index bba62831ee4..2f5651b200f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -69,6 +69,7 @@ import type { } from "../contracts/harness-driver.js"; import { DurablePrpControlPlane, + inspectStagedRunAttachment, SemanticToolNotDispatchedError, durableRecoveryInternals, inspectWarmRunTransition, @@ -1241,6 +1242,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD observeCredentials?: boolean; onCredentialsUpdated?: (event: PrpEvent) => Promise; onAccountingEvent?: (event: PrpEvent) => Promise; + onRecoveryAccountingEvent?: (event: PrpEvent, sourceIdentity: DurableRecoveryIdentity) => Promise; recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"]; opencodePermissionMode?: NativeOpenCodePermissionMode; acpxAgent?: QualifiedAcpxAgent; @@ -1379,7 +1381,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD itemId: string; }; /** Existing server-owned routes only; never provision two ingress owners. */ - warmTransitionRegistrationMode?: "routed_connect"; + warmTransitionRegistrationMode?: "routed_connect" | "routed_same_endpoint"; /** Rechecks the server-owned recovery claim after each asynchronous boundary. */ authorizeWarmTransitionRecovery?: ( stage: "before_bootstrap" | "before_spawn" | "before_authentication", @@ -3562,6 +3564,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #pump: NodeJS.Timeout | null = null; #eventSourceSeq = 0; #eventIdentity: DurableRecoveryIdentity | null = null; + #sourceRecoveryIdentity: DurableRecoveryIdentity | null = null; + #sourceAttachmentDeadline: number | null = null; #pendingWarmRecoveryCompletion: { transitionId: string; identity: DurableRecoveryIdentity; @@ -3790,6 +3794,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ): Promise> { if (this.#closed) throw new Error("PRP Codex transport is closed"); this.#throwIfFailed(); + if (this.#sourceRecoveryIdentity && !["initialize", "collaborationMode/list"].includes(method)) { + throw new Error("native_runner_source_attachment_pending"); + } if ( this.#pendingWarmRecoveryCompletion !== null && !["thread/read", "initialize", "collaborationMode/list"].includes(method) @@ -4099,7 +4106,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } #recoverPendingSemanticInputsWhenReady(): void { - if (this.#startupComplete && this.#handlerInstalled) { + if (this.#startupComplete && this.#handlerInstalled && this.#sourceRecoveryIdentity === null && this.#pendingWarmRecoveryCompletion === null) { void Promise.resolve().then(() => this.#core?.recoverPendingSemanticInputs()) .catch(error => this.#failTransport(error instanceof Error ? error : new Error(String(error)))); } @@ -4109,7 +4116,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // Remote runner ingress already has a bounded reconnect budget. Reuse the // same budget here so a transient tunnel reconnect cannot trip the shorter // generic command timeout and replace an otherwise healthy warm runner. - const reconnectGraceMs = this.options.runnerReconnectGraceMs ?? 5_000; + const reconnectGraceMs = Math.min(this.options.runnerReconnectGraceMs ?? 5_000, + this.#sourceAttachmentDeadline === null ? Infinity : Math.max(0, this.#sourceAttachmentDeadline - Date.now())); await waitForWarmAttachmentReadiness({ graceMs: reconnectGraceMs, waitForConnection: (deadline) => this.#awaitWarmRunnerConnection(deadline), @@ -4239,7 +4247,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { .update(`${prior.runId}:${desired.runId}:${desired.turnId}`) .digest("hex") .slice(0, 32)}`; - const runAttachTemplate = this.#runAttachTemplate + const stagedCandidate = inspectStagedRunAttachment(core.store.state); + const staged = stagedCandidate && recoveryIdentityMatches(stagedCandidate.targetIdentity, desired) ? stagedCandidate : undefined; + const runAttachTemplate = staged ? structuredClone(staged.runAttachTemplate) : this.#runAttachTemplate ? retargetRunAttachPayload( this.#runAttachTemplate, desired, @@ -4256,7 +4266,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { currentRunGrant?.runtimeContext, currentRunGrant ? { text: currentRunGrant.instructions, context: currentRunGrant.runtimeContext } : undefined, ); - if (this.options.prepareLaunch) { + if (this.options.prepareLaunch && !staged) { const priorProvider = record(runAttachTemplate.provider); const priorRequirements = new RunnerLaunchPreparationStore(this.#root).load(prior)?.requirements; await this.#prepareOwnedProvider(runAttachTemplate, desired, @@ -4266,6 +4276,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } this.#assertStartupOpen(); this.#runAttachTemplate = structuredClone(runAttachTemplate); + if (!staged && (this.options.provider ?? "codex") === "codex" && !this.options.prepareLaunch && this.options.onStagedRunAttachment) { + // Registration already installed the target writer. Stage and queue + // share the existing attachment checkpoint; no duplicate owner reads. + core.stageRunAttachment(desired, runAttachTemplate); + } const payload = { ...runAttachTemplate, paperclipNextAuthority: { identity: desired, connection }, @@ -4278,7 +4293,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await this.#waitCommand( "run.attach", commandId, - this.#coldAdmissionDeadline(), + this.#sourceAttachmentDeadline === null ? this.#coldAdmissionDeadline() + : Math.min(Date.now() + 30_000, this.#sourceAttachmentDeadline), true, ); const attached = core.getCommand(commandId); @@ -4288,6 +4304,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { activationStarted = true; core.rotateRunIdentity(desired, runAttachTemplate); + if (core.store.state.stagedRunAttachment && registration?.connection?.mode === "listen" && this.options.readRunnerState) { + const deadline = Math.min(Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000), this.#sourceAttachmentDeadline ?? Infinity); + const artifact = this.options.runnerArtifact ?? approvedRunnerArtifact(this.options.runnerBinary ?? defaultCapabilityRunnerdBinary()); + for (;;) { + this.#assertStartupOpen(); + const physical = await this.options.readRunnerState(); + const proof = inspectWarmRunTransition({ controlPlaneState: core.store.state, runnerState: physical, + expectedNewIdentity: desired, expectedRunnerVersion: artifact.version, expectedRunnerDigest: artifact.digest }); + if (proof && recoveryIdentityMatches(proof.runnerIdentity, desired)) { + await previousRelease?.(); previousReleased = true; break; + } + if (Date.now() >= deadline || await this.#runnerHasExited()) throw new Error("native_runner_warm_transition_activation_pending"); + await new Promise(done => setTimeout(done, 100)); + } + } await registration?.activate?.(); if (registration?.failure) { void registration.failure.catch((error: unknown) => { @@ -4297,13 +4328,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); } await this.#awaitRegistrationReady(registration?.ready); - const activationDeadline = - Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000); + const activationDeadline = Math.min( + Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000), this.#sourceAttachmentDeadline ?? Infinity); while ( !recoveryIdentityMatches(core.store.state.identity, desired) || core.store.state.warmTransition !== undefined || core.activeRunnerConnectionCount() === 0 ) { + this.#assertStartupOpen(); if ( Date.now() >= activationDeadline || (await this.#runnerHasExited()) @@ -4312,11 +4344,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } + this.#assertStartupOpen(); this.#eventIdentity = structuredClone(desired); this.#eventSourceSeq = 0; this.#deferredTurnStartEvents = []; this.#durableTurnId = desired.turnId; - await previousRelease?.(); + if (!previousReleased) await previousRelease?.(); previousReleased = true; this.#controlPlaneRelease = registration?.release ?? null; } catch (error) { @@ -5244,7 +5277,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { - await this.options.onAccountingEvent?.(event); + if (this.#sourceRecoveryIdentity && event.runId === this.#sourceRecoveryIdentity.runId) { + if (!this.options.onRecoveryAccountingEvent && this.options.onAccountingEvent) throw new Error("native_runner_source_accounting_unavailable"); + await this.options.onRecoveryAccountingEvent?.(event, this.#sourceRecoveryIdentity); + } else await this.options.onAccountingEvent?.(event); if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); } @@ -5386,7 +5422,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #resumeWork(): Promise { if (this.options.acpxCandidateAdmission) throw new Error("ACPX candidate durable runner recovery is not admitted"); - const desiredIdentity = this.options.prpIdentity; + let desiredIdentity = this.options.prpIdentity; if (desiredIdentity === undefined) { throw new Error("PRP provider resume state is unavailable"); } @@ -5450,6 +5486,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ) { candidateRunnerState = await this.options.readRunnerState(); } + const staged = inspectStagedRunAttachment(controlPlaneState, desiredIdentity); + if (controlPlaneState?.stagedRunAttachment !== undefined && staged === null) { + throw new Error("native_runner_staged_attachment_binding_mismatch"); + } let warmRecovery: { runnerState: Record; runnerIdentity: DurableRecoveryIdentity; @@ -5475,8 +5515,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { !controlPlaneState || !candidateRunnerState || (this.options.controlPlaneRegistration !== undefined && - this.options.warmTransitionRegistrationMode !== "routed_connect") || - this.options.adoptExistingRunner !== undefined + this.options.warmTransitionRegistrationMode !== "routed_connect" + && !(staged && this.options.adoptExistingRunner && this.options.warmTransitionRegistrationMode === "routed_same_endpoint")) || + (this.options.adoptExistingRunner !== undefined && !staged) ) { throw new Error( "native_runner_warm_transition_requires_exact_owned_endpoint", @@ -5501,14 +5542,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? new URL(connection.connectUrl) : null; if ( - connection.mode !== "connect" || - endpoint === null || - endpoint.search || - endpoint.hash || - endpoint.username || - endpoint.password || + (connection.mode !== "connect" && !(connection.mode === "listen" && staged + && this.options.adoptExistingRunner && this.options.controlPlaneRegistration + && this.options.warmTransitionRegistrationMode === "routed_same_endpoint")) || + (connection.mode === "connect" && (endpoint === null || endpoint.search || endpoint.hash || endpoint.username || endpoint.password)) || (!this.options.controlPlaneRegistration && - (endpoint.protocol !== "ws:" || + (endpoint === null || endpoint.protocol !== "ws:" || endpoint.hostname !== "127.0.0.1" || endpoint.pathname !== "/durableRecovery/connect" || !endpoint.port)) @@ -5522,7 +5561,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { transitionId: receipt.transitionId, commandId: receipt.commandId, ...(!this.options.controlPlaneRegistration - ? { port: Number(endpoint.port) } + ? { port: Number(endpoint!.port) } : {}), }; // Fence concurrent public work before exposing the core or awaiting @@ -5532,6 +5571,40 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { identity: structuredClone(desiredIdentity), }; } + const completedStaged = staged && recoveryIdentityMatches(identity, staged.targetIdentity) + && record(record(controlPlaneState?.completedWarmTransition).receipt).commandId === staged.commandId; + const queuedSourceAttachment = staged && staged.commandId !== null && warmRecovery === null + && recoveryIdentityMatches(identity, staged.sourceIdentity) ? staged : null; + if (queuedSourceAttachment && !this.options.adoptExistingRunner) { + throw new Error("native_runner_queued_attachment_requires_verified_live_source"); + } + let sourceAttachmentTarget: DurableRecoveryIdentity | null = null; + if (controlPlaneState && warmRecovery === null && (this.options.provider ?? "codex") === "codex" + && !this.options.prepareLaunch && (staged || this.options.onStagedRunAttachment || this.options.onRecoveryAccountingEvent) + && identity.runId !== desiredIdentity.runId) { + // Restore and acknowledge the source under its own authority. The target + // remains a durable, undispatched intent until every old obligation settles. + if (!this.options.adoptExistingRunner) { + if (localStateOwner) { + if (!candidateRunnerState) throw new Error("native_runner_authority_rotation_state_unavailable"); + assertSuspendedRunnerState(candidateRunnerState, identity); + } else { + if (!this.options.readRunnerState || !this.options.prepareExternalRunnerState) throw new Error("native_runner_prp_run_rotation_unavailable"); + await this.options.prepareExternalRunnerState(); + assertSuspendedRunnerState(await this.options.readRunnerState(), identity); + } + } + sourceAttachmentTarget = queuedSourceAttachment ? null : desiredIdentity; + desiredIdentity = identity; + this.#sourceRecoveryIdentity = structuredClone(identity); + this.#sourceAttachmentDeadline = Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000); + } else if (staged && !completedStaged && !warmRecovery) { + throw new Error("native_runner_staged_attachment_profile_unsupported"); + } + if (warmRecovery && staged) { + this.#sourceRecoveryIdentity = structuredClone(staged.sourceIdentity); + this.#sourceAttachmentDeadline = Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000); + } const exactAuthority = controlPlaneState !== null && identity.runnerInstanceId === desiredIdentity.runnerInstanceId && @@ -5697,7 +5770,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { - await this.options.onAccountingEvent?.(event); + if (this.#sourceRecoveryIdentity && event.runId === this.#sourceRecoveryIdentity.runId) { + if (!this.options.onRecoveryAccountingEvent && this.options.onAccountingEvent) throw new Error("native_runner_source_accounting_unavailable"); + await this.options.onRecoveryAccountingEvent?.(event, this.#sourceRecoveryIdentity); + } else await this.options.onAccountingEvent?.(event); if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); } @@ -5768,14 +5844,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { preparedContext, preparedInstructions, saved?.prepared, saved?.requirements); this.#runAttachTemplate = structuredClone(template); } - if (rotatedAuthority) { + if (rotatedAuthority || sourceAttachmentTarget) { if (provider === "acpx" && this.options.acpxAgent === "copilot" && typeof this.options.baseInstructions !== "string") { throw new Error("native_runner_current_instruction_grant_unavailable"); } + const attachmentIdentity = sourceAttachmentTarget ?? desiredIdentity; const runAttachTemplate = rotatedRunAttachPayload( controlPlaneState, - desiredIdentity, + attachmentIdentity, this.#authorizedTools, this.options.resumeCompletionContract, runtimeContext, @@ -5816,8 +5893,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { runAttachTemplate.provider = environmentPrepared.provider; } this.#runAttachTemplate = structuredClone(runAttachTemplate); - core.persistRunAttachTemplate(runAttachTemplate); - core.queueCommand("run.attach", runAttachTemplate); + if (sourceAttachmentTarget) { + await this.options.onStagedRunAttachment?.(core, { sourceIdentity: desiredIdentity, + targetIdentity: sourceAttachmentTarget, templateDigest: commandDigest(runAttachTemplate) }); + core.stageRunAttachment(sourceAttachmentTarget, runAttachTemplate); + } else { + core.persistRunAttachTemplate(runAttachTemplate); + core.queueCommand("run.attach", runAttachTemplate); + } } const committedEvents = core.store.state.committedEvents; const runAttachment = recoveredRunAttachment(core.store.state); @@ -5829,7 +5912,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? `command_resume_probe_${randomUUID().replaceAll("-", "")}` : null; if (recoveryProbeCommandId !== null) { - core.queueCommand("runner.drain", {}, recoveryProbeCommandId); + core.queueCommand(sourceAttachmentTarget ? "session.open" : "runner.drain", {}, recoveryProbeCommandId); } // A controller retry can open the exact authority after run.attach has // already reached a durable outcome. Re-observe that command instead of @@ -5878,6 +5961,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : "restored provider identity from the exact durable checkpoint; awaiting live confirmation", ); } + if (staged && !sourceAttachmentTarget) { + await this.options.onStagedRunAttachment?.(core, { sourceIdentity: staged.sourceIdentity, + targetIdentity: staged.targetIdentity, templateDigest: staged.templateDigest }); + if (warmRecovery && this.options.adoptExistingRunner) core.prepareWarmTransitionAdoption({ + transitionId: warmRecovery.transitionId, runnerState: warmRecovery.runnerState }); + } type RecoveryRegistration = Awaited< ReturnType< NonNullable< @@ -5888,67 +5977,48 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { let oldTransitionRegistration: RecoveryRegistration | null = null; let newTransitionRegistration: RecoveryRegistration | null = null; this.#assertStartupOpen(); - if (warmRecovery && this.options.controlPlaneRegistration) { + const recoveringAttachment = warmRecovery || queuedSourceAttachment; + const queuedCommand = queuedSourceAttachment ? core.getCommand(queuedSourceAttachment.commandId!) : undefined; + const expectedRecoveryConnection = (warmRecovery?.proof.receipt.connection + ?? record(queuedCommand?.payload.paperclipNextAuthority).connection) as RunnerProcessConnection | undefined; + const recoverySource = warmRecovery?.proof.receipt.oldIdentity ?? queuedSourceAttachment?.sourceIdentity; + const recoveryTarget = warmRecovery?.proof.receipt.newIdentity ?? queuedSourceAttachment?.targetIdentity; + if (recoveringAttachment && this.options.controlPlaneRegistration) { try { - oldTransitionRegistration = await this.options.controlPlaneRegistration( - core, - warmRecovery.proof.receipt.oldIdentity, - ); - const oldConnection = oldTransitionRegistration.connection ?? { - mode: "connect", - connectUrl: oldTransitionRegistration.connectUrl, - }; - if ( - oldConnection.mode !== "connect" || - typeof oldConnection.connectUrl !== "string" - ) { - throw new Error( - "native_runner_warm_transition_registered_endpoint_mismatch", - ); + if (!recoverySource || !recoveryTarget || !expectedRecoveryConnection + || (expectedRecoveryConnection.mode !== "connect" && !(staged && this.options.adoptExistingRunner + && this.options.warmTransitionRegistrationMode === "routed_same_endpoint"))) { + throw new Error("native_runner_warm_transition_requires_exact_owned_endpoint"); } + oldTransitionRegistration = await this.options.controlPlaneRegistration(core, recoverySource); this.#assertStartupOpen(); - newTransitionRegistration = await this.options.controlPlaneRegistration( - core, - warmRecovery.proof.receipt.newIdentity, - ); - const newConnection = newTransitionRegistration.connection ?? { - mode: "connect", - connectUrl: newTransitionRegistration.connectUrl, - }; - if ( - newConnection.mode !== "connect" || - durableRecoveryInternals.canonicalJson(newConnection) !== - durableRecoveryInternals.canonicalJson( - warmRecovery.proof.receipt.connection, - ) - ) { - throw new Error( - "native_runner_warm_transition_registered_endpoint_mismatch", - ); + newTransitionRegistration = await this.options.controlPlaneRegistration(core, recoveryTarget); + const connection = newTransitionRegistration.connection ?? { mode: "connect", connectUrl: newTransitionRegistration.connectUrl }; + if (durableRecoveryInternals.canonicalJson(connection) !== durableRecoveryInternals.canonicalJson(expectedRecoveryConnection)) { + throw new Error("native_runner_warm_transition_registered_endpoint_mismatch"); } } catch (error) { - await Promise.allSettled( - [oldTransitionRegistration, newTransitionRegistration].map((entry) => - Promise.resolve().then(() => entry?.release()), - ), - ); + await Promise.allSettled([oldTransitionRegistration, newTransitionRegistration].map(entry => Promise.resolve().then(() => entry?.release()))); throw error; } } - const registration = - warmRecovery && oldTransitionRegistration && newTransitionRegistration - ? recoveryIdentityMatches( - warmRecovery.runnerIdentity, - warmRecovery.proof.receipt.oldIdentity, - ) - ? oldTransitionRegistration - : newTransitionRegistration - : this.options.controlPlaneRegistration - ? await this.options.controlPlaneRegistration(core) - : null; + const registration = oldTransitionRegistration && newTransitionRegistration + ? recoveryIdentityMatches(warmRecovery?.runnerIdentity ?? queuedSourceAttachment!.sourceIdentity, recoverySource!) + ? oldTransitionRegistration : newTransitionRegistration + : this.options.controlPlaneRegistration + ? await this.options.controlPlaneRegistration(core, this.#sourceRecoveryIdentity ?? undefined) : null; this.#startupFailureCode = registration?.startupFailureCode ?? "runner_local_connect_failed"; - if (registration === null) await core.start(warmRecovery?.port); + if (registration === null) { + let queuedPort: number | undefined; + if (queuedSourceAttachment) { + const endpoint = expectedRecoveryConnection?.mode === "connect" ? new URL(expectedRecoveryConnection.connectUrl) : null; + if (!endpoint || endpoint.protocol !== "ws:" || endpoint.hostname !== "127.0.0.1" || endpoint.pathname !== "/durableRecovery/connect" + || !endpoint.port || endpoint.search || endpoint.hash || endpoint.username || endpoint.password) throw new Error("native_runner_warm_transition_snapshot_mismatch"); + queuedPort = Number(endpoint.port); + } + await core.start(warmRecovery?.port ?? queuedPort); + } else { this.#controlPlaneCheckpoint = registration.checkpoint ?? null; this.#controlPlaneRelease = @@ -5969,7 +6039,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#assertStartupOpen(); // Route and immutable endpoint admission precedes every launch-material // write. A refused transition leaves its retained home/context untouched. - if (runtimeContext !== null && !this.options.prepareLaunch) { + if (runtimeContext !== null && !this.options.prepareLaunch && this.#sourceRecoveryIdentity === null) { writeFileSync( localRuntimeContextPath, `${JSON.stringify(runtimeContext)}\n`, @@ -5978,7 +6048,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (provider === "aws_agentcore" && !this.options.prepareLaunch) mkdirSync(codexHome, { recursive: true, mode: 0o700 }); - if (!this.options.prepareLaunch && provider === "codex" && this.options.homeMode !== "preconfigured") { + if (!this.options.prepareLaunch && provider === "codex" && this.options.homeMode !== "preconfigured" && this.#sourceRecoveryIdentity === null) { await releaseMaterializedNativeRuntimeSkills( resolve(localCodexHome, "skills"), ); @@ -6018,7 +6088,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } // A replacement executor restores its cold provider before acknowledging recovery. this.#assertStartupOpen(); - const admissionDeadline = this.#coldAdmissionDeadline(adoptedRunner === undefined); + const admissionDeadline = this.#sourceAttachmentDeadline === null + ? this.#coldAdmissionDeadline(adoptedRunner === undefined) + : Math.min(Date.now() + 30_000, this.#sourceAttachmentDeadline); const handle = adoptedRunner ? null : spawnRunner({ @@ -6064,18 +6136,30 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await this.#publishSpawnedProcess(handle); } this.#assertStartupOpen(); + let targetRecoveryActivated = registration === newTransitionRegistration; if (oldTransitionRegistration && newTransitionRegistration) { - await oldTransitionRegistration.activate?.(); + // A guest listener still serves only its actual physical run. Open that + // route first; probing the future/retired path cannot prove readiness. + await registration?.activate?.(); this.#assertStartupOpen(); - await newTransitionRegistration.activate?.(); + if (registration === oldTransitionRegistration) { + // Source authentication can consume its receipt and disappear before + // a sampled count sees it. The durable/physical proof loop below owns + // that progress; only target authentication completes recovery. + if (expectedRecoveryConnection?.mode !== "listen") { + await newTransitionRegistration.activate?.(); targetRecoveryActivated = true; + } + } } else await registration?.activate?.(); this.#assertStartupOpen(); - if (registration?.failure) { - void registration.failure.catch((error: unknown) => { - this.#failTransport( - error instanceof Error ? error : new Error(String(error)), - ); - }); + if (registration?.failure && !recoveringAttachment) { + void registration.failure.catch((error: unknown) => this.#failTransport(error instanceof Error ? error : new Error(String(error)))); + } else if (recoveringAttachment) { + // The old listener can retire normally during recovery. Authentication + // and the shared deadline, rather than its stale readiness promise, own + // the transition. Only the target remains live after activation. + void oldTransitionRegistration?.failure?.catch(() => undefined); + void newTransitionRegistration?.failure?.catch(() => undefined); } if (!adoptedRunner) await this.#awaitRegistrationReady(registration?.ready); this.#assertStartupOpen(); @@ -6088,22 +6172,40 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#evidence.runnerProcessGroupId = handle?.processGroupId ?? null; } this.#publish(); - if (warmRecovery) { - const deadline = - Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000); + if (recoveringAttachment) { + const deadline = Math.min(Date.now() + (this.options.runnerReconnectGraceMs ?? 5_000), this.#sourceAttachmentDeadline ?? Infinity); + let sourceRouteRetired = false, nextPhysicalInspection = 0, previousConnections = core.activeRunnerConnectionCount(); while ( - !recoveryIdentityMatches(core.store.state.identity, desiredIdentity) || - core.store.state.warmTransition !== undefined + !recoveryIdentityMatches(core.store.state.identity, recoveryTarget!) || + core.store.state.warmTransition !== undefined || core.activeRunnerConnectionCount() !== 1 ) { this.#assertStartupOpen(); + const connections = core.activeRunnerConnectionCount(); + if (connections === 0 && previousConnections > 0) nextPhysicalInspection = 0; + previousConnections = connections; + if (!sourceRouteRetired && expectedRecoveryConnection?.mode === "listen" && oldTransitionRegistration + && newTransitionRegistration && this.options.readRunnerState && Date.now() >= nextPhysicalInspection) { + nextPhysicalInspection = Date.now() + 250; + const physical = await this.options.readRunnerState(); + const proof = inspectWarmRunTransition({ controlPlaneState: core.store.state, runnerState: physical, + expectedNewIdentity: recoveryTarget!, expectedRunnerVersion: runnerArtifact.version, expectedRunnerDigest: runnerArtifact.digest }); + if (proof && recoveryIdentityMatches(proof.runnerIdentity, recoveryTarget!)) { + // The same participant has consumed the old ACK and now serves + // the target path. Stop obsolete ingress retries before they can + // compete with the target's authenticated connection. + await oldTransitionRegistration.release(); sourceRouteRetired = true; + if (!targetRecoveryActivated) { await newTransitionRegistration.activate?.(); targetRecoveryActivated = true; } + } + } if (Date.now() >= deadline || (await this.#runnerHasExited())) { throw new Error("native_runner_warm_transition_recovery_pending"); } await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } - if (core.getCommand(warmRecovery.commandId)?.status !== "completed") { + if (core.getCommand(warmRecovery?.commandId ?? queuedSourceAttachment!.commandId!)?.status !== "completed") { throw new Error("native_runner_warm_transition_result_unproven"); } + desiredIdentity = recoveryTarget!; this.#durableTurnId = desiredIdentity.turnId; this.#eventIdentity = structuredClone(desiredIdentity); this.#eventSourceSeq = 0; @@ -6112,8 +6214,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await oldTransitionRegistration.release(); this.#assertStartupOpen(); this.#controlPlaneRelease = newTransitionRegistration.release; - this.#controlPlaneCheckpoint = - newTransitionRegistration.checkpoint ?? null; + this.#controlPlaneCheckpoint = newTransitionRegistration.checkpoint ?? null; + void newTransitionRegistration.failure?.catch(error => this.#failTransport(error instanceof Error ? error : new Error(String(error)))); } } this.#assertStartupOpen(); @@ -6121,7 +6223,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (adoptedRunner) { await this.#awaitAdoptedRunnerConnection( adoptedRunner, - registration?.ready, + newTransitionRegistration?.ready ?? registration?.ready, ); } // Only a replacement executor can be restoring a cold Pi process. Share @@ -6130,7 +6232,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await this.#waitCommand("run.attach", runAttachment.commandId, admissionDeadline, true); } if (recoveryProbeCommandId !== null) { - await this.#waitCommand("runner.drain", recoveryProbeCommandId, admissionDeadline, true); + await this.#waitCommand(sourceAttachmentTarget ? "session.open" : "runner.drain", recoveryProbeCommandId, admissionDeadline, true); if (this.#checkpointProviderIdentityExpectation !== null) { // A replacement runner can restore the exact provider while its fresh // session.resumed event is compacted or delayed behind the completed @@ -6159,8 +6261,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { admissionDeadline, ); this.#startupComplete = true; + if (recoveringAttachment && staged) { + this.#sourceRecoveryIdentity = null; + this.#sourceAttachmentDeadline = null; + } + if (sourceAttachmentTarget) { + await this.#awaitWarmRunAttachmentReady(); + if (!core.semanticToolResultsSettled()) throw new Error("native_runner_source_semantic_reconciliation_required"); + if (runtimeContext !== null) writeFileSync(localRuntimeContextPath, `${JSON.stringify(runtimeContext)}\n`, { mode: 0o600 }); + await this.#attachRunWork(sourceAttachmentTarget); + this.#sourceRecoveryIdentity = null; + this.#sourceAttachmentDeadline = null; + } this.#diagnostic( - rotatedAuthority + rotatedAuthority || sourceAttachmentTarget ? "runnerd attached the durable provider session to a fresh PRP run authority" : "runnerd restored its durable PRP session and provider thread", ); @@ -6169,6 +6283,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } async #recoverSemanticToolInput(call: Parameters>[0]) { + if (this.#sourceRecoveryIdentity) throw new Error("native_runner_source_semantic_reconciliation_required"); if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { return this.#handleSemanticToolInput(call); } @@ -6181,6 +6296,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { NonNullable >[0], ) { + if (this.#sourceRecoveryIdentity) throw new Error("native_runner_source_semantic_reconciliation_required"); try { this.#throwIfFailed(); } catch { @@ -6522,6 +6638,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await completing; if (this.#pendingWarmRecoveryCompletion === completion) { this.#pendingWarmRecoveryCompletion = null; + this.#recoverPendingSemanticInputsWhenReady(); } } finally { if (this.#warmRecoveryCompletionInFlight === completing) { diff --git a/packages/paperclip-runner/src/live/runnerd-staged-handoff.test.ts b/packages/paperclip-runner/src/live/runnerd-staged-handoff.test.ts new file mode 100644 index 00000000000..23a465d4b56 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-staged-handoff.test.ts @@ -0,0 +1,241 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer, request } from "node:http"; +import { createHash, randomBytes } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { + DurablePrpControlPlane, inspectStagedRunAttachmentHeader, + type DurablePrpCheckpoint, type RunnerProcessConnection, type PrpWireConnection, +} from "../control-plane/durable-prp-control-plane.js"; +import { + createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary, + readRunnerdArtifactBinding, +} from "./runnerd-codex-transport.js"; + +// The guest rejects WebSocket extensions. Use the same uncompressed wire +// contract as production, without importing the server connector into Runner. +async function openOwnedWire(url: string): Promise<{ wire: PrpWireConnection; closed: Promise; close(): void }> { + return new Promise((resolvePeer, reject) => { + const key = randomBytes(16).toString("base64"); + const req = request(url.replace("ws:", "http:"), { headers: { + Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Version": "13", "Sec-WebSocket-Key": key, + } }); + req.on("error", reject); req.on("response", response => { response.resume(); reject(new Error(`Guest upgrade refused: ${response.statusCode}`)); }); + req.setTimeout(1_000, () => req.destroy(new Error("Guest upgrade timeout"))); + req.on("upgrade", (response, socket, head) => { + socket.setTimeout(0); + const expected = createHash("sha1").update(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest("base64"); + if (response.headers["sec-websocket-accept"] !== expected) { socket.destroy(); reject(new Error("Invalid guest upgrade")); return; } + const listeners: Array<(value: unknown) => void> = [], closes: Array<() => void> = []; + let buffer: Buffer = head, ended = false; + let closeResolve!: () => void; + const closed = new Promise(done => { closeResolve = done; }); + const finish = () => { if (ended) return; ended = true; for (const close of closes) close(); closeResolve(); }; + const send = (payload: Buffer, opcode: number) => { + const mask = randomBytes(4), prefix = payload.length < 126 ? Buffer.from([0x80 | opcode, 0x80 | payload.length]) : Buffer.alloc(payload.length <= 65535 ? 4 : 10); + if (payload.length >= 126) { prefix[0] = 0x80 | opcode; prefix[1] = 0x80 | (payload.length <= 65535 ? 126 : 127); + if (payload.length <= 65535) prefix.writeUInt16BE(payload.length, 2); else prefix.writeBigUInt64BE(BigInt(payload.length), 2); } + const masked = Buffer.from(payload); for (let index = 0; index < masked.length; index++) masked[index] ^= mask[index % 4]!; + socket.write(Buffer.concat([prefix, mask, masked])); + }; + const parse = () => { + while (buffer.length >= 2) { + const opcode = buffer[0]! & 15, initial = buffer[1]! & 127; + if (buffer.length < (initial === 126 ? 4 : initial === 127 ? 10 : 2)) return; + const size = initial === 126 ? buffer.readUInt16BE(2) : initial === 127 ? Number(buffer.readBigUInt64BE(2)) : initial; + const offset = initial === 126 ? 4 : initial === 127 ? 10 : 2; + if (buffer.length < offset + size) return; + const payload = buffer.subarray(offset, offset + size); buffer = buffer.subarray(offset + size); + if (opcode === 8) { socket.destroy(); finish(); return; } + if (opcode === 9) send(payload, 10); + if (opcode === 1 || opcode === 2) for (const listener of listeners) listener(JSON.parse(payload.toString("utf8"))); + } + }; + socket.on("data", bytes => { buffer = Buffer.concat([buffer, bytes]); parse(); }); socket.on("close", finish); socket.on("error", finish); + const close = () => { socket.destroy(); finish(); }; + resolvePeer({ wire: { sendJson: value => send(Buffer.from(JSON.stringify(value)), 1), close, + onJson: listener => { listeners.push(listener); }, onClose: listener => { closes.push(() => listener({})); } }, closed, close }); + queueMicrotask(parse); + }); req.end(); + }); +} + +// Real encrypted PRP and real Runner/provider processes; no model or server imports. +it.each(["connect", "listen"] as const)("recovers a staged %s handoff at every durable boundary without another process", async mode => { + const root = await mkdtemp(join(tmpdir(), "runner-staged-handoff-")); + const shared = join(root, "shared"), calls = join(root, "calls.log"); + const source = { runnerInstanceId: "runner-staged", environmentLeaseId: "lease-staged", runId: "source-run", + normalizedSessionId: "session-staged", turnId: "source-turn", itemId: "source-item" }; + const target = { ...source, runId: "target-run", turnId: "target-turn", itemId: "target-item" }; + const http = createServer(); + const routes = new Map(); + http.on("upgrade", (request, socket, head) => { + const route = routes.get(request.url ?? ""); + if (route) route.core.handleUpgrade(request, socket, request.url!, head); else socket.destroy(); + }); + await new Promise(done => http.listen(0, "127.0.0.1", done)); + const address = http.address(); + if (!address || typeof address === "string") throw new Error("Missing owned endpoint"); + const guest = createServer(); + await new Promise(done => guest.listen(0, "127.0.0.1", done)); + const guestAddress = guest.address(); + if (!guestAddress || typeof guestAddress === "string") throw new Error("Missing owned guest endpoint"); + const guestPort = guestAddress.port; + await new Promise(done => guest.close(() => done())); + const within = async (promise: Promise, label: string, ms = 8_000): Promise => { + let timer: ReturnType | undefined; + try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`staged fixture timeout: ${label}`)), ms); })]); } + finally { clearTimeout(timer); } + }; + const dead = (pid: number) => { try { process.kill(pid, 0); return false; } catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; } }; + // Each boundary uses a fresh company-free fixture so no previous command or + // activation can satisfy a later assertion. + try { + for (const boundary of ["queued", "prepared", "activated", "completed"] as const) { + const directory = join(shared, boundary), callLog = `${calls}-${boundary}`; + let checkpoint!: DurablePrpCheckpoint, currentCore!: DurablePrpControlPlane; + let lost = false, release!: () => void, reached!: () => void; + const held = new Promise(done => { release = done; }); + const atBoundary = new Promise(done => { reached = done; }); + const cores = new Set(); + const registered = new WeakSet(); + let generation = 0; + const registration = async (core: DurablePrpControlPlane, identity = core.store.state.identity) => { + currentCore = core; + if (!registered.has(core)) { + registered.add(core); cores.add(core); + const epoch = generation; + core.setCheckpointPersistence(async saved => { + if (epoch !== generation) throw new Error("retired fixture controller"); + checkpoint = saved; + const proof = inspectStagedRunAttachmentHeader(saved.records.get("header")!); + if (!lost && proof?.phase === (boundary === "queued" ? "attachment_queued" : boundary)) { + lost = true; reached(); await held; + if (epoch !== generation) throw new Error("retired fixture controller"); + } + }); + } + const path = `/api/runner/v1/connect/${identity.runId}`, token = Symbol(path); + const connection: RunnerProcessConnection = mode === "connect" + ? { mode: "connect", connectUrl: `ws://127.0.0.1:${address.port}${path}` } + : { mode: "listen", listenAddress: "0.0.0.0", listenPort: guestPort, listenPath: path }; + if (mode === "connect") { + routes.set(path, { core, token }); + return { connection, release: () => { if (routes.get(path)?.token === token) routes.delete(path); } }; + } + let stopped = false, socket: { close(): void } | undefined; + let readyResolve!: () => void, readyReject!: (error: Error) => void; + const ready = new Promise((done, reject) => { readyResolve = done; readyReject = reject; }); + void ready.catch(() => undefined); + const connect = async () => { + const deadline = Date.now() + 8_000; + while (!stopped && Date.now() < deadline) { + try { + const peer = await openOwnedWire(`ws://127.0.0.1:${guestPort}${path}`); socket = peer; + if (stopped) { peer.close(); return; } + core.attachWireConnection(peer.wire); readyResolve(); await peer.closed; + } catch { /* The target listener appears only after the old receipt ACK. */ } + // Match the production ingress connector's first retry delay (250ms). + if (!stopped) await new Promise(done => setTimeout(done, 250)); + } + if (!stopped) readyReject(new Error("Owned listener did not become available")); + }; + return { connection, activate: () => { void connect(); }, ready: () => ready, + release: () => { stopped = true; socket?.close(); } }; + }; + const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--call-log", callLog, "--record-process-start", "--state-file", join(directory, "fake.json")], + stateDirectory: directory, readRunnerState: async () => JSON.parse(await readFile(join(directory, "runner/runner-state.json"), "utf8")), lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, prpIdentity: source, controlPlaneRegistration: registration, + onStagedRunAttachment: async () => undefined, onRecoveryAccountingEvent: async () => undefined }; + const first = createCapabilityRunnerdCodexTransport(options); + let replacement: ReturnType | undefined; + let runnerPid: number | null = null, providerPid: number | null = null; + try { + const opened = await within(first.transport.request("thread/start", { cwd: root, dynamicTools: [] }), "source open"); + runnerPid = first.evidence().runnerPid; providerPid = first.evidence().providerPid; + const startedAt = first.transport.processInfo().startedAt; + const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); + expect(runnerPid).toBeGreaterThan(0); expect(providerPid).toBeGreaterThan(0); + const attach = first.transport.attachRun!({ runId: target.runId, turnId: target.turnId, itemId: target.itemId }); + void attach.catch(() => undefined); + await within(atBoundary, `reach ${boundary}`); + expect(inspectStagedRunAttachmentHeader(checkpoint.records.get("header")!)?.attachment.targetIdentity).toEqual(target); + generation += 1; + // Disconnect the controller only. The bounded failed commit is released + // after its route is retired; a stale callback cannot publish generation B. + const detached = first.detachControllerForRestart(); void detached.catch(() => undefined); + release(); await within(detached, "detach").catch(() => undefined); + const next = join(root, `replacement-${boundary}`); + checkpoint.restoreSessionDirectory(next); + const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); + const recoveryOptions: Parameters[0] = { ...options, stateDirectory: next, + prpIdentity: target, runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + runnerStateDirectory: join(directory, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(directory, "runner/runner-state.json"), "utf8")), + prepareExternalRunnerState: async () => { throw new Error("Verified live handoff must not restore a backup"); }, + runnerProcessLauncher: noSpawn, warmTransitionRegistrationMode: mode === "listen" ? "routed_same_endpoint" : "routed_connect", + resumeProviderSession: { driverSessionId: String((opened.thread as { id: string }).id), providerSessionId: String((opened.thread as { sessionId: string }).sessionId) }, + adoptExistingRunner: { pid: runnerPid, processGroupId: runnerPid, startedAt, + isAlive: () => !dead(runnerPid!), signal: signal => { if (dead(runnerPid!)) return true; process.kill(-runnerPid!, signal); return true; } } }; + if (boundary === "queued") { + const noProof = createCapabilityRunnerdCodexTransport({ ...recoveryOptions, adoptExistingRunner: undefined }); + await expect(noProof.transport.request("thread/read", {})).rejects.toThrow("queued_attachment_requires_verified_live_source"); + await noProof.detachControllerForRestart().catch(() => undefined); + checkpoint.restoreSessionDirectory(next); + const wrongEndpoint = createCapabilityRunnerdCodexTransport({ ...recoveryOptions, + controlPlaneRegistration: async (core, identity) => { + const registered = await registration(core, identity); + if (identity?.runId !== target.runId) return registered; + return { ...registered, connection: mode === "listen" + ? { mode: "listen", listenAddress: "0.0.0.0", listenPort: guestPort + 1, listenPath: `/api/runner/v1/connect/${target.runId}` } + : { mode: "connect", connectUrl: `ws://127.0.0.1:${address.port}/changed` } }; + } }); + await expect(wrongEndpoint.transport.request("thread/read", {})).rejects.toThrow("registered_endpoint_mismatch"); + await wrongEndpoint.detachControllerForRestart().catch(() => undefined); + expect(noSpawn).not.toHaveBeenCalled(); + checkpoint.restoreSessionDirectory(next); + } + replacement = createCapabilityRunnerdCodexTransport(recoveryOptions); + const read = await within(replacement.transport.request("thread/read", {}), `recover ${boundary}`); + expect(read.thread).toMatchObject({ id: (opened.thread as { id: string }).id, + sessionId: (opened.thread as { sessionId: string }).sessionId }); + expect(dead(providerPid!)).toBe(false); + expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); + expect(replacement.evidence().runnerPid).toBe(runnerPid); + expect(currentCore.activeRunnerConnectionCount()).toBe(1); + expect(currentCore.store.state.identity).toEqual(target); + expect(currentCore.store.state.stagedRunAttachment).toBeDefined(); + expect(currentCore.markStagedTargetCheckpointPublished({ ...target, itemId: "wrong" })).toBe(false); + await currentCore.flushCheckpoint(); + const beforeMark = checkpoint; + expect(currentCore.markStagedTargetCheckpointPublished(target)).toBe(true); + await currentCore.flushCheckpoint(); expect(checkpoint).toBe(beforeMark); + await within(replacement.transport.request("thread/goal/get", {}), "post-checkpoint read"); + expect(currentCore.store.state.stagedRunAttachment).toBeUndefined(); + expect(noSpawn).not.toHaveBeenCalled(); + const effects = (await readFile(callLog, "utf8")).trim().split(/\r?\n/); + expect(effects.filter(line => line === "process-start")).toHaveLength(1); + expect(effects.filter(line => line === "thread/start")).toHaveLength(1); + expect(effects.filter(line => line === "turn/start")).toHaveLength(0); + } catch (error) { + const physical = JSON.parse(await readFile(join(directory, "runner/runner-state.json"), "utf8").catch(() => "{}")); + throw new Error(`${mode}/${boundary}: ${String(error)}; ${JSON.stringify({ + active: currentCore?.activeRunnerConnectionCount(), coreRun: currentCore?.store.state.identity.runId, + coreWarm: currentCore?.store.state.warmTransition?.phase, header: checkpoint && inspectStagedRunAttachmentHeader(checkpoint.records.get("header")!)?.phase, + runnerRun: physical.runId, runnerWarm: physical.warmTransition?.phase, lifecycle: physical.lifecycle, + failureCode: physical.recoverableFailure?.code, failureKeys: Object.keys(physical.recoverableFailure ?? {}), + })}`, { cause: error }); + } finally { + release(); + if (!runnerPid && !replacement) { await first.transport.close().catch(() => undefined); await first.retireProbeResources?.(1_000).catch(() => undefined); } + else await first.detachControllerForRestart().catch(() => undefined); + await replacement?.transport.close().catch(() => undefined); + for (const pid of [providerPid, runnerPid]) if (pid && !dead(-pid)) { try { process.kill(-pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } } + for (const core of cores) await core.stop().catch(() => undefined); + for (const pid of [providerPid, runnerPid]) if (pid) await vi.waitFor(() => expect(dead(pid)).toBe(true)); + } + } + } finally { http.closeAllConnections(); await new Promise(done => http.close(() => done())); await rm(root, { recursive: true, force: true }); } +}, 90_000); diff --git a/server/src/services/native-runtime/native-recovery-usage-delivery.test.ts b/server/src/services/native-runtime/native-recovery-usage-delivery.test.ts new file mode 100644 index 00000000000..c2fdaec2e21 --- /dev/null +++ b/server/src/services/native-runtime/native-recovery-usage-delivery.test.ts @@ -0,0 +1,92 @@ +import { randomUUID } from "node:crypto"; +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DurableRecoveryIdentity, PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js"; + +const accounting = vi.hoisted(() => ({ open: vi.fn(), capture: vi.fn() })); +vi.mock("./native-usage-delivery.js", () => ({ createNativeUsageContinuationDelivery: accounting.open })); +beforeEach(() => { vi.clearAllMocks(); accounting.capture.mockResolvedValue(undefined); accounting.open.mockResolvedValue({ capture: accounting.capture }); }); + +function fixture() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sessionId = randomUUID(); + const source: DurableRecoveryIdentity = { runId: randomUUID(), runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(), + normalizedSessionId: sessionId, turnId: randomUUID(), itemId: randomUUID() }; + const build = (runId: string) => buildNativeExecutionInput({ companyId, agentId, runId, normalizedSessionId: sessionId, + provider: "codex", issue: { id: issueId, identifier: "REC-1", title: "Continue", description: null, workMode: "standard" }, + taskPrompt: "Continue once", workspace: { id: randomUUID(), cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Preserve the result" }] } }, + runtimeContext: nativeRuntimeContextFixture() }); + const execution = build(randomUUID()), previous = build(source.runId); + const row = { run: { status: "succeeded", finishedAt: new Date() as Date | null, runtimeMode: "native", runnerInstanceId: source.runnerInstanceId, + nativeSessionId: sessionId, runnerProfileJson: { nativeExecutionInput: previous } }, + owner: { phase: "committed", resultId: randomUUID() as string | null, leaseOwner: null as string | null, companyId, issueId } }; + const where = vi.fn(async (_predicate: SQL) => [row]); + const select = vi.fn(() => ({ from: () => ({ innerJoin: () => ({ where }) }) })); + const db = { select } as unknown as Db; + const matchesExecution = vi.fn(() => true); + const deliver = () => createNativeRecoveryUsageDelivery({ db, execution, sourceIdentity: { ...source }, matchesExecution }); + const event: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: source.runnerInstanceId, + sourceEventId: "source-usage", sourceSeq: 1, runId: source.runId, normalizedSessionId: sessionId, turnId: source.turnId, + eventType: "usage.reported", priority: 0, emittedAt: new Date().toISOString(), payload: {} }; + return { source, execution, previous, row, select, where, matchesExecution, deliver, event }; +} + +describe("source accounting recovery authority", () => { + it("pins the relational source tuple and hydrates once without binding target accounting", async () => { + const f = fixture(), deliver = f.deliver(); + await deliver(f.event, f.source); + await deliver({ ...f.event, sourceEventId: "source-terminal", sourceSeq: 2, eventType: "turn.completed" }, f.source); + expect(f.select).toHaveBeenCalledOnce(); + expect(new PgDialect().sqlToQuery(f.where.mock.calls[0][0]).params).toEqual([ + f.source.runId, f.execution.binding.companyId, f.execution.binding.agentId, f.execution.binding.issueId, + ]); + expect(f.matchesExecution).toHaveBeenCalledWith(f.previous); + expect(accounting.open).toHaveBeenCalledExactlyOnceWith(expect.anything(), { + companyId: f.execution.binding.companyId, runId: f.source.runId, runnerInstanceId: f.source.runnerInstanceId, provider: f.previous.provider, + }); + expect(accounting.capture).toHaveBeenCalledTimes(2); + expect(accounting.capture).toHaveBeenCalledWith(f.event); + }); + + it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)( + "rejects a changed pinned %s before source hydration", async key => { + const f = fixture(); + await expect(f.deliver()(f.event, { ...f.source, [key]: randomUUID() })).rejects.toThrow("native_recovery_usage_source_unproven"); + expect(f.select).not.toHaveBeenCalled(); + expect(accounting.open).not.toHaveBeenCalled(); + }); + + it.each(["runId", "sourceInstanceId", "normalizedSessionId"] as const)("rejects mismatched event %s before source hydration", async key => { + const f = fixture(); + await expect(f.deliver()({ ...f.event, [key]: randomUUID() }, f.source)).rejects.toThrow("native_recovery_usage_source_unproven"); + expect(f.select).not.toHaveBeenCalled(); + }); + + it.each(["nonterminal", "unfinished", "uncommitted", "no-result", "held", "company", "issue", "scope"])( + "keeps an unproven source held: %s", async change => { + const f = fixture(); + if (change === "nonterminal") f.row.run.status = "running"; + if (change === "unfinished") f.row.run.finishedAt = null; + if (change === "uncommitted") f.row.owner.phase = "observed"; + if (change === "no-result") f.row.owner.resultId = null; + if (change === "held") f.row.owner.leaseOwner = "live-owner"; + if (change === "company") f.row.owner.companyId = randomUUID(); + if (change === "issue") f.row.owner.issueId = randomUUID(); + if (change === "scope") f.matchesExecution.mockReturnValue(false); + await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_recovery_usage_source_unproven"); + expect(accounting.open).not.toHaveBeenCalled(); + expect(accounting.capture).not.toHaveBeenCalled(); + }); + + it("keeps a durable source receipt failure unacknowledged", async () => { + const f = fixture(); + accounting.capture.mockRejectedValue(new Error("native_usage_after_finalization_requires_reconciliation")); + await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + }); +}); diff --git a/server/src/services/native-runtime/native-recovery-usage-delivery.ts b/server/src/services/native-runtime/native-recovery-usage-delivery.ts new file mode 100644 index 00000000000..9401c72dd71 --- /dev/null +++ b/server/src/services/native-runtime/native-recovery-usage-delivery.ts @@ -0,0 +1,47 @@ +import { and, eq } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { parseNativeExecutionInput, type DurableRecoveryIdentity, type NativeExecutionInput, type PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { createNativeUsageContinuationDelivery } from "./native-usage-delivery.js"; + +const identityKeys = ["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const; + +/** A cold handoff may discover accounting that belongs to its completed source. + * Hydrate that producer once; never route its receipts through the new run. */ +export function createNativeRecoveryUsageDelivery(input: { + db: Db; + execution: NativeExecutionInput; + sourceIdentity: Record; + matchesExecution(previous: NativeExecutionInput): boolean; +}) { + let delivery: ReturnType | undefined; + const deny = (): never => { throw new Error("native_recovery_usage_source_unproven"); }; + const open = async (source: DurableRecoveryIdentity) => { + const { binding } = input.execution; + const [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations }) + .from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) + .where(and(eq(heartbeatRuns.id, source.runId), eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId))); + if (!prior || prior.run.status !== "succeeded" || prior.run.finishedAt === null || + prior.run.runtimeMode !== "native" || prior.run.runnerInstanceId !== source.runnerInstanceId || + prior.run.nativeSessionId !== source.normalizedSessionId || prior.owner.phase !== "committed" || + !prior.owner.resultId || prior.owner.leaseOwner || prior.owner.companyId !== binding.companyId || + prior.owner.issueId !== binding.issueId) return deny(); + const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput); + if (previous.binding.companyId !== binding.companyId || previous.binding.agentId !== binding.agentId || + previous.binding.issueId !== binding.issueId || previous.binding.runId !== source.runId || + !input.matchesExecution(previous)) return deny(); + return createNativeUsageContinuationDelivery(input.db, { + companyId: binding.companyId, runId: source.runId, + runnerInstanceId: source.runnerInstanceId, provider: previous.provider, + }); + }; + return async (event: PrpEvent, source: DurableRecoveryIdentity) => { + if (source.runId === input.execution.binding.runId || + identityKeys.some(key => typeof input.sourceIdentity[key] !== "string" || + !input.sourceIdentity[key] || source[key] !== input.sourceIdentity[key]) || + event.runId !== source.runId || event.sourceInstanceId !== source.runnerInstanceId || + event.normalizedSessionId !== source.normalizedSessionId) return deny(); + delivery ??= open(source); + await (await delivery).capture(event); + }; +} diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index f032829c639..fd79245b6e5 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -8,6 +8,18 @@ import { } from "./native-restart-recovery.js"; describe("native restart recovery classification", () => { + it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => { + const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false, + processStartMatches: false, hasCheckpoint, hasProviderEvidence: false }; + expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBeNull(); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true })).toEqual({ + claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification", + }); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true, + checkpointFailed: true }).claimKind).toBeNull(); + expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: false, + stagedAttachmentVerified: true }).claimKind).toBe(hasCheckpoint ? null : "bootstrap_incomplete"); + }); it.each([true, false])("uses the remote fenced lease instead of the old container PID (live=%s)", async live => { const now = new Date(); const probe = vi.fn(() => { throw new Error("must not probe another container's PID"); }); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 4a42c652759..53ccc1b07f7 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -7,7 +7,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js"; -import { hasNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; +import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { @@ -69,6 +70,8 @@ export type NativeRestartRecoveryClaim = recoveryRequestId: string | null; remote: { kind?: "sandbox"; providerLeaseId: string; remoteCwd: string } | { kind: "plugin_task"; leaseId: string }; + /** Discovery evidence only; executor revalidates the encrypted header. */ + stagedAttachment?: Pick; } | { kind: "resume_dead_runner"; @@ -322,6 +325,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: { checkpointIdentityMatches?: boolean; hasProviderEvidence: boolean; checkpointFailed?: boolean; + stagedAttachmentVerified?: boolean; providerAttempt?: number; }): { claimKind: NativeRestartRecoveryClaim["kind"] | null; @@ -336,6 +340,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: { // inspection; the executor must prove the original lease and authenticate // the surviving runner before it can resume delivery. if (input.remoteSandbox) { + if (input.stagedAttachmentVerified === true) { + return { claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification" }; + } if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) { return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" }; } @@ -677,6 +684,17 @@ export async function claimNativeRestartRecoveries(input: { } as const; } + // Only the classic remote path stages attachments. After the owner is + // eligible for takeover, inspect its bounded header even if activation + // already published a runtime snapshot. Never scan command/event payloads. + const stagedHeader = remoteWorkspace && !pluginTask && durableAuthority + ? await loadNativeRunnerCheckpointHeader(tx, { + companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id, + }) : null; + const stagedAttachment = stagedHeader ? await resolveNativeStagedAttachmentRecovery({ + db: tx, currentRun: row.run, header: stagedHeader, + }) : null; + const runnerPidAlive = !remoteProcess && processIsAlive(row.run.processPid); const runnerGroupAlive = @@ -865,6 +883,7 @@ export async function claimNativeRestartRecoveries(input: { knownProviderProcessIdentityAmbiguous: providerProcesses.ambiguousLivePids.length > 0, hasCheckpoint, + stagedAttachmentVerified: stagedAttachment !== null, checkpointIdentityMatches: checkpointIdentityMatches && (!remoteSandbox || remoteWorkspace !== null), @@ -1115,6 +1134,10 @@ export async function claimNativeRestartRecoveries(input: { return { kind: claimKind, ...common, + ...(stagedAttachment ? { stagedAttachment: { + sourceIdentity: stagedAttachment.sourceIdentity, targetIdentity: stagedAttachment.targetIdentity, + templateDigest: stagedAttachment.templateDigest, + } } : {}), remote: pluginTask ? { kind: "plugin_task", leaseId: pluginTask.id } : { providerLeaseId: remoteWorkspace!.providerLeaseId, remoteCwd: remoteWorkspace!.remoteCwd, diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index fa7573ddc05..00859cfd3f0 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -180,6 +180,16 @@ export async function registerNativeRunnerCheckpoint(input: { await input.authority.flushCheckpoint(); } +/** Exact-run bounded metadata read; never loads command/event payload records. */ +export async function loadNativeRunnerCheckpointHeader(db: Pick, binding: Binding): Promise { + const [row] = await db.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), + eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1); + if (!row) return null; + try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); } + catch { return null; } +} + export async function loadNativeRunnerCheckpoint(db: Pick, binding: Binding): Promise { const rows = await db.select().from(nativeRunnerCheckpoints).where(and( eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), diff --git a/server/src/services/native-runtime/native-runner-execution-driver.test.ts b/server/src/services/native-runtime/native-runner-execution-driver.test.ts index 28bc0b34db1..51c79577e02 100644 --- a/server/src/services/native-runtime/native-runner-execution-driver.test.ts +++ b/server/src/services/native-runtime/native-runner-execution-driver.test.ts @@ -6,9 +6,12 @@ import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./ const transport = vi.hoisted(() => ({ register: vi.fn(), connect: vi.fn(), + checkpoint: vi.fn(), + checkpointRegistration: vi.fn(), })); vi.mock("../../realtime/runner-prp-ws.js", () => ({ registerRunnerPrpAuthority: transport.register })); vi.mock("../../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: transport.connect })); +vi.mock("./native-runner-checkpoint.js", () => ({ nativeRunnerCheckpointRegistration: transport.checkpointRegistration })); const binding = { companyId: "company", issueId: "issue", agentId: "agent", runId: "run-a" }; const authority = {} as Parameters["controlPlaneRegistration"]>>[0]; @@ -30,9 +33,41 @@ beforeEach(() => { vi.clearAllMocks(); transport.register.mockResolvedValue({ connectUrl: "ws://127.0.0.1/core", release: vi.fn() }); transport.connect.mockReturnValue({ ready: Promise.resolve(), close: vi.fn(async () => {}) }); + transport.checkpointRegistration.mockReturnValue(transport.checkpoint); + transport.checkpoint.mockResolvedValue(undefined); }); describe("shared Runner execution driver", () => { + it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => { + const prepare = vi.fn(); + const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" }; + const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" }, + controllerPersistence: persistence, prepareRemote: prepare }); + const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", + runId: "run-a", turnId: "turn-a", itemId: "item-a" }; + const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" }; + await driver.onStagedRunAttachment!(authority, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` }); + expect(transport.checkpointRegistration).toHaveBeenCalledExactlyOnceWith({ ...persistence, + binding: { companyId: binding.companyId, issueId: binding.issueId, runId: "run-c" } }); + expect(transport.checkpoint.mock.calls).toEqual([[authority, sourceIdentity], [authority, targetIdentity]]); + expect(prepare).not.toHaveBeenCalled(); + expect(transport.register).not.toHaveBeenCalled(); + expect(transport.connect).not.toHaveBeenCalled(); + }); + + it("installs the target writer first when restoring an already persisted stage", async () => { + const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session", + runId: "run-a", turnId: "turn-a", itemId: "item-a" }; + const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" }; + const restored = { store: { state: { stagedRunAttachment: { sourceIdentity, targetIdentity } } } } as unknown as typeof authority; + const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" }, + controllerPersistence: { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" } }); + await driver.onStagedRunAttachment!(restored, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` }); + expect(transport.checkpoint.mock.calls).toEqual([[restored, targetIdentity], [restored, sourceIdentity]]); + expect(transport.register).not.toHaveBeenCalled(); + expect(transport.connect).not.toHaveBeenCalled(); + }); + it("keeps local execution on the registered loopback route", async () => { const driver = createNativeRunnerExecutionDriver({ target: { kind: "local" }, binding }); const connection = await driver.controlPlaneRegistration!(authority); diff --git a/server/src/services/native-runtime/native-runner-execution-driver.ts b/server/src/services/native-runtime/native-runner-execution-driver.ts index 6bf3b44ded5..0929d99d82a 100644 --- a/server/src/services/native-runtime/native-runner-execution-driver.ts +++ b/server/src/services/native-runtime/native-runner-execution-driver.ts @@ -92,6 +92,20 @@ export async function prepareNativeRemoteRunner(input: RemotePreparation) { export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDriverInput): RunnerExecutionDriver { return { ...input.process, + onStagedRunAttachment: input.controllerPersistence && input.binding.issueId + ? async (authority, proof) => { + // Both journals use the current business lease. Install persistence + // before staging; endpoint acquisition still belongs to attachment. + const register = nativeRunnerCheckpointRegistration({ ...input.controllerPersistence!, + binding: { companyId: input.binding.companyId, issueId: input.binding.issueId!, runId: input.binding.runId }, + }); + // A restored stage already publishes under the target. Its writer + // must exist before registration flushes that restored revision. + const identities = authority.store?.state.stagedRunAttachment + ? [proof.targetIdentity, proof.sourceIdentity] : [proof.sourceIdentity, proof.targetIdentity]; + for (const identity of identities) await register(authority, identity); + } + : undefined, controlPlaneRegistration: async (authority, attachmentIdentity) => { await input.beforeConnect?.(attachmentIdentity, authority); if (input.controllerPersistence && input.binding.issueId) { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 990d9adb4d6..85ccc827d34 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,5 +1,6 @@ import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; +import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js"; import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js"; @@ -105,7 +106,7 @@ import { } from "node:fs"; import { tmpdir, userInfo } from "node:os"; import { DatabaseSync } from "node:sqlite"; -import { nativeRunnerCheckpointPersistence, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { loadNativeRunnerCheckpointHeader, nativeRunnerCheckpointPersistence, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; import { REMOTE_CONTROLLER_LEASE_MS, REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runner-recovery-contract.js"; import type { NativeTaskSessionPublicationContext } from "./native-task-session-publication.js"; import { basename, dirname, join, posix, resolve } from "node:path"; @@ -116,6 +117,7 @@ import type { import type { NativeFinalizationResult } from "@paperclipai/shared"; import type { DurablePrpControlPlane, + DurableRecoveryIdentity, HarnessRuntimeRequestResolution, NativeExecutionInput, NativeRuntimeContextSnapshot, @@ -140,6 +142,7 @@ import { createRunnerdCodexTransport, executeNativeSession, applyNativeSessionGoalControl, + inspectStagedRunAttachment, inspectWarmRunTransition, parseNativeExecutionInput, parsePaperclipQuestionSet, @@ -213,6 +216,7 @@ import { } from "./remote-runner-recovery-diagnostics.js"; import { readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS } from "./remote-runner-recovery-probe.js"; import { inspectRemoteWarmSessionRecovery, type RemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; +import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { emitStartupReuse, environmentStartupReuse } from "@paperclipai/adapter-utils/startup-reuse"; import { persistActivity, publishActivity } from "../activity-log.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; @@ -564,6 +568,7 @@ export type NativeInstructionWorkingCopy = { type WarmNativeSession = { onReleased: () => void; + sessionCheckpointPersisted?: (snapshot: PersistedNativeSession) => void; pluginTaskId?: string; observabilityCompatibilityKey?: string; runId: string; @@ -1805,6 +1810,14 @@ class SessionAuthorityEpoch { this.#assertCurrent(); } + async acceptRecoveryAccountingEvent(event: PrpEvent, capture: () => Promise) { + if (!isNativeUsageAccountingEvent(event)) return; + this.#assertCurrent(); + if (event.runId === this.runId) throw new Error("native_recovery_usage_source_unproven"); + await capture(); + this.#assertCurrent(); + } + definitions() { this.#assertCurrent(); return this.#authority.definitions(); @@ -5114,6 +5127,12 @@ async function migrateRunnerdStateRootForExecution(input: { if (!isSafeNativeStateDirectory(scoped)) { throw new Error("runner_state_directory_unsafe"); } + const staged = await verifyStagedAttachmentRestart(input); + if (staged) { + // Physical inspection belongs to backend admission after lease renewal + // starts. This phase only validates the immutable source/target proof. + return; + } if (hasRetainedWarmTransitionEvidence(scoped)) { // A copied receipt is not restart/process authority. Keep both valid // unsupported and invalid forward evidence out of destructive legacy @@ -5626,6 +5645,11 @@ function readRunnerdDurableIdentity( ).toString("utf8"), ), ); + if (state.schema === "paperclip.runner.durable.control-plane-state.staged-attachment.v1") { + const staged = inspectStagedRunAttachment(state); + if (!staged) throw new Error("native_runner_staged_attachment_binding_mismatch"); + return { ...staged.sourceIdentity }; + } if (state.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA && !(allowExternal && state.schema === "paperclip.runner.external-control-plane-state.v1")) return null; return record(state.identity); @@ -5634,6 +5658,50 @@ function readRunnerdDurableIdentity( } } +function readStagedRunnerdAttachment(root: string) { + const path = resolve(root, "control-plane", "control-plane-state.json"); + if (!existsSync(path)) return null; + const state = record(JSON.parse(readBoundedNativeFile(path, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, + "runner_durable_identity_too_large").toString("utf8"))); + if (state.stagedRunAttachment === undefined && + state.schema !== "paperclip.runner.durable.control-plane-state.staged-attachment.v1") return null; + const staged = inspectStagedRunAttachment(state); + if (!staged) throw new Error("native_runner_staged_attachment_binding_mismatch"); + return staged; +} + +/** A staged attachment has not published a target session. Re-establish its + * exact source and target from private durable evidence before any remote RPC. */ +async function verifyStagedAttachmentRestart(input: { + db: Db; execution: NativeExecutionInput; restartRecovery?: NativeRestartRecoveryClaim; + runnerExecutionTarget?: AdapterExecutionTarget | null; +}, expected?: NativeStagedAttachmentRecovery): Promise { + const root = scopedRunnerdStateRoot(input.execution); + const localStage = readStagedRunnerdAttachment(root); + const staged = expected ?? localStage; + if (!staged) return null; + const deny = (): never => { throw new Error("native_runner_staged_attachment_recovery_unproven"); }; + const target = input.runnerExecutionTarget; + if (input.restartRecovery?.kind !== "reattach_remote_runner" || input.restartRecovery.remote.kind === "plugin_task" || + target?.kind !== "remote" || target.transport !== "sandbox" || target.runnerTask || + target.sandboxLeaseAcquisition?.outcome !== "resumed") return deny(); + const [current] = await input.db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), + eq(heartbeatRuns.agentId, input.execution.binding.agentId), eq(heartbeatRuns.nativeIssueId, input.execution.binding.issueId))); + const header = await loadNativeRunnerCheckpointHeader(input.db, input.execution.binding); + if (!current || current.status !== "running" || !header) return deny(); + const verified = await resolveNativeStagedAttachmentRecovery({ db: input.db, currentRun: current, header, + expectedTarget: staged.targetIdentity, + matchesExecution: (previous, execution) => nativeSessionScopeKey(previous) === nativeSessionScopeKey(execution) && + nativeSessionConfigDigest(previous, "remote") === nativeSessionConfigDigest(execution, "remote"), + }); + if (!verified || nativeSha256(verified.currentExecution) !== nativeSha256(input.execution) || + verified.templateDigest !== staged.templateDigest || + nativeSha256(verified.sourceIdentity) !== nativeSha256(staged.sourceIdentity) || + (localStage && nativeSha256(readStagedRunnerdAttachment(root)) !== nativeSha256(localStage))) return deny(); + return verified; +} + function durableIdentityMatchesExecution( identity: Record | null, execution: NativeExecutionInput, @@ -8180,6 +8248,8 @@ async function executePaperclipNativeSessionWithinScope( const restoredController = input.restartRecovery?.kind === "reattach_remote_runner" ? await restoreNativeRunnerCheckpoint(input.db, input.execution.binding, scopedRunnerdStateRoot(input.execution)) : null; + const stagedRestart = input.restartRecovery?.kind === "reattach_remote_runner" + ? await verifyStagedAttachmentRestart(input) : null; if (input.useRunnerd) await restoreWarmRunnerdController(input); const identityRedactor = createAgentIdentityRedactor(input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY); const redactIdentityText = (text: string) => redactSensitiveText( @@ -9086,7 +9156,8 @@ async function executePaperclipNativeSessionWithinScope( let managedCredentialSession: NativeSession | undefined; let githubAccess: NativeGitHubAccess | undefined; let releaseGitHubRun: (() => void) | undefined; - let persistedWarmSession: PersistedNativeSession | null | undefined; + let persistedWarmSession: PersistedNativeSession | null | undefined = stagedRestart && !stagedRestart.hasTargetCheckpoint + ? stagedRestart.checkpoint : undefined; if (warmSessionId !== null && warmConfigDigest !== null && !input.restartRecovery) { const entry = warmNativeSessions.get(warmSessionId); if (entry) { @@ -9468,6 +9539,12 @@ async function executePaperclipNativeSessionWithinScope( requireSessionCloseBeforeReturn: runnerdBackend !== null || input.instructionWorkingCopy !== undefined, onSessionClosed: input.instructionWorkingCopy?.collectStopped, onCheckpoint: async (snapshot) => { + // The runtime invokes this only after its existing database + // snapshot transaction commits. Retirement coalesces into the + // next PRP command; this notification performs no persistence. + const checkpointOwner = warmSessionId === null ? undefined + : warmNativeSessions.get(warmSessionId)?.sessionCheckpointPersisted; + (checkpointOwner ?? runnerdBackend?.sessionCheckpointPersisted)?.(snapshot); if (snapshot.governedWait) { const startup = nativeSessionStartups.get(input.execution.binding.runId); if (startup) startup.governedWait = true; @@ -9564,11 +9641,15 @@ async function executePaperclipNativeSessionWithinScope( if (existing.ownerToken !== warmSessionOwnerToken) { throw new Error("native_session_supervisor_busy"); } + if (existing.session !== session) { + existing.sessionCheckpointPersisted = runnerdBackend?.sessionCheckpointPersisted; + } existing.session = session; existing.runId = input.execution.binding.runId; existing.issueId = input.execution.binding.issueId; } else warmNativeSessions.set(warmSessionId, { + sessionCheckpointPersisted: runnerdBackend?.sessionCheckpointPersisted, onReleased: () => releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, }), @@ -11442,6 +11523,62 @@ export async function verifyRemoteRunnerReattachment(input: { }; } +/** A crash before source bootstrap leaves a suspended Runner; a crash during + * its drain leaves the same live source. Neither permits a replacement until + * the physical process and all six persisted identity fields are proven. */ +async function verifyStagedRemoteRunner(input: { + recovery: NativeStagedAttachmentRecovery; + execution: NativeExecutionInput; + target: AdapterExecutionTarget | null | undefined; + claim: Extract; +}) { + const { target } = input; + if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { + throw new Error("native_runner_staged_attachment_profile_unsupported"); + } + const sourceIdentity = input.recovery.sourceIdentity; + const stateDirectory = posix.join(target.remoteCwd, ".paperclip-runtime", "paperclip-runner", "sessions", + createHash("sha256").update(sourceIdentity.normalizedSessionId).digest("hex"), "runner"); + const read = () => readRemoteRunnerState({ runner: target.runner!, stateDirectory }); + const state = await read(); + const control = record(JSON.parse(readBoundedNativeFile(resolve(scopedRunnerdStateRoot(input.execution), + "control-plane", "control-plane-state.json"), NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, + "runner_durable_identity_too_large").toString("utf8"))); + const warm = inspectWarmRunTransition({ controlPlaneState: control, runnerState: state, + expectedNewIdentity: input.recovery.targetIdentity, expectedRunnerVersion: input.recovery.runnerVersion, + expectedRunnerDigest: input.recovery.runnerDigest }); + if (!warm && (control.warmTransition !== undefined || state.warmTransition !== undefined)) { + throw new Error("native_runner_staged_attachment_recovery_unproven"); + } + const identity = warm?.runnerIdentity ?? input.recovery.protocolIdentity; + const sameIdentity = (value: Record) => + Object.entries(identity).every(([key, expected]) => value[key] === expected); + if (!sameIdentity(state)) throw new Error("runner_remote_recovery_identity_mismatch"); + const claim = { ...input.claim, runId: identity.runId }; + const verify = () => verifyRemoteRunnerReattachment({ claim, target, identity: { ...identity }, + runId: identity.runId, normalizedSessionId: identity.normalizedSessionId }); + if (state.lifecycle !== "suspended") return verify(); + const settled = (value: Record) => sameIdentity(value) && value.lifecycle === "suspended" && + Array.isArray(value.outbox) && value.outbox.length === 0 && value.pendingTerminalDelivery == null && + value.pendingProviderCleanup == null && value.warmTransition == null; + if (!settled(state)) throw new Error("native_runner_staged_attachment_source_unsettled"); + try { + const retiring = await verify(); + const deadline = Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS; + while (await retiring.isAlive()) { + if (Date.now() >= deadline) throw new Error("native_runner_staged_attachment_source_retiring"); + await new Promise(resolveWait => setTimeout(resolveWait, 100)); + } + } catch (error) { + const diagnostic = readRemoteRunnerRecoveryError(error); + if (diagnostic?.stage !== "process_verification" || diagnostic.reason !== "process_not_running") throw error; + } + // An absent/unsafe marker or a changed process generation never proves exit. + // The package rechecks this same suspended source before launching it. + if (!settled(await read())) throw new Error("native_runner_staged_attachment_source_unsettled"); + return undefined; +} + async function waitForRemoteRunnerProcessIdentity(input: { runner: CommandManagedRuntimeRunner; identityPath: string; @@ -11870,7 +12007,7 @@ export async function createRunnerdBackend(input: { contextSnapshot: Record; }, ) => Promise; -}): Promise { +}): Promise { const sessionScopeId = nativeSessionScopeKey(input.execution); const scopeOwner = executingRunnerdSessionScopes.get(sessionScopeId); if (scopeOwner && scopeOwner !== input.execution.binding.runId) { @@ -12020,11 +12157,16 @@ async function prepareRemoteExternalRunner({ input, target, runner, root, identi return { runnerBinary, readProviderState, ...executionDriver }; } +type ManagedRunnerdBackend = NativeSessionBackend & { + bindManagedSession(session: NativeSession): NativeSession; + sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void; +}; + async function createRunnerdBackendWithinSessionClaim( input: Parameters[0], sessionScopeId: string, retainedTransition?: VerifiedWarmTransitionBinding, -): Promise { +): Promise { const timeController = createRunPreparationTiming(input.execution.binding.runId, "native.controller"); let recoveryPending = retainedTransition !== undefined; const target = input.runnerExecutionTarget ?? { kind: "local" as const }; @@ -12157,6 +12299,17 @@ async function createRunnerdBackendWithinSessionClaim( : durableIdentityMatchesSession(durableIdentity, input.execution) ? durableIdentity : null; + const stagedRecovery = input.restartRecovery?.kind === "reattach_remote_runner" + ? await verifyStagedAttachmentRestart(input) : null; + let stagedRecoveryPending = stagedRecovery !== null; + const recoveryUsageIdentity = stagedRecovery?.sourceIdentity ?? input.remoteWarmRecovery?.identity ?? durableIdentity; + const recoveryUsage = input.onAccountingEvent && recoveryUsageIdentity + ? createNativeRecoveryUsageDelivery({ + db: input.db, execution: input.execution, sourceIdentity: recoveryUsageIdentity, + matchesExecution: previous => nativeSessionScopeKey(previous) === nativeSessionScopeKey(input.execution) && + nativeSessionConfigDigest(previous, target.kind) === nativeSessionConfigDigest(input.execution, target.kind), + }) + : undefined; const effectiveRunnerInstanceId = durableBinding?.runnerInstanceId ?? input.runnerInstanceId; const effectiveEnvironmentLeaseId = @@ -13663,7 +13816,9 @@ async function createRunnerdBackendWithinSessionClaim( ? input.restartRecovery.process : null; const adoptedProcess = - input.remoteWarmRecovery + stagedRecovery && input.restartRecovery?.kind === "reattach_remote_runner" + ? await verifyStagedRemoteRunner({ recovery: stagedRecovery, execution: input.execution, target, claim: input.restartRecovery }) + : input.remoteWarmRecovery ? await (async () => { await input.remoteWarmRecovery!.revalidate(); return verifyRemoteRunnerReattachment({ @@ -13698,6 +13853,10 @@ async function createRunnerdBackendWithinSessionClaim( ) => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_tool_authority_unavailable"); + const attachmentRunId = runnerControlPlane?.store?.state.identity.runId; + if (attachmentRunId && attachmentRunId !== current.runId) { + throw new Error("native_runner_source_semantic_reconciliation_required"); + } return current.execute(call); }; if (requiresRemoteProviderPack) { @@ -13745,8 +13904,19 @@ async function createRunnerdBackendWithinSessionClaim( onAccountingEvent: input.onAccountingEvent ? async event => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_authority_unavailable"); + if (input.remoteWarmRecovery && event.runId === input.remoteWarmRecovery.identity.runId) { + if (!recoveryUsage) throw new Error("native_recovery_usage_source_unproven"); + await current.acceptRecoveryAccountingEvent(event, () => recoveryUsage(event, + input.remoteWarmRecovery!.identity as DurableRecoveryIdentity)); + return; + } await current.acceptAccountingEvent(event); } : undefined, + onRecoveryAccountingEvent: input.onAccountingEvent ? async (event, source) => { + const current = sessionToolAuthorityEpochs.get(sessionScopeId); + if (!current || !recoveryUsage) throw new Error("native_recovery_usage_source_unproven"); + await current.acceptRecoveryAccountingEvent(event, () => recoveryUsage(event, source)); + } : undefined, onCredentialsUpdated: (event) => appendNativeCredentialUpdate(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, @@ -13920,7 +14090,7 @@ async function createRunnerdBackendWithinSessionClaim( input.execution.interactionResponses.length > 0 ? "allow_replacement_after_governed_wait" : undefined), - prpIdentity: input.remoteWarmRecovery ? { + prpIdentity: stagedRecovery ? stagedRecovery.targetIdentity : input.remoteWarmRecovery ? { runnerInstanceId: input.remoteWarmRecovery.identity.runnerInstanceId, environmentLeaseId: input.remoteWarmRecovery.identity.environmentLeaseId, runId: input.remoteWarmRecovery.identity.runId, @@ -13947,10 +14117,19 @@ async function createRunnerdBackendWithinSessionClaim( turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}`, }, - warmTransitionRegistrationMode: retainedTransition - ? "routed_connect" - : undefined, - authorizeWarmTransitionRecovery: retainedTransition + warmTransitionRegistrationMode: stagedRecovery && remoteTarget?.transport === "sandbox" && + remoteTarget.effectiveCapabilities?.runnerWebSocketIngress === true + ? "routed_same_endpoint" + : retainedTransition || stagedRecovery ? "routed_connect" : undefined, + authorizeWarmTransitionRecovery: stagedRecovery + ? async () => { + if (!stagedRecoveryPending) return; + const current = await verifyStagedAttachmentRestart(input, stagedRecovery); + if (!current || current.templateDigest !== stagedRecovery.templateDigest) { + throw new Error("native_runner_staged_attachment_recovery_unproven"); + } + } + : retainedTransition ? async ( stage: "before_bootstrap" | "before_spawn" | "before_authentication", @@ -14025,6 +14204,13 @@ async function createRunnerdBackendWithinSessionClaim( beforeConnect: async (_identity, authority) => { runnerControlPlane = authority; await input.remoteWarmRecovery?.revalidate(); + if (stagedRecovery && authority.store.state.stagedRunAttachment) { + const current = await verifyStagedAttachmentRestart(input, stagedRecovery); + if (!current || current.templateDigest !== stagedRecovery.templateDigest || + nativeSha256(current.targetIdentity) !== nativeSha256(stagedRecovery.targetIdentity)) { + throw new Error("native_runner_staged_attachment_recovery_unproven"); + } + } if (retainedTransition && recoveryPending) { const current = await verifyWarmTransitionRestart(input); if (current.stateFingerprint !== retainedTransition.stateFingerprint) { @@ -14116,6 +14302,15 @@ async function createRunnerdBackendWithinSessionClaim( }; return { bindManagedSession: wrapManagedSession, + sessionCheckpointPersisted: snapshot => { + const staged = runnerControlPlane?.store.state.stagedRunAttachment; + if (staged && snapshot.identity.runId === staged.targetIdentity.runId && + snapshot.identity.sessionId === staged.targetIdentity.normalizedSessionId) { + if (runnerControlPlane!.markStagedTargetCheckpointPublished(staged.targetIdentity)) { + stagedRecoveryPending = false; + } + } + }, descriptor: () => backend.descriptor(), openSession: async (sessionInput) => { const session = await backend.openSession(sessionInput); diff --git a/server/src/services/native-runtime/native-staged-attachment-recovery.test.ts b/server/src/services/native-runtime/native-staged-attachment-recovery.test.ts new file mode 100644 index 00000000000..1c872bba7ab --- /dev/null +++ b/server/src/services/native-runtime/native-staged-attachment-recovery.test.ts @@ -0,0 +1,187 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Db } from "@paperclipai/db"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DurablePrpControlPlane, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js"; +import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; +import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { resolveNativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; + +const companyId = "10000000-0000-4000-8000-000000000001"; +const issueId = "20000000-0000-4000-8000-000000000002"; +const agentId = "30000000-0000-4000-8000-000000000003"; +const sourceRunId = "40000000-0000-4000-8000-000000000004"; +const currentRunId = "50000000-0000-4000-8000-000000000005"; +const normalizedSessionId = "60000000-0000-4000-8000-000000000006"; +const runnerInstanceId = "70000000-0000-4000-8000-000000000007"; +const sourceIdentity = { runId: sourceRunId, normalizedSessionId, runnerInstanceId, + environmentLeaseId: "environment-1", turnId: "opaque-source-turn", itemId: "opaque-source-item" }; +const targetIdentity = { ...sourceIdentity, runId: currentRunId, turnId: "opaque-target-turn", itemId: "opaque-target-item" }; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); + +function execution(runId: string) { + return buildNativeExecutionInput({ companyId, runId, agentId, normalizedSessionId, + issue: { id: issueId, identifier: "TEST-1", title: "Continue", description: null, workMode: "standard" }, + taskPrompt: "Continue normally", provider: "codex", workspace: { id: "workspace-1", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + completionContract: { id: "80000000-0000-4000-8000-000000000008", sha256: `sha256:${"a".repeat(64)}`, + schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Continue", criteria: [{ id: "done", requirement: "Answer" }] } }, + runtimeContext: nativeRuntimeContextFixture(), + }); +} + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "native-staged-proof-")); roots.push(root); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity: sourceIdentity, + expectedRunnerVersion: "0.0.0", expectedRunnerDigest: `sha256:${"f".repeat(64)}` }); + let saved!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async checkpoint => { saved = checkpoint; }); + core.persistRunAttachTemplate({ provider: { provider: "codex", command: "codex" }, authorizedTools: {} }); + await core.flushCheckpoint(); + const sourceHeader = saved.records.get("header")!; + core.stageRunAttachment(targetIdentity, { provider: { provider: "codex", command: "codex" }, authorizedTools: {} }); + await core.flushCheckpoint(); + const header = saved.records.get("header")!; + const base = { companyId, agentId, issueId, nativeIssueId: issueId, runnerInstanceId, + nativeSessionId: normalizedSessionId, runtimeMode: "native", status: "running", finishedAt: null as Date | null }; + const currentRun = { ...base, id: currentRunId, runnerProfileJson: { nativeExecutionInput: execution(currentRunId) } }; + const sourceRun = { ...base, id: sourceRunId, status: "succeeded", finishedAt: new Date(), runnerProfileJson: { + nativeExecutionInput: execution(sourceRunId), nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"), + sessionCheckpoint: { sessionId: "provider-thread-1", providerSessionId: "provider-thread-1", identity: { + companyId, issueId, agentId, runId: sourceRunId, sessionId: normalizedSessionId }, + }, + } }; + const source = { run: sourceRun, coordinator: { runId: sourceRunId, companyId, issueId, + phase: "committed", resultId: "result-1", leaseOwner: null, leaseExpiresAt: null }, resultId: "result-1", resultStatus: "accepted" }; + function database(sourceRows = [source], contents = sourceHeader, envelope = {}) { + const material = createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId, runId: sourceRunId, part: "header", contents, ...envelope })); + const batches: unknown[][] = [sourceRows, [{ material }]]; + const select = vi.fn(() => { + const chain = { from: () => chain, innerJoin: () => chain, where: () => chain, + limit: vi.fn(async () => batches.shift() ?? []) }; + return chain; + }); + return { db: { select } as unknown as Pick, select }; + } + return { header, sourceHeader, source, sourceRun, currentRun, database }; +} + +describe("private staged attachment restart evidence", () => { + it("proves both frozen run bindings before the target has a session snapshot", async () => { + const f = await fixture(); const { db, select } = f.database(); + const matchesExecution = vi.fn(() => true); + const result = await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun, + expectedTarget: targetIdentity, matchesExecution }); + expect(result).toMatchObject({ sourceIdentity, targetIdentity, sourceRun: { id: sourceRunId }, hasTargetCheckpoint: false, targetCheckpoint: null, + protocolIdentity: sourceIdentity, phase: "source_bootstrap", receipt: null }); + expect(matchesExecution).toHaveBeenCalledOnce(); + expect(select).toHaveBeenCalledTimes(2); // one source point lookup and one encrypted header point lookup + expect(f.currentRun.runnerProfileJson).not.toHaveProperty("sessionCheckpoint"); + }); + + it("retains staged evidence when the target runtime has already published its snapshot", async () => { + const f = await fixture(); + const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint); + snapshot.identity.runId = currentRunId; + Object.assign(snapshot, { cursor: "target-cursor", activeTurnId: "target-current-turn", pendingRuntimeRequests: [{ requestId: "target-request" }] }); + Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot }); + const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun }); + expect(result).toMatchObject({ sourceIdentity, targetIdentity, checkpoint: { identity: { runId: currentRunId } }, + hasTargetCheckpoint: true, targetCheckpoint: snapshot }); + expect(f.currentRun.runnerProfileJson).toHaveProperty("sessionCheckpoint", snapshot); + }); + + it("returns the codec's queued phase without assuming the Runner has activated the target", async () => { + const f = await fixture(); const header = JSON.parse(f.header); + // Header inspection deliberately does not certify separate command records. + // Full restoration/authentication owns that later gate; no RPC runs here. + header.state.stagedRunAttachment.commandId = "staged-attach-command"; + const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: JSON.stringify(header), currentRun: f.currentRun }); + expect(result).toMatchObject({ commandId: "staged-attach-command", protocolIdentity: sourceIdentity, + phase: "attachment_queued", receipt: null, targetIdentity }); + }); + + it.each(["runId", "companyId", "issueId", "agentId", "sessionId", "providerSessionId"])( + "rejects a present target checkpoint with changed %s instead of replaying the source", async key => { + const f = await fixture(); const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint); + snapshot.identity.runId = currentRunId; + if (key === "providerSessionId") snapshot.providerSessionId = "foreign"; + else Object.assign(snapshot.identity, { [key]: "foreign" }); + Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot }); + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull(); + }); + + it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)( + "rejects a different expected target %s before querying the source", async key => { + const f = await fixture(); const { db, select } = f.database(); + expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun, + expectedTarget: { ...targetIdentity, [key]: "foreign" } })).toBeNull(); + expect(select).not.toHaveBeenCalled(); + }); + + it.each(["agentId", "companyId", "nativeIssueId", "nativeSessionId", "runnerInstanceId"] as const)( + "rejects mixed source DB %s", async key => { + const f = await fixture(); Object.assign(f.source.run, { [key]: "foreign" }); + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull(); + }); + + it.each(["binding", "schema", "runner", "session"])("rejects current frozen evidence mismatch: %s", async kind => { + const f = await fixture(); const { db, select } = f.database(); + if (kind === "binding") f.currentRun.runnerProfileJson.nativeExecutionInput.binding.agentId = sourceRunId; + if (kind === "schema") Object.assign(f.currentRun.runnerProfileJson.nativeExecutionInput, { schema: "unsupported" }); + if (kind === "runner") f.currentRun.runnerInstanceId = sourceRunId; + if (kind === "session") f.currentRun.nativeSessionId = sourceRunId; + expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun })).toBeNull(); + expect(select).not.toHaveBeenCalled(); + }); + + it.each(["phase", "leaseOwner", "resultStatus", "status", "finishedAt", "frozenRun", "workspace", "provider", "sourceTuple", "artifactVersion", "artifactDigest"])( + "rejects unsupported or stale source evidence: %s", async kind => { + const f = await fixture(); let sourceHeader = f.sourceHeader; + if (kind === "phase") f.source.coordinator.phase = "observed"; + if (kind === "leaseOwner") Object.assign(f.source.coordinator, { leaseOwner: "another-owner", leaseExpiresAt: new Date(0) }); + if (kind === "resultStatus") f.source.resultStatus = "rejected"; + if (kind === "status") f.source.run.status = "running"; + if (kind === "finishedAt") Object.assign(f.source.run, { finishedAt: null }); + if (kind === "frozenRun") f.source.run.runnerProfileJson.nativeExecutionInput.binding.runId = currentRunId; + if (kind === "workspace") f.source.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "foreign"; + if (kind === "provider") Object.assign(f.source.run.runnerProfileJson.nativeExecutionInput.provider, { model: "another-model" }); + if (kind === "sourceTuple") { const h = JSON.parse(sourceHeader); h.state.identity.turnId = "foreign"; sourceHeader = JSON.stringify(h); } + if (kind === "artifactVersion") { const h = JSON.parse(sourceHeader); h.runnerVersion = "other-version"; sourceHeader = JSON.stringify(h); } + if (kind === "artifactDigest") { const h = JSON.parse(sourceHeader); h.runnerDigest = `sha256:${"d".repeat(64)}`; sourceHeader = JSON.stringify(h); } + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], sourceHeader), header: f.header, currentRun: f.currentRun })).toBeNull(); + }); + + it("keeps private executor scope/config verification mandatory when supplied", async () => { + const f = await fixture(); + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun, + matchesExecution: () => false })).toBeNull(); + }); + + it.each(["companyId", "issueId", "runId", "part"])("rejects encrypted source header with another %s", async key => { + const f = await fixture(); + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], f.sourceHeader, { [key]: "foreign" }), + header: f.header, currentRun: f.currentRun })).toBeNull(); + }); + + it("does not accept a source lacking the current tool contract or provider identity", async () => { + const f = await fixture(); + f.source.run.runnerProfileJson.nativeToolContractFingerprint = "old-contract"; + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull(); + f.source.run.runnerProfileJson.nativeToolContractFingerprint = nativeToolContractFingerprintForTarget("remote", "codex"); + f.source.run.runnerProfileJson.sessionCheckpoint.providerSessionId = ""; + expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull(); + }); + + it("rejects corrupted or ordinary headers instead of promoting them to staged evidence", async () => { + const f = await fixture(); + for (const header of [f.sourceHeader, "{}", "not-json"]) { + const { db, select } = f.database(); + expect(await resolveNativeStagedAttachmentRecovery({ db, header, currentRun: f.currentRun })).toBeNull(); + expect(select).not.toHaveBeenCalled(); + } + }); +}); diff --git a/server/src/services/native-runtime/native-staged-attachment-recovery.ts b/server/src/services/native-runtime/native-staged-attachment-recovery.ts new file mode 100644 index 00000000000..e64bdf3402c --- /dev/null +++ b/server/src/services/native-runtime/native-staged-attachment-recovery.ts @@ -0,0 +1,124 @@ +import { isDeepStrictEqual } from "node:util"; +import { and, eq } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { + inspectDurablePrpCheckpointHeader, + inspectStagedRunAttachmentHeader, + parseNativeExecutionInput, + type DurableRecoveryIdentity, + type NativeExecutionInput, + type PersistedNativeSession, + type StagedRunAttachment, +} from "../../vendor/paperclip-runner/index.js"; +import { loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; +import { rebindNativeSessionCheckpoint } from "./native-session-resume.js"; + +type FrozenRun = Pick; + +type StagedHeaderProof = NonNullable>; +export type NativeStagedAttachmentRecovery = Omit & + Pick & { + sourceRun: FrozenRun; + sourceExecution: NativeExecutionInput; + currentExecution: NativeExecutionInput; + /** In-memory input only. It is not a published/resumable target snapshot. */ + checkpoint: PersistedNativeSession; + /** A published target snapshot takes precedence over the source rebound. */ + hasTargetCheckpoint: boolean; + targetCheckpoint: PersistedNativeSession | null; + runnerVersion: string; + runnerDigest: string; +}; + +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record : {}; +} + +function boundTargetCheckpoint(run: FrozenRun, source: PersistedNativeSession): PersistedNativeSession | null { + const snapshot = record(run.runnerProfileJson?.sessionCheckpoint), identity = record(snapshot.identity); + if (typeof snapshot.sessionId !== "string" || !snapshot.sessionId || identity.runId !== run.id || + identity.companyId !== run.companyId || identity.issueId !== run.nativeIssueId || identity.agentId !== run.agentId || + identity.sessionId !== run.nativeSessionId) return null; + // A target checkpoint may advance cursor/goal/tool progress, but cannot change + // the provider session retained by this attachment. Do not rebound it again. + if ((source.providerSessionId != null && source.providerSessionId !== snapshot.providerSessionId) || + (source.providerIdentity != null && !isDeepStrictEqual(source.providerIdentity, snapshot.providerIdentity))) return null; + return structuredClone(snapshot) as unknown as PersistedNativeSession; +} + +function frozenExecution(run: FrozenRun): NativeExecutionInput | null { + try { + const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput); + return run.runtimeMode === "native" && execution.binding.runId === run.id && + execution.binding.companyId === run.companyId && execution.binding.agentId === run.agentId && + execution.binding.issueId === run.nativeIssueId && execution.session.normalizedSessionId === run.nativeSessionId + ? execution : null; + } catch { return null; } +} + +function matchesRun(identity: DurableRecoveryIdentity, run: FrozenRun): boolean { + return identity.runId === run.id && identity.runnerInstanceId === run.runnerInstanceId && + identity.normalizedSessionId === run.nativeSessionId; +} + +/** Staging proves which prior attachment may be inspected, not that it is alive + * or that work may be dispatched. Classification reads only two indexed point + * lookups: the named source/result and its encrypted header. The executor must + * re-read this proof with its private scope/config matcher before any RPC, then + * independently prove the physical Runner and authenticate the target tuple. */ +export async function resolveNativeStagedAttachmentRecovery(input: { + db: Pick; + currentRun: FrozenRun; + header: string; + expectedTarget?: DurableRecoveryIdentity; + matchesExecution?: (previous: NativeExecutionInput, current: NativeExecutionInput) => boolean; +}): Promise { + const proof = inspectStagedRunAttachmentHeader(input.header); + const current = frozenExecution(input.currentRun); + if (!proof || !current || current.provider.kind !== "codex" || !["running", "failed"].includes(input.currentRun.status) || + !matchesRun(proof.attachment.targetIdentity, input.currentRun) || + (input.expectedTarget && !isDeepStrictEqual(proof.attachment.targetIdentity, input.expectedTarget))) return null; + const { attachment } = proof; + const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations, + resultId: nativeRunResults.id, resultStatus: nativeRunResults.schemaStatus }).from(heartbeatRuns) + .innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id), + eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId))) + .innerJoin(nativeRunResults, and(eq(nativeRunResults.id, nativeRunFinalizations.resultId), + eq(nativeRunResults.runId, heartbeatRuns.id), eq(nativeRunResults.companyId, heartbeatRuns.companyId), + eq(nativeRunResults.issueId, heartbeatRuns.nativeIssueId))) + .where(and(eq(heartbeatRuns.id, attachment.sourceIdentity.runId), eq(heartbeatRuns.companyId, input.currentRun.companyId), + eq(heartbeatRuns.agentId, input.currentRun.agentId), eq(heartbeatRuns.nativeIssueId, current.binding.issueId))) + .limit(1); + if (!source || source.run.companyId !== current.binding.companyId || source.run.agentId !== current.binding.agentId || + source.run.nativeIssueId !== current.binding.issueId || !matchesRun(attachment.sourceIdentity, source.run) || + source.run.status !== "succeeded" || source.run.finishedAt === null || source.coordinator.phase !== "committed" || + source.coordinator.companyId !== current.binding.companyId || source.coordinator.issueId !== current.binding.issueId || + source.coordinator.runId !== source.run.id || !source.coordinator.resultId || + source.coordinator.resultId !== source.resultId || source.resultStatus !== "accepted" || + source.coordinator.leaseOwner !== null || source.coordinator.leaseExpiresAt !== null) return null; + const previous = frozenExecution(source.run); + if (!previous || (input.matchesExecution && !input.matchesExecution(previous, current))) return null; + // Reuse the existing provider, tool-contract, workspace, runtime-context and + // normalized-session compatibility owner; no second set of digest formulas. + const checkpoint = rebindNativeSessionCheckpoint({ previousRun: source.run, currentExecution: current, executionTargetKind: "remote" }); + if (!checkpoint || !(typeof checkpoint.providerSessionId === "string" && checkpoint.providerSessionId.length > 0 || + checkpoint.providerIdentity && Object.keys(checkpoint.providerIdentity).length > 0)) return null; + const hasTargetCheckpoint = input.currentRun.runnerProfileJson?.sessionCheckpoint != null; + const targetCheckpoint = hasTargetCheckpoint ? boundTargetCheckpoint(input.currentRun, checkpoint) : null; + if (hasTargetCheckpoint && !targetCheckpoint) return null; + const sourceHeader = await loadNativeRunnerCheckpointHeader(input.db, { + companyId: source.run.companyId, issueId: current.binding.issueId, runId: source.run.id, + }); + if (!sourceHeader || !isDeepStrictEqual(inspectDurablePrpCheckpointHeader(sourceHeader), attachment.sourceIdentity)) return null; + // The codec above has validated this bounded header's artifact fields. A + // staged target cannot silently select different bytes from its source. + const sourceArtifact = JSON.parse(sourceHeader) as { runnerVersion: string; runnerDigest: string }; + if (sourceArtifact.runnerVersion !== proof.runnerVersion || sourceArtifact.runnerDigest !== proof.runnerDigest) return null; + return { ...attachment, sourceRun: source.run, sourceExecution: previous, currentExecution: current, + checkpoint, hasTargetCheckpoint, targetCheckpoint, + protocolIdentity: proof.protocolIdentity, phase: proof.phase, receipt: proof.receipt, + runnerVersion: proof.runnerVersion, runnerDigest: proof.runnerDigest }; +} diff --git a/server/src/services/native-runtime/native-usage-delivery.integration.test.ts b/server/src/services/native-runtime/native-usage-delivery.integration.test.ts index 21eab93cb2c..e59ccb2a965 100644 --- a/server/src/services/native-runtime/native-usage-delivery.integration.test.ts +++ b/server/src/services/native-runtime/native-usage-delivery.integration.test.ts @@ -12,8 +12,8 @@ import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, import { type PrpEvent } from "../../vendor/paperclip-runner/index.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { accountRunCost } from "../run-cost-accounting.js"; -import { usageReceiptSpoolPath } from "../usage-receipts.js"; -import { createNativeUsageDelivery } from "./native-usage-delivery.js"; +import { usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../usage-receipts.js"; +import { createNativeUsageContinuationDelivery, createNativeUsageDelivery } from "./native-usage-delivery.js"; const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url)); const fixtureProfile = process.env.CI ? "release" : "debug"; @@ -44,6 +44,89 @@ describe("producer-owned native usage", () => { payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {}, } as PrpEvent); + const finalizedSource = async () => { + const input = await seed(); + await db.update(heartbeatRuns).set({ runnerInstanceId: input.runnerInstanceId }).where(eq(heartbeatRuns.id, input.runId)); + const delivery = await createNativeUsageDelivery(db, input); + const report = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed"); + await delivery.capture(report); + await delivery.capture(done); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect(await accountRunCost(db, input.runId)).toBe(true); + const { billing: _billing, ...source } = input; + return { input, source, report, done }; + }; + + it("continues exact finalized source receipts without acquiring ownership or resetting accounting", async () => { + const { input, source, report, done } = await finalizedSource(); + const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + await expect(createNativeUsageDelivery(db, input)).rejects.toThrow("native_usage_run_not_available"); + const continuation = await createNativeUsageContinuationDelivery(db, source); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before); + await continuation.capture(report); + await continuation.capture(done); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before); + expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1); + await expect(continuation.capture({ ...report, sourceInstanceId: randomUUID() })).rejects.toThrow("native_usage_source_mismatch"); + await expect(continuation.capture({ ...report, payload: { ...report.payload as object, runDeltaAvailable: false } })) + .rejects.toThrow("native_usage_source_conflict"); + }); + + it("journals an equal finalized source tail using the source billing without repricing", async () => { + const { input, source } = await finalizedSource(); + const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + // Even a structurally supplied extra current-run classification cannot own + // the old receipt: the continuation derives billing from saved source data. + const withCurrentBilling = { ...source, billing: { provider: "anthropic", biller: "current-grant", billingType: "subscription_included" } }; + const continuation = await createNativeUsageContinuationDelivery(db, withCurrentBilling); + await continuation.capture(event(input, 3, "usage.reported")); + const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId)); + expect(receipts).toHaveLength(3); + expect(receipts.find(row => row.sequence === 3)?.receiptJson.receipt).toMatchObject(billing); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25); + }); + + it.each(["tokens", "cost", "incomplete", "unpriceable", "basis"])( + "holds a differing finalized source tail before acknowledgement: %s", async change => { + const { input, source } = await finalizedSource(); + const continuation = await createNativeUsageContinuationDelivery(db, source); + const late = event(input, 3, "usage.reported"); + if (change === "incomplete") (late.payload as Record).runDeltaAvailable = false; + if (change === "basis") delete (late.payload as Record).runDeltaBasis; + if (["tokens", "cost", "unpriceable"].includes(change)) (late.payload as Record).runDelta = { + inputTokens: change === "tokens" ? 150 : change === "unpriceable" ? "unknown" : 120, + outputTokens: 10, providerCostUsd: change === "cost" ? 0.5 : 0.25, + }; + await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25); + }); + + it.each(["company", "run", "runner", "source"])("rejects a changed continuation binding without mutation: %s", async change => { + const { input, source } = await finalizedSource(); + if (change === "source") await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || ${JSON.stringify({ accountingReceiptSourceId: randomUUID() })}::jsonb` }) + .where(eq(heartbeatRuns.id, input.runId)); + const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + await expect(createNativeUsageContinuationDelivery(db, { ...source, + ...(change === "company" ? { companyId: randomUUID() } : {}), + ...(change === "run" ? { runId: randomUUID() } : {}), + ...(change === "runner" ? { runnerInstanceId: randomUUID() } : {}), + })).rejects.toThrow("native_usage_continuation_source_mismatch"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before); + }); + + it("rejects changed immutable source receipt facts during hydration", async () => { + const { input, source } = await finalizedSource(); + const [receipt] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1))); + const changed = structuredClone(receipt.receiptJson) as UsageReceiptEnvelope; + changed.producerEvent!.usage!.runDelta = { inputTokens: 999, outputTokens: 10 }; + await db.update(runUsageReceipts).set({ receiptJson: changed }).where(eq(runUsageReceipts.id, receipt.id)); + await expect(createNativeUsageContinuationDelivery(db, source)).rejects.toThrow("native_usage_source_conflict"); + }); + it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => { const input = await seed(); expect(existsSync(usageReceiptSpoolPath())).toBe(false); diff --git a/server/src/services/native-runtime/native-usage-delivery.ts b/server/src/services/native-runtime/native-usage-delivery.ts index 2bbf52d0d1b..0d6f05d7de8 100644 --- a/server/src/services/native-runtime/native-usage-delivery.ts +++ b/server/src/services/native-runtime/native-usage-delivery.ts @@ -16,11 +16,13 @@ const uuid = (identity: string) => { return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`; }; -export async function createNativeUsageDelivery(db: Db, input: { +type NativeUsageDeliveryInput = { companyId: string; runId: string; runnerInstanceId: string; provider: NativeExecutionInput["provider"]; billing: Pick; -}) { +}; + +export async function createNativeUsageDelivery(db: Db, input: NativeUsageDeliveryInput) { const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`); const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql` coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb || @@ -29,9 +31,39 @@ export async function createNativeUsageDelivery(db: Db, input: { }).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id }); if (!bound) throw new Error("native_usage_run_not_available"); - const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and( + return createNativeUsageReceiver(db, input, sourceId, await loadNativeUsageReceipts(db, input, sourceId)); +} + +/** Drain the authenticated previous producer's accounting without acquiring or + * resetting its ownership. Finalized sources retain normal receipt equality + * and reconciliation rules; this grants no business/tool execution authority. */ +export async function createNativeUsageContinuationDelivery(db: Db, input: Omit) { + const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`); + const [source] = await db.select({ runnerInstanceId: heartbeatRuns.runnerInstanceId, usage: heartbeatRuns.usageJson }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + if (source?.runnerInstanceId !== input.runnerInstanceId || source.usage?.accountingReceiptOwner !== "native_producer" + || source.usage.accountingReceiptSourceId !== sourceId) throw new Error("native_usage_continuation_source_mismatch"); + const saved = await loadNativeUsageReceipts(db, input, sourceId); + if (!saved.length) throw new Error("native_usage_continuation_receipts_missing"); + const classification = (receipt: AdapterUsageCheckpoint) => ({ + provider: receipt.provider, biller: receipt.biller, billingType: receipt.billingType, + }); + const receipts = saved.map(row => (row.receipt as UsageReceiptEnvelope).receipt as AdapterUsageCheckpoint); + const billing = classification(receipts[0]); + if (receipts.some(receipt => receiptFingerprint(classification(receipt)) !== receiptFingerprint(billing) + || receipt.model !== (input.provider.model ?? "unknown"))) throw new Error("native_usage_source_conflict"); + return createNativeUsageReceiver(db, { ...input, billing }, sourceId, saved, true); +} + +function loadNativeUsageReceipts(db: Db, input: Pick, sourceId: string) { + return db.select({ receipt: runUsageReceipts.receiptJson, hash: runUsageReceipts.receiptHash, + sequence: runUsageReceipts.sequence }).from(runUsageReceipts).where(and( eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence)); +} + +function createNativeUsageReceiver(db: Db, input: NativeUsageDeliveryInput, sourceId: string, + saved: Awaited>, verifySaved = false) { const accepted = new Map(); const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null; let observedUsage: Record | null = null; @@ -60,6 +92,12 @@ export async function createNativeUsageDelivery(db: Db, input: { for (const row of saved) { const envelope = row.receipt as UsageReceiptEnvelope; if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer"); + if (verifySaved && (envelope.companyId !== input.companyId || envelope.runId !== input.runId + || envelope.sourceId !== sourceId || envelope.sequence !== row.sequence || envelope.adapterType !== "paperclip_runner" + || envelope.id !== uuid(`${input.runId}:${input.runnerInstanceId}:${envelope.producerEvent.sourceEventId}`) + || receiptFingerprint(envelope) !== row.hash || accepted.has(envelope.producerEvent.sourceEventId))) { + throw new Error("native_usage_source_conflict"); + } accepted.set(envelope.producerEvent.sourceEventId, envelope); observe(envelope); } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 94143837aee..3777cdea771 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -94,6 +94,9 @@ export const RUNNER_PRP_PROTOCOL_MIN = runner.RUNNER_PRP_PROTOCOL_MIN; export const RUNNER_PRP_PROTOCOL_MAX = runner.RUNNER_PRP_PROTOCOL_MAX; export const runnerCodexDynamicToolsFit = runner.runnerCodexDynamicToolsFit; export const inspectWarmRunTransition = runner.inspectWarmRunTransition; +export const inspectStagedRunAttachment = runner.inspectStagedRunAttachment; +export const inspectStagedRunAttachmentHeader = runner.inspectStagedRunAttachmentHeader; +export type StagedRunAttachment = import("@paperclipai/paperclip-runner").StagedRunAttachment; export const readRunnerdArtifactBinding = runner.readRunnerdArtifactBinding; export const readLinuxProcessStartedAt = runner.readLinuxProcessStartedAt; export const NativeSessionCloseUnrecoverableError =