diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 7c47d59f1ad..31b869b688a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1768,6 +1768,28 @@ not grant retry authority or change the budget. When the budget is exhausted, the task is blocked and its recovery action states that provider startup failed before any provider work began. +An ingress endpoint failure can use that same bounded retry only when the +environment provider returns a structured HTTP 429 or 5xx failure and the +complete, encrypted controller checkpoint proves that this exact run never +issued a Runner credential. The proof requires the new persisted v2 state and +its retained origin and credential history. Old or partial checkpoints, warm +handoffs, prior credentials, delivery or provider progress do not qualify. The +executor rechecks the checkpoint fingerprint and current execution claim under +the checkpoint owner's lock. Stop or a changed claim prevents settlement. +Authentication, permission and configuration failures do not gain retry authority +from a zero-usage proof. A terminal failed run with that proof can settle its +usage as zero through the existing accounting owner. This does not establish +physical termination, release a workspace hold, or create a semantic result. + +The automatic next attempt can admit absent remote Runner state only after it +rechecks that proof against both the encrypted checkpoint and its complete local +projection under the exact current claim. The frozen execution envelope must +still match. It validates raw local state before checkpoint projection can omit +streaming events or normalize event policy. It keeps the original pending `run.prepare` and `session.open` +command IDs. A proved unused Codex home can rematerialize its sealed skills for +that attempt. Provider progress, issued credentials, legacy state, Stop and +configuration changes retain the normal state-identity hold. + A failed, unowned native sandbox retry can outlive an explicit Retry that already completed successfully. Startup recovery and ordinary retry admission retire that old retry only with the same company, task and agent, a linked completed successor, diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index d1e0be1a1af..a899fdfa42f 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -164,6 +164,22 @@ refuses their old launch identities. ## External controller checkpoints +New controllers create `paperclip.runner.durable.control-plane-state.v2` with +`paperclip.runner.bootstrap-history.v1`. This history retains its original full +run identity, issued credential count and adoption flag across run rotations. +Every launch persists credential issuance before handing the ticket to a process. +Credential pruning cannot remove this history. Older controllers reject v2 state; +current controllers can still restore v1 state without granting it this proof. +Checkpoint decoding and external persistence never add missing origin history. + +`inspectUnclaimedBootstrapCheckpoint` validates the complete four-part bootstrap +journal and exact identity. Only pending `run.prepare` and `session.open` commands +are allowed. Any credential issuance, connection, command delivery, event, warm +transition or unsupported history prevents the proof. Its fingerprint lets an +embedding owner recheck the exact revision under its execution fence. The proof +describes provider usage only. It does not authorize replay or prove that a +sandbox, callback bridge or other process has terminated. + The optional controller persistence callback stores recovery control state, not an output transcript. Text deltas do not trigger callbacks; usage updates are coalesced into the next control checkpoint. Authentication, command intents and diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 3f9736ba32d..dc2ccd7140e 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -33,6 +33,8 @@ import { decodeDurablePrpCheckpoint, captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, + inspectUnclaimedBootstrapCheckpoint, + inspectUnclaimedBootstrapState, inspectStagedRunAttachment, inspectRetainedRunInstructions, inspectStagedRunAttachmentHeader, @@ -57,6 +59,120 @@ const identity: DurableRecoveryIdentity = { const expectedRunnerVersion = "0.3.0"; const expectedRunnerDigest = `sha256:${"a".repeat(64)}`; +it("recognizes a complete unclaimed bootstrap through cold checkpoint restoration", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-unclaimed-bootstrap-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async value => { checkpoint = value; }); + try { + const template = { provider: { provider: "codex", command: "fixture" }, authorizedTools: {} }; + core.persistRunAttachTemplate(template); + core.queueCommand("run.prepare", template); + core.queueCommand("session.open", { reuse: "same_session" }); + await core.flushCheckpoint(); + const proof = inspectUnclaimedBootstrapCheckpoint(checkpoint.records, identity); + expect(proof).toEqual({ fingerprint: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest }); + expect(core.unclaimedBootstrapCheckpoint()).toEqual(proof); + const projection = resolve(root, "projection"); + checkpoint.restoreSessionDirectory(projection); + const local = JSON.parse(readFileSync(resolve(projection, "control-plane", "control-plane-state.json"), "utf8")); + const artifact = { version: expectedRunnerVersion, digest: expectedRunnerDigest }; + expect(inspectUnclaimedBootstrapState(JSON.stringify(local), identity, artifact)).toEqual(proof); + const event = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", + sourceInstanceId: identity.runnerInstanceId, sourceEventId: "local-progress", sourceSeq: 1, + runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, turnId: identity.turnId, + itemId: identity.itemId, eventType: "item.delta", priority: 2, emittedAt: "2026-01-01T00:00:00.000Z", + payload: { delta: "Prior local progress" } }; + const progressed = { ...local, committedEvents: [{ sourceSeq: 1, sourceEventId: event.sourceEventId, + eventType: event.eventType, priority: 2, envelope: { kind: "event", payload: event }, deliveryCount: 1, logicalEffectCount: 1 }] }; + expect(inspectUnclaimedBootstrapState(JSON.stringify(progressed), identity, artifact)).toBeNull(); + delete local.checkpointEventPolicy; + expect(inspectUnclaimedBootstrapState(JSON.stringify(local), identity, artifact)).toBeNull(); + expect(inspectUnclaimedBootstrapCheckpoint(decodeDurablePrpCheckpoint(checkpoint.records).records, identity)).toEqual(proof); + for (const field of Object.keys(identity) as Array) { + expect(inspectUnclaimedBootstrapCheckpoint(checkpoint.records, { ...identity, [field]: "foreign" })).toBeNull(); + } + for (const omitted of ["header", "cursor", "command:1", "command:2"]) { + const truncated = new Map(checkpoint.records); truncated.delete(omitted); + expect(inspectUnclaimedBootstrapCheckpoint(truncated, identity)).toBeNull(); + } + for (const mutate of [ + (header: { state: Record }) => { delete header.state.freshBootstraps; }, + (header: { schema: string }) => { header.schema = "unsupported.checkpoint"; }, + (header: { state: Record }) => { header.state.schema = "paperclip.runner.durable.control-plane-state.v1"; }, + (header: { state: Record }) => { delete header.state.bootstrapHistory; }, + (header: { state: Record }) => { delete header.state.checkpointEventPolicy; }, + (header: { state: Record }) => { header.state.connectionCount = 1; }, + (header: { state: Record }) => { header.state.commandDeliveryCounts = { previouslyDelivered: 1 }; }, + (header: { state: Record }) => { header.state.warmTransition = {}; }, + (header: { state: Record }) => { header.state.stagedRunAttachment = {}; }, + ]) { + const changed = new Map(checkpoint.records), header = JSON.parse(changed.get("header")!); + mutate(header); changed.set("header", JSON.stringify(header)); + expect(inspectUnclaimedBootstrapCheckpoint(changed, identity)).toBeNull(); + } + core.queueCommand("turn.start", { prompt: "must not be treated as unused" }); + expect(core.unclaimedBootstrapCheckpoint()).toBeNull(); + await core.flushCheckpoint(); + expect(inspectUnclaimedBootstrapCheckpoint(checkpoint.records, identity)).toBeNull(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } +}); + +it("retains ever-issued bootstrap history after credential pruning and cold restore", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-issued-bootstrap-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async value => { checkpoint = value; }); + try { + const template = { provider: { provider: "codex", command: "fixture" }, authorizedTools: {} }; + core.persistRunAttachTemplate(template); + core.queueCommand("run.prepare", template); + core.queueCommand("session.open", { reuse: "same_session" }); + core.issueBootstrapTicket(); + expect(core.unclaimedBootstrapCheckpoint()).toBeNull(); + await core.flushCheckpoint(); + const records = new Map(checkpoint.records), header = JSON.parse(records.get("header")!); + expect(header.state.freshBootstraps).toBe(1); + header.state.tickets = {}; records.set("header", JSON.stringify(header)); + const restored = decodeDurablePrpCheckpoint(records); + expect(inspectUnclaimedBootstrapCheckpoint(restored.records, identity)).toBeNull(); + await core.start(); + const client = await authenticate(core, core.issueBootstrapTicket()); + expect(client).not.toBeNull(); + await core.flushCheckpoint(); + expect(inspectUnclaimedBootstrapCheckpoint(checkpoint.records, identity)).toBeNull(); + client!.socket.destroy(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } +}); + +it("never blesses bootstrap history after run identity reuse or an older producer restore", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-bootstrap-history-rotation-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async value => { checkpoint = value; }); + try { + core.issueBootstrapTicket(); + core.store.state.tickets = {}; + const next = { ...identity, runId: "next-run", turnId: "next-turn", itemId: "next-item" }; + core.rotateRunIdentity(next); + core.rotateRunIdentity(identity); + const template = { provider: { provider: "codex", command: "fixture" } }; + core.persistRunAttachTemplate(template); + core.queueCommand("run.prepare", template); + core.queueCommand("session.open", { reuse: "same_session" }); + await core.flushCheckpoint(); + expect(core.store.state.freshBootstraps).toBe(0); + expect(core.store.state.bootstrapHistory).toMatchObject({ credentialsIssued: 1, adopted: true }); + expect(inspectUnclaimedBootstrapCheckpoint(checkpoint.records, identity)).toBeNull(); + const records = new Map(checkpoint.records), header = JSON.parse(records.get("header")!); + delete header.state.bootstrapHistory; + header.state.schema = "paperclip.runner.durable.control-plane-state.v1"; + records.set("header", JSON.stringify(header)); + expect(inspectUnclaimedBootstrapCheckpoint(decodeDurablePrpCheckpoint(records).records, identity)).toBeNull(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } +}); + it("recovers only the matching run's admitted instructions from a valid checkpoint", async () => { const root = mkdtempSync(resolve(tmpdir(), "runner-retained-instructions-")); const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); @@ -99,6 +215,7 @@ it("persists an undispatched source-to-target attachment before bootstrap and pr expect(checkpoint.runId).toBe(target.runId); expect(checkpoint.predecessor).toEqual({ runId: identity.runId, normalizedSessionId: identity.normalizedSessionId }); expect(checkpoint.workCommandIds).toEqual([]); + expect(inspectUnclaimedBootstrapCheckpoint(checkpoint.records, target)).toBeNull(); const proof = inspectStagedRunAttachmentHeader(checkpoint.records.get("header")!); expect(proof).toMatchObject({ attachment: { sourceIdentity: identity, targetIdentity: target, commandId: null } }); expect(proof?.attachment).not.toHaveProperty("runAttachTemplate"); @@ -4505,7 +4622,7 @@ describe("DurablePrpControlPlane", () => { expect(core.store.state.checkpointEventPolicy).toBe("control-v1"); } expect(core.store.state.schema).toBe( - "paperclip.runner.durable.control-plane-state.v1", + "paperclip.runner.durable.control-plane-state.v2", ); await expect(receiveSecure(successor!)).resolves.toMatchObject({ kind: "command", diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 87e7eae5042..218ea36aeec 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -53,7 +53,8 @@ export const RUNNER_PRP_PROTOCOL_MAX = 3; const protocolVersion = RUNNER_PRP_PROTOCOL_MAX; const secureFrameSchema = "paperclip.runner.secure-frame.v1"; const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; +const legacyCoreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; +const coreStateSchema = "paperclip.runner.durable.control-plane-state.v2"; const stagedCoreStateSchema = "paperclip.runner.durable.control-plane-state.staged-attachment.v1"; const externalCoreStateSchema = "paperclip.runner.external-control-plane-state.v1"; const transitionCoreStateSchema = @@ -150,11 +151,17 @@ export interface StagedRunAttachment { } interface StoredCoreState { - schema: typeof coreStateSchema | typeof transitionCoreStateSchema | typeof stagedCoreStateSchema; + schema: typeof coreStateSchema | typeof legacyCoreStateSchema | typeof transitionCoreStateSchema | typeof stagedCoreStateSchema; stagedRunAttachment?: Omit & { runAttachTemplate?: Record }; identity: DurableRecoveryIdentity; /** External checkpoints contain control state, not a duplicate output log. */ checkpointEventPolicy?: "control-v1"; + bootstrapHistory?: { + schema: "paperclip.runner.bootstrap-history.v1"; + originIdentity: DurableRecoveryIdentity; + credentialsIssued: number; + adopted: boolean; + }; warmTransition?: { receipt: DurableWarmRunTransition; phase: "awaiting_result" | "prepared" | "activated"; @@ -439,7 +446,7 @@ export function inspectDurablePrpCheckpointHeader(contents: string): DurableReco const header = parseCheckpointHeader(contents); if (header.schema !== "paperclip.runner.controller-checkpoint.v2" || header.state.checkpointEventPolicy !== "control-v1" - || ![coreStateSchema, transitionCoreStateSchema, stagedCoreStateSchema].includes(String(header.state.schema)) + || ![coreStateSchema, legacyCoreStateSchema, transitionCoreStateSchema, stagedCoreStateSchema].includes(String(header.state.schema)) || (header.state.schema === stagedCoreStateSchema && !validStagedRunAttachment(header.state as unknown as StoredCoreState, true))) return null; return { ...header.state.identity }; } catch { return null; } @@ -523,6 +530,48 @@ export function decodeDurablePrpCheckpoint(records: ReadonlyMap) } } +function isUnclaimedBootstrapState(state: StoredCoreState, expected: DurableRecoveryIdentity): boolean { + const history = state.bootstrapHistory; + return !(state.schema !== coreStateSchema || state.checkpointEventPolicy !== "control-v1" + || canonicalJson(state.identity) !== canonicalJson(expected) + || history?.schema !== "paperclip.runner.bootstrap-history.v1" + || canonicalJson(history.originIdentity) !== canonicalJson(expected) + || history.credentialsIssued !== 0 || history.adopted !== false + || state.warmTransition || state.completedWarmTransition || state.stagedRunAttachment + || state.connectionCount !== 0 || state.ackedSourceSeq !== 0 || state.committedEvents.length !== 0 + || state.freshBootstraps !== 0 || Object.keys(state.tickets).length !== 0 + || Object.keys(state.leases).length !== 0 || state.lastLeaseId !== null || state.lastLeaseExpiresAt !== null + || Object.keys(state.commandDeliveryCounts).length !== 0 || state.replayDeliveries !== 0 + || state.duplicateCommandResults !== 0 || state.malformedFrames !== 0 + || state.commands.length !== 2 || state.commands[0]?.type !== "run.prepare" + || state.commands[1]?.type !== "session.open" + || state.commands.some(command => command.status !== "pending" || command.result !== null) + || !state.runAttachTemplate || canonicalJson(state.commands[0].payload) !== canonicalJson(state.runAttachTemplate)); +} + +export function inspectUnclaimedBootstrapCheckpoint( + records: ReadonlyMap, expected: DurableRecoveryIdentity, +): { fingerprint: string; runnerVersion: string; runnerDigest: string } | null { + try { + if (records.size !== 4 || [...records.values()].reduce((bytes, value) => bytes + Buffer.byteLength(value), 0) > 8 * 1024 * 1024) return null; + const { state, runnerVersion, runnerDigest } = checkpointState(records); + if (!isUnclaimedBootstrapState(state, expected)) return null; + return { runnerVersion, runnerDigest, fingerprint: `sha256:${createHash("sha256") + .update(canonicalJson([...records].sort(([left], [right]) => left.localeCompare(right)))).digest("hex")}` }; + } catch { return null; } +} + +export function inspectUnclaimedBootstrapState( + contents: string, expected: DurableRecoveryIdentity, verifiedArtifact: { version: string; digest: string }, +): ReturnType { + try { + if (Buffer.byteLength(contents) > 8 * 1024 * 1024) return null; + const state: unknown = JSON.parse(contents); + if (!isStoredCoreState(state, expected) || !isUnclaimedBootstrapState(state, expected)) return null; + return inspectUnclaimedBootstrapCheckpoint(captureLegacyDurablePrpCheckpoint(contents, expected, verifiedArtifact).records, expected); + } catch { return null; } +} + export interface RunnerProcessResult { code: number | null; signal: NodeJS.Signals | null; @@ -762,7 +811,7 @@ function warmTransitionReceipt( function validStoredWarmTransition(state: StoredCoreState): boolean { const transition = state.warmTransition; - if (!transition) return state.schema === coreStateSchema || (state.schema === stagedCoreStateSchema && validStagedRunAttachment(state)); + if (!transition) return state.schema === coreStateSchema || state.schema === legacyCoreStateSchema || (state.schema === stagedCoreStateSchema && validStagedRunAttachment(state)); if ( (state.schema !== transitionCoreStateSchema && state.schema !== stagedCoreStateSchema) || !["awaiting_result", "prepared", "activated"].includes(transition.phase) || @@ -979,7 +1028,7 @@ function isStoredCoreState( const commands = value.commands; const events = value.committedEvents; if ( - (value.schema !== coreStateSchema && + (value.schema !== coreStateSchema && value.schema !== legacyCoreStateSchema && value.schema !== transitionCoreStateSchema && value.schema !== stagedCoreStateSchema) || canonicalJson(value.identity) !== canonicalJson(identity) || (value.checkpointEventPolicy !== undefined && value.checkpointEventPolicy !== "control-v1") || @@ -998,6 +1047,16 @@ function isStoredCoreState( return false; } if (!validStagedRunAttachment(value as unknown as StoredCoreState)) return false; + if (value.bootstrapHistory !== undefined) { + const history = value.bootstrapHistory; + if (!isRecord(history)) return false; + const origin = history.originIdentity; + if (history.schema !== "paperclip.runner.bootstrap-history.v1" || !isRecord(origin) + || !["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"] + .every(key => typeof origin[key] === "string" && stableIdPattern.test(origin[key])) + || !Number.isSafeInteger(history.credentialsIssued) || (history.credentialsIssued as number) < 0 + || typeof history.adopted !== "boolean") return false; + } if (!validStoredWarmTransition(value as unknown as StoredCoreState)) return false; if (value.completedWarmTransition !== undefined) { @@ -1429,6 +1488,7 @@ function initialCoreState(identity: DurableRecoveryIdentity): StoredCoreState { return { schema: coreStateSchema, identity, + bootstrapHistory: { schema: "paperclip.runner.bootstrap-history.v1", originIdentity: structuredClone(identity), credentialsIssued: 0, adopted: false }, runAttachTemplate: null, tickets: {}, leases: {}, @@ -2415,6 +2475,8 @@ export class DurablePrpControlPlane { ]), ); Object.assign(this.#store.state, initialCoreState(identity), { + bootstrapHistory: this.#store.state.bootstrapHistory + ? { ...structuredClone(this.#store.state.bootstrapHistory), adopted: true } : undefined, leases, runAttachTemplate: runAttachTemplate === undefined @@ -2471,6 +2533,14 @@ export class DurablePrpControlPlane { this.#store.save(); } + unclaimedBootstrapCheckpoint() { + if (!isUnclaimedBootstrapState(this.#store.state, this.#identity)) return null; + return inspectUnclaimedBootstrapCheckpoint( + captureCheckpoint(this.#store.state, this.#expectedRunnerVersion, this.#expectedRunnerDigest).records, + this.#identity, + ); + } + issueBootstrapTicket(ttlMs = 5_000): string { this.#store.assertWritable(); if (this.#store.state.warmTransition) { @@ -2497,6 +2567,7 @@ export class DurablePrpControlPlane { usedAt: null, }; this.#store.state.freshBootstraps += 1; + if (this.#store.state.bootstrapHistory) this.#store.state.bootstrapHistory.credentialsIssued += 1; this.#store.save(); return ticket; } @@ -2571,6 +2642,10 @@ export class DurablePrpControlPlane { warmTransitionId: transition.receipt.transitionId, }; candidate.freshBootstraps += 1; + if (candidate.bootstrapHistory) { + candidate.bootstrapHistory.credentialsIssued += 1; + candidate.bootstrapHistory.adopted = true; + } this.#store.commit(candidate); return ticket; } @@ -3455,6 +3530,8 @@ export class DurablePrpControlPlane { } if (transition.phase === "prepared") { const candidate = initialCoreState(transition.receipt.newIdentity); + candidate.bootstrapHistory = this.#store.state.bootstrapHistory + ? { ...structuredClone(this.#store.state.bootstrapHistory), adopted: true } : undefined; candidate.stagedRunAttachment = structuredClone(this.#store.state.stagedRunAttachment); candidate.preparedLaunchConfigurations = structuredClone(this.#store.state.preparedLaunchConfigurations); candidate.schema = candidate.stagedRunAttachment ? stagedCoreStateSchema : transitionCoreStateSchema; diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 30e064085a0..c6c5de0ddea 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -48,6 +48,8 @@ export { decodeDurablePrpCheckpoint, captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, + inspectUnclaimedBootstrapCheckpoint, + inspectUnclaimedBootstrapState, type PrpWireConnection, type PrpWireAttachment, type TransportCloseReason, diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts index f45407609d5..2a0ff6779d7 100644 --- a/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts @@ -33,6 +33,7 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) disconnectActiveRunner() {} persistRunAttachTemplate() {} persistPreparedLaunchConfiguration() {} + unclaimedBootstrapCheckpoint() { return null; } issueBootstrapTicket() { return {}; } async flushCheckpoint() {} async recoverPendingSemanticInputs() {} diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts index bdab53a9376..4f38d7edf40 100644 --- a/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts @@ -31,6 +31,7 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) disconnectActiveRunner() {} persistRunAttachTemplate() {} persistPreparedLaunchConfiguration() {} + unclaimedBootstrapCheckpoint() { return null; } issueBootstrapTicket() { return {}; } async flushCheckpoint() {} async recoverPendingSemanticInputs() {} diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d2de7e622be..82e23dfa1ae 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -2727,11 +2727,13 @@ async function settleRetainedRunnerdSessionOwned( input.recordEpoch({ ...epochIdentity, phase: "launch_intent" }), ); await authorize(); + const bootstrapTicket = core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS); + await core.flushCheckpoint(); handle = spawnRunner({ connectUrl: core.connectUrl, stateDirectory: resolve(root, "runner"), identity: input.identity, - ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), + ticket: bootstrapTicket, maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 30_000, @@ -5129,6 +5131,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { mkdirSync(codexHome, { recursive: true, mode: 0o700 }); } if (provider === "codex" && this.options.homeMode !== "preconfigured") { + if (this.#core?.unclaimedBootstrapCheckpoint()) { + await releaseMaterializedNativeRuntimeSkills(resolve(localCodexHome, "skills")); + } await prepareIsolatedCodexHome({ context: sourceRuntimeContext, codexHome: localCodexHome, @@ -5346,6 +5351,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { mode: 0o700, }); } + const unclaimedBootstrap = core.unclaimedBootstrapCheckpoint(); const prepared = await this.#prepareLaunch(params, identity, dynamicTools); const { runAttachTemplate } = prepared; // Preserve the first generation's provider attachment seed independently @@ -5353,8 +5359,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // continuation; the control-plane copy serves a controller/runner resume. this.#runAttachTemplate = structuredClone(runAttachTemplate); core.persistRunAttachTemplate(runAttachTemplate); - core.queueCommand("run.prepare", runAttachTemplate); - core.queueCommand("session.open", { reuse: "same_session" }); + if (!unclaimedBootstrap) { + core.queueCommand("run.prepare", runAttachTemplate); + core.queueCommand("session.open", { reuse: "same_session" }); + } const registration = this.options.controlPlaneRegistration ? await this.options.controlPlaneRegistration(core) : null; diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index 84dfc42386a..c4afd420162 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -12,6 +12,7 @@ export * from "./conformance/harness-driver.js"; export { startOpenCodeMcpBridge } from "./drivers/opencode/mcp-bridge.js"; export { prepareIsolatedCodexHome, releaseMaterializedNativeRuntimeSkills } from "./drivers/runtime-context-materializer.js"; export { resolveRunnerdCodexSkillInputs } from "./live/runnerd-codex-transport.js"; +export { spawnRunner } from "./control-plane/durable-prp-control-plane.js"; export * from "./conformance/semantic-conformance.js"; export * from "./mock-core/deterministic-harness-driver.js"; export * from "./mock-core/mock-control-plane-adapter.js"; diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index c70ed11422b..db72ee3b15e 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -46,11 +46,18 @@ cd packages/plugins/sandbox-providers/daytona pnpm install --ignore-workspace --no-lockfile pnpm build pnpm test +pnpm exec vitest run --config vitest.sdk-ingress.config.ts pnpm typecheck ``` These commands assume the repo root has already been installed once so the local `@paperclipai/plugin-sdk` workspace package is available to the compiler during development. +The dedicated `vitest.sdk-ingress.config.ts` requires this package's installed, pinned Daytona SDK. It sends +local HTTP faults through the real SDK and worker RPC without a provider account +or sandbox. The root workspace does not install standalone provider dependencies; +its normal Daytona tests use mocked SDK boundaries. This target fails if the SDK +is missing and runs all ingress acceptance cases without a conditional skip. + The capacity regression uses Python 3 for a real local PTY, Bash, and GNU core utilities (`gtar`, `gdd`, and `gbase64` on macOS). It skips when those prerequisites are unavailable or their version checks do not match. With `DAYTONA_API_KEY` set, run diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 8938696d932..2761032d6ee 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -9,7 +9,7 @@ import type { Resources, Sandbox, } from "@daytonaio/sdk"; -import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError, withEnvironmentSyncTransferStep, preserveEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk"; +import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, PluginEnvironmentRunnerIngressError, readEnvironmentCreationCleanupError, withEnvironmentSyncTransferStep, preserveEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk"; import type { PluginContext, PluginEnvironmentCreationCleanup, @@ -30,6 +30,7 @@ import type { PluginEnvironmentExecuteResult, PluginEnvironmentRunnerIngressEndpointParams, PluginEnvironmentRunnerIngressEndpoint, + PluginEnvironmentRunnerIngressFailure, PluginEnvironmentGetInteractiveSetupParams, PluginEnvironmentInteractiveSetupSession, PluginEnvironmentLease, @@ -444,6 +445,17 @@ function creationHttpStatus(error: unknown): number | undefined { } catch { return undefined; } } +async function acquireRunnerIngress(phase: PluginEnvironmentRunnerIngressFailure["phase"], operation: () => Promise): Promise { + try { return await operation(); } + catch (error) { + const httpStatus = creationHttpStatus(error); + if (httpStatus !== undefined && httpStatus >= 400) { + throw new PluginEnvironmentRunnerIngressError({ phase, httpStatus }); + } + throw error; + } +} + function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } @@ -3504,21 +3516,21 @@ const plugin = definePlugin(trackDaytonaHooks({ config, }, async () => { - const sandbox = await getSandbox({ + const sandbox = await acquireRunnerIngress("sandbox_lookup", () => getSandbox({ driverKey: params.driverKey, companyId: params.companyId, environmentId: params.environmentId, providerLeaseId, config, - }); - await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); - await withLivenessTimeout( + })); + await acquireRunnerIngress("sandbox_start", () => ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs))); + await acquireRunnerIngress("sandbox_refresh", () => withLivenessTimeout( "sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData(), - ); - const preview = await sandbox.getPreviewLink(params.port); - if (typeof preview.url !== "string" || typeof preview.token !== "string") { + )); + const preview = await acquireRunnerIngress("private_preview", () => sandbox.getPreviewLink(params.port)); + if (!isRecord(preview) || typeof preview.url !== "string" || typeof preview.token !== "string") { throw new Error("Daytona returned an incomplete private preview endpoint."); } const url = new URL(preview.url); diff --git a/packages/plugins/sandbox-providers/daytona/src/runner-ingress.sdk-integration.ts b/packages/plugins/sandbox-providers/daytona/src/runner-ingress.sdk-integration.ts new file mode 100644 index 00000000000..c4de8ccb36f --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/runner-ingress.sdk-integration.ts @@ -0,0 +1,77 @@ +import { createServer } from "node:http"; +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { describe, expect, it, vi } from "vitest"; +import { createRequest, definePlugin, isJsonRpcResponse, JsonRpcCallError, parseMessage, + readEnvironmentRunnerIngressFailure, serializeMessage, startWorkerRpcHost, type JsonRpcResponse } from "@paperclipai/plugin-sdk"; +import plugin, { __resetDaytonaSandboxHandleCacheForTest } from "./plugin.js"; + +describe("Daytona ingress SDK failure over worker RPC", () => { + it("keeps a missing API credential as a configuration failure", async () => { + vi.stubEnv("DAYTONA_API_KEY", ""); + try { + const error = await plugin.definition.onEnvironmentRunnerIngressEndpoint!({ + companyId: "fixture-company", environmentId: "fixture-environment", driverKey: "daytona", + runId: "fixture-run", port: 43127, path: "/api/runner/v1/connect/fixture-run", + lease: { providerLeaseId: "fixture-sandbox" }, config: {}, + } as never).catch(error => error); + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain("require an API key"); + expect(readEnvironmentRunnerIngressFailure(error)).toBeNull(); + } finally { vi.unstubAllEnvs(); __resetDaytonaSandboxHandleCacheForTest(); } + }); + it.each([ + { status: 502, phase: "sandbox_lookup" }, { status: 401, phase: "sandbox_lookup" }, + { status: 403, phase: "sandbox_lookup" }, { status: 400, phase: "sandbox_lookup" }, + { status: 502, phase: "private_preview" }, + ])("retains the pinned SDK's HTTP $status at $phase without provider response data", async ({ status, phase }) => { + let requests = 0; + let apiUrl = ""; + const server = createServer((request, response) => { + requests++; + if (phase === "private_preview" && !request.url?.includes("43127")) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ id: "fixture-sandbox", name: "fixture-sandbox", organizationId: "fixture-org", + state: "started", toolboxProxyUrl: apiUrl, labels: {}, autoStopInterval: 0 })); + return; + } + response.writeHead(status, { "content-type": "text/html", "x-private-token": "fixture-private-header" }); + response.end(`${status} Bad Gatewayfixture-private-url fixture-private-token`); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + apiUrl = `http://127.0.0.1:${address.port}`; + const stdin = new PassThrough(), stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({ async setup() {}, + onEnvironmentRunnerIngressEndpoint: plugin.definition.onEnvironmentRunnerIngressEndpoint, + }) }); + try { + const response = new Promise(resolve => lines.on("line", line => { + const message = parseMessage(line); + if (isJsonRpcResponse(message)) resolve(message); + })); + stdin.write(serializeMessage(createRequest("environmentRunnerIngressEndpoint", { + companyId: "fixture-company", environmentId: "fixture-environment", driverKey: "daytona", + runId: "fixture-run", port: 43127, path: "/api/runner/v1/connect/fixture-run", + lease: { providerLeaseId: "fixture-sandbox" }, + config: { apiKey: "fixture-private-key", apiUrl, timeoutMs: 2000 }, + }, "ingress"))); + const result = await response; + expect(requests).toBeGreaterThan(0); + if (!("error" in result) || !result.error) throw new Error("Expected SDK rejection"); + expect(readEnvironmentRunnerIngressFailure(new JsonRpcCallError(result.error))) + .toEqual({ phase, httpStatus: status }); + expect(result.error.message).toContain(String(status)); + expect(JSON.stringify(result)).not.toContain("fixture-private"); + expect(result.error.data).toEqual({ schema: "paperclip/environment-runner-ingress-error/v1", + failure: { phase, httpStatus: status } }); + } finally { + worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + __resetDaytonaSandboxHandleCacheForTest(); + } + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/vitest.sdk-ingress.config.ts b/packages/plugins/sandbox-providers/daytona/vitest.sdk-ingress.config.ts new file mode 100644 index 00000000000..020b47c6e14 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/vitest.sdk-ingress.config.ts @@ -0,0 +1,7 @@ +import path from "node:path"; +import { defineConfig } from "vitest/config"; +import config from "./vitest.config.js"; + +export default defineConfig({ ...config, + test: { ...config.test, include: [path.join(import.meta.dirname, "src/runner-ingress.sdk-integration.ts")] }, +}); diff --git a/packages/plugins/sdk/README.md b/packages/plugins/sdk/README.md index 18d521e793c..415a7bfbf98 100644 --- a/packages/plugins/sdk/README.md +++ b/packages/plugins/sdk/README.md @@ -52,6 +52,18 @@ The SDK is stable enough for local development and first-party examples, but the If you are authoring a plugin for others to deploy, treat npm-packaged installation as the supported path and treat repo-local example installs as a development convenience. +## Runner ingress failures + +Environment providers can throw `PluginEnvironmentRunnerIngressError` from +`onEnvironmentRunnerIngressEndpoint` with a validated SDK HTTP status and one +closed phase: `sandbox_lookup`, `sandbox_start`, `sandbox_refresh` or +`private_preview`. The worker forwards only this closed status/phase envelope. +It omits response bodies, URLs, headers and causes. The host independently +validates it with `readEnvironmentRunnerIngressFailure`. Do not derive a status +from error text. Unknown and configuration failures keep their normal errors. +The envelope alone grants no retry, accounting or cleanup authority; native +execution also requires its complete, current controller checkpoint proof. + ## Worker quick start ```ts diff --git a/packages/plugins/sdk/src/environment-runner-ingress-error.ts b/packages/plugins/sdk/src/environment-runner-ingress-error.ts new file mode 100644 index 00000000000..fdb47797c2a --- /dev/null +++ b/packages/plugins/sdk/src/environment-runner-ingress-error.ts @@ -0,0 +1,42 @@ +const SCHEMA = "paperclip/environment-runner-ingress-error/v1"; +const PHASES = ["sandbox_lookup", "sandbox_start", "sandbox_refresh", "private_preview"] as const; + +export interface PluginEnvironmentRunnerIngressFailure { + phase: typeof PHASES[number]; + httpStatus: number; +} + +function field(value: unknown, name: string): unknown { + if (!value || typeof value !== "object") return undefined; + try { return Object.getOwnPropertyDescriptor(value, name)?.value; } + catch { return undefined; } +} + +function validate(value: unknown): PluginEnvironmentRunnerIngressFailure | null { + const phase = field(value, "phase"), httpStatus = field(value, "httpStatus"); + if (!PHASES.includes(phase as PluginEnvironmentRunnerIngressFailure["phase"]) + || typeof httpStatus !== "number" || !Number.isInteger(httpStatus) || httpStatus < 400 || httpStatus > 599) return null; + return { phase: phase as PluginEnvironmentRunnerIngressFailure["phase"], httpStatus }; +} + +export class PluginEnvironmentRunnerIngressError extends Error { + readonly failure: PluginEnvironmentRunnerIngressFailure | null; + + constructor(failure: PluginEnvironmentRunnerIngressFailure) { + const safe = validate(failure); + super(`Runner connection endpoint could not be acquired${safe ? ` (HTTP ${safe.httpStatus})` : ""}.`); + this.name = "PluginEnvironmentRunnerIngressError"; + this.failure = safe; + } +} + +export function environmentRunnerIngressErrorData(error: unknown): unknown { + if (!(error instanceof PluginEnvironmentRunnerIngressError)) return undefined; + const failure = validate(field(error, "failure")); + return failure ? { schema: SCHEMA, failure } : undefined; +} + +export function readEnvironmentRunnerIngressFailure(error: unknown): PluginEnvironmentRunnerIngressFailure | null { + const data = field(error, "data"); + return field(data, "schema") === SCHEMA ? validate(field(data, "failure")) : null; +} diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index d70d8988d02..8a6c4e97074 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -462,6 +462,8 @@ export { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorD export type { PluginEnvironmentCreationCleanup, PluginEnvironmentAcquisitionDiagnostic, PluginEnvironmentAcquisitionProviderState } from "./environment-creation-cleanup.js"; export { preserveEnvironmentSyncErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, withEnvironmentSyncTransferStep, recordEnvironmentSyncError } from "./environment-sync-error.js"; export type { PluginEnvironmentSyncErrorDiagnostic } from "./environment-sync-error.js"; +export { PluginEnvironmentRunnerIngressError, environmentRunnerIngressErrorData, readEnvironmentRunnerIngressFailure } from "./environment-runner-ingress-error.js"; +export type { PluginEnvironmentRunnerIngressFailure } from "./environment-runner-ingress-error.js"; export type { AiConnectionPool, AiConnectionPoolConfig, AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, AiConnectionRouterSelection } from "@paperclipai/shared"; diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index b7a907298cf..53ca756118c 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -6,6 +6,7 @@ import type { AgentLifecycleRequest, AgentLifecycleResult } from "@paperclipai/s import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared"; import { environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; import { environmentSyncErrorData, withEnvironmentSyncErrorCapture } from "./environment-sync-error.js"; +import { environmentRunnerIngressErrorData } from "./environment-runner-ingress-error.js"; import { createInvocationTracer } from "./invocation-tracer.js"; import { createPluginIdleDrain } from "./idle-drain.js"; /** @@ -1644,7 +1645,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost sendMessage(createErrorResponse(id, errorCode, errorMessage, method === "environmentAcquireLease" || method === "environmentDestroyLease" ? environmentCreationCleanupErrorData(err, method === "environmentAcquireLease") - : method === "environmentSyncOut" ? environmentSyncErrorData(err) : undefined)); + : method === "environmentSyncOut" ? environmentSyncErrorData(err) + : method === "environmentRunnerIngressEndpoint" ? environmentRunnerIngressErrorData(err) : undefined)); } finally { done?.(); } diff --git a/packages/plugins/sdk/tests/environment-runner-ingress-error.test.ts b/packages/plugins/sdk/tests/environment-runner-ingress-error.test.ts new file mode 100644 index 00000000000..2d6ee37387f --- /dev/null +++ b/packages/plugins/sdk/tests/environment-runner-ingress-error.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { environmentRunnerIngressErrorData, PluginEnvironmentRunnerIngressError, readEnvironmentRunnerIngressFailure } from "../src/environment-runner-ingress-error.js"; + +describe("runner ingress failure contract", () => { + const failure = { phase: "private_preview", httpStatus: 502 } as const; + it("serializes only producer-owned closed data", () => { + const extended = { ...failure, token: "private-token", url: "private-url" }; + const error = new PluginEnvironmentRunnerIngressError(extended); + const data = environmentRunnerIngressErrorData(error); + expect(readEnvironmentRunnerIngressFailure({ data })).toEqual(failure); + expect(JSON.stringify(data)).not.toContain("private-"); + expect(JSON.stringify(error)).not.toContain("private-"); + expect(error.message).toBe("Runner connection endpoint could not be acquired (HTTP 502)."); + expect(error.cause).toBeUndefined(); + expect(environmentRunnerIngressErrorData(Object.assign(new Error("forged"), { failure, data }))).toBeUndefined(); + }); + it.each([{}, null, { ...failure, phase: "unknown" }, { ...failure, httpStatus: "502" }, + { ...failure, httpStatus: 200 }, { ...failure, httpStatus: 600 }, { ...failure, httpStatus: 502.5 }])("rejects unsupported evidence %j", value => { + expect(readEnvironmentRunnerIngressFailure({ data: { schema: "paperclip/environment-runner-ingress-error/v1", failure: value } })).toBeNull(); + }); + it("does not invoke inherited or accessor fields", () => { + let reads = 0; + const getter = Object.defineProperty({}, "data", { get() { reads++; throw new Error("private"); } }); + expect(readEnvironmentRunnerIngressFailure(getter)).toBeNull(); + expect(readEnvironmentRunnerIngressFailure(Object.create({ data: environmentRunnerIngressErrorData( + new PluginEnvironmentRunnerIngressError(failure)) }))).toBeNull(); + expect(reads).toBe(0); + }); +}); diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index bce13ef5285..0d42c06cb98 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -320,6 +320,7 @@ test("the real serialized shard partition is complete, non-overlapping and durat const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts"; const nativeRunnerSuitePath = "server/src/services/native-runtime/native-codex-runner.integration.test.ts"; +const ingressRetrySuitePath = "server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts"; const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts"; const dotRunnerSuitePath = "server/src/__tests__/dot-runner.test.ts"; const restartRecoverySuitePath = @@ -364,10 +365,11 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites); assert.ok(!files.includes(chatSuitePath)); assert.ok(!files.includes(nativeRunnerSuitePath)); + assert.ok(!files.includes(ingressRetrySuitePath)); assert.ok(!files.includes(dotRunnerSuitePath)); assert.ok(!files.includes(restartRecoverySuitePath)); assert.ok(!files.includes(usageDeliverySuitePath)); - assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, ...restartWaitSuites, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); + assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, ingressRetrySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, ...restartWaitSuites, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); assert.equal(new Set(files).size, files.length); const defaultRun = dryRunJson([], prEnv); assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); @@ -414,6 +416,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => { assert.deepEqual(lane.selectedGeneralServerSuites, [ "server/src/services/native-runtime/plugin-runner-probe.test.ts", "server/src/services/native-runtime/native-codex-runner.integration.test.ts", + ingressRetrySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, ...restartWaitSuites, @@ -657,7 +660,7 @@ test("default chat isolation executes every partition sequentially before other assert.ok(server.args.includes("@paperclipai/server")); const excluded = server.args.flatMap((arg, index) => arg === "--exclude" ? [server.args[index + 1]] : []); assert.equal(excluded.filter(file => file === chatSuitePath.replace(/^server\//, "")).length, 1); - for (const file of [nativeRunnerSuitePath, usageDeliverySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, + for (const file of [nativeRunnerSuitePath, ingressRetrySuitePath, usageDeliverySuitePath, dotRunnerSuitePath, restartRecoverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"]) { assert.ok(!excluded.includes(file.replace(/^server\//, "")), `default must retain ${file}`); } diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 579037b06ae..fe5fdb4c22a 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -94,6 +94,7 @@ const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts"; const nativeRunnerSuites = [ "server/src/services/native-runtime/plugin-runner-probe.test.ts", "server/src/services/native-runtime/native-codex-runner.integration.test.ts", + "server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts", "server/src/__tests__/dot-runner.test.ts", // These process cases otherwise skip in server shards without Runner binaries. "server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts", diff --git a/server/src/redaction.ts b/server/src/redaction.ts index dd3edf95680..108f6fab277 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -143,6 +143,7 @@ export const PAPERCLIP_PUBLIC_SCHEMA_IDS = new Set([ "paperclip.runner.conformance.fixture.v1", "paperclip.runner.conformance.output.v1", "paperclip.runner.durable.control-plane-state.v1", + "paperclip.runner.durable.control-plane-state.v2", "paperclip.runner.durable.diagnostics.v1", "paperclip.runner.durable.state.v1", "paperclip.runner.durable.trace.v1", diff --git a/server/src/services/native-runtime/native-bootstrap-retry.integration.test.ts b/server/src/services/native-runtime/native-bootstrap-retry.integration.test.ts new file mode 100644 index 00000000000..f1912bea58b --- /dev/null +++ b/server/src/services/native-runtime/native-bootstrap-retry.integration.test.ts @@ -0,0 +1,124 @@ +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { agents, completionContracts, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, + nativeRunnerCheckpoints, nativeRunResults } from "@paperclipai/db"; +import { DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { loadNativeRunnerCheckpoint, nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; + +describeEmbeddedPostgres("unclaimed native bootstrap retry admission", () => { + const environment = nativeRestartFixture(); + it.each([ + "current proof", "stale claim", "expired claim", "Stop", "paused agent", "changed model", "changed trust", + "changed account", "provider progress", "issued and pruned credential", "adopted authority", "legacy state", + "truncated checkpoint", "local fingerprint mismatch", "local streamed progress", "local missing event policy", + "hidden delivery", "unknown command", "foreign company", + ])("admits only a current complete proof: %s", async scenario => { + const { db, runId, issueId, native } = await environment.seedRun(`ADMISSION-${scenario}`); + const { companyId, agentId, runtimeRoot } = environment.current; + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const execution = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, + runtimeContext: nativeRuntimeContextFixture(), + binding: { companyId, agentId, issueId, runId, executionWorkspaceId: runId }, + provider: { kind: "codex", model: "fixture-model", approvalPolicy: "never" }, + task: { identifier: "fixture", title: issue!.title, description: null, prompt: "Controlled admission fixture.", workMode: "standard" }, + workspace: { cwd: runtimeRoot, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract!.id, sha256: contract!.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue!, { revision: contract!.revision }) }, interactionResponses: [], credentialBindings: [], + }); + await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) + || ${JSON.stringify({ nativeExecutionInput: execution })}::jsonb` }).where(eq(heartbeatRuns.id, runId)); + const leaseOwner = `fixture:${randomUUID()}`, bootId = randomUUID(); + await db.update(nativeRunFinalizations).set({ leaseOwner, controllerBootId: bootId, controllerGeneration: 7, + leaseExpiresAt: new Date(Date.now() + 60_000), attempt: 1 }).where(eq(nativeRunFinalizations.runId, runId)); + const identity = { runnerInstanceId: native.runnerInstanceId, environmentLeaseId: runId, runId, + normalizedSessionId: native.normalizedSessionId, turnId: `turn-${runId}`, itemId: `item-${runId}` }; + const root = join(runtimeRoot, runId, "control-plane"); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion: "0.3.0", + expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const template = { provider: { provider: "codex", command: "fixture" }, authorizedTools: {} }; + core.persistRunAttachTemplate(template); + core.queueCommand("run.prepare", template); + core.queueCommand("session.open", { reuse: "same_session" }); + try { + if (scenario === "issued and pruned credential") { core.issueBootstrapTicket(); core.store.state.tickets = {}; } + if (scenario === "adopted authority") core.store.state.bootstrapHistory!.adopted = true; + if (scenario === "legacy state") { + core.store.state.schema = "paperclip.runner.durable.control-plane-state.v1"; + delete core.store.state.bootstrapHistory; + } + if (scenario === "hidden delivery") core.store.state.commandDeliveryCounts = { removedCommand: 1 }; + if (scenario === "unknown command") core.queueCommand("turn.start", { prompt: "Prior work must stay held." }); + core.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, { + companyId, issueId, runId, authorityRunId: runId, bootId, leaseOwner, generation: 7, + })); + await core.flushCheckpoint(); + const projection = join(runtimeRoot, runId, "projection"); + const checkpoint = await loadNativeRunnerCheckpoint(db, execution.binding); + expect(checkpoint).not.toBeNull(); + checkpoint!.restoreSessionDirectory(projection); + let localStateContents = await readFile(join(projection, "control-plane", "control-plane-state.json"), "utf8"); + const original = JSON.parse(localStateContents); + if (scenario === "local streamed progress" || scenario === "local missing event policy") { + expect(await admitUnclaimedNativeBootstrapRetry({ db, execution, leaseOwner, localStateContents })).toBe(true); + if (scenario === "local missing event policy") delete original.checkpointEventPolicy; + else { + const event = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", + sourceInstanceId: identity.runnerInstanceId, sourceEventId: "fixture-local-progress", sourceSeq: 1, runId, + normalizedSessionId: identity.normalizedSessionId, turnId: identity.turnId, itemId: identity.itemId, + eventType: "item.delta", priority: 2, emittedAt: "2026-01-01T00:00:00.000Z", payload: { delta: "Prior local progress" } }; + original.committedEvents = [{ sourceSeq: 1, sourceEventId: event.sourceEventId, eventType: event.eventType, + priority: event.priority, envelope: { kind: "event", payload: event }, deliveryCount: 1, logicalEffectCount: 1 }]; + } + localStateContents = JSON.stringify(original); + } + if (scenario === "local fingerprint mismatch") { + original.commands[1].payload = { reuse: "same_session", changed: true }; + localStateContents = JSON.stringify(original); + } + if (scenario === "truncated checkpoint") await db.delete(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.runId, runId), eq(nativeRunnerCheckpoints.part, "command:2"))); + if (scenario === "provider progress") await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 900, + eventType: "turn.started", stream: "system", level: "info", message: "Prior provider progress", payload: {} }); + if (scenario === "stale claim") await db.update(nativeRunFinalizations).set({ leaseOwner: "successor", + controllerGeneration: 8 }).where(eq(nativeRunFinalizations.runId, runId)); + if (scenario === "expired claim") await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, runId)); + if (scenario === "Stop") await db.update(heartbeatRuns).set({ resultJson: { + nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId, issueId, runId, + scope: "run", dispatchState: "pending", intentAuditId: randomUUID() }, + } }).where(eq(heartbeatRuns.id, runId)); + if (scenario === "paused agent") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + const proposed = structuredClone(execution); + if (scenario === "changed model") proposed.provider.model = "different-model"; + if (scenario === "changed trust" && proposed.provider.kind === "codex") proposed.provider.approvalPolicy = "on-request"; + if (scenario === "changed account") proposed.credentialBindings = [{ bindingId: "different-account", service: "openai", + destination: "environment", expiresAt: null, displayName: null }]; + if (scenario === "foreign company") proposed.binding.companyId = randomUUID(); + const [beforeRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const [beforeClaim] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(await admitUnclaimedNativeBootstrapRetry({ db, execution: proposed, leaseOwner, localStateContents })) + .toBe(scenario === "current proof"); + const [afterRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const [afterClaim] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(afterRun).toEqual(beforeRun); + expect(afterClaim).toEqual(beforeClaim); + expect(afterRun.usageJson?.accountingProviderWorkStarted).not.toBe(false); + expect(afterRun.costAccountedAt).toBeNull(); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual([]); + } finally { + await core.stop(); + if (scenario === "paused agent") await db.update(agents).set({ status: "active" }).where(eq(agents.id, agentId)); + } + }); +}); diff --git a/server/src/services/native-runtime/native-bootstrap-retry.ts b/server/src/services/native-runtime/native-bootstrap-retry.ts new file mode 100644 index 00000000000..5312a6b4aa9 --- /dev/null +++ b/server/src/services/native-runtime/native-bootstrap-retry.ts @@ -0,0 +1,51 @@ +import { and, eq, isNull, sql } from "drizzle-orm"; +import { heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { inspectUnclaimedBootstrapState, parseNativeExecutionInput, + type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { nativeSha256 } from "./canonical.js"; +import { loadNativeUnclaimedBootstrapCheckpoint } from "./native-runner-checkpoint.js"; +import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; +import { isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js"; +import { readNativeSessionResumptionAgent } from "./native-session-resumption-admission.js"; + +export async function admitUnclaimedNativeBootstrapRetry(input: { + db: Db; + execution: NativeExecutionInput; + leaseOwner: string; + localStateContents: string; + preparedExecutionAdmission?: { admissionSha256: string; preparedSha256: string }; +}): Promise { + const { binding } = input.execution; + return input.db.transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`); + const [issue] = await tx.select().from(issues).where(and( + eq(issues.id, binding.issueId), eq(issues.companyId, binding.companyId))).for("update"); + const [coordinator] = await tx.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.runId, binding.runId), eq(nativeRunFinalizations.companyId, binding.companyId), + eq(nativeRunFinalizations.issueId, binding.issueId), eq(nativeRunFinalizations.leaseOwner, input.leaseOwner), + eq(nativeRunFinalizations.phase, "observed"), isNull(nativeRunFinalizations.resultId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, binding.runId), eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId))).for("update"); + if (!issue || !coordinator || !run || coordinator.attempt < 1 || coordinator.attempt >= 3 + || !coordinator.leaseExpiresAt || coordinator.leaseExpiresAt.getTime() <= Date.now() + || run.runtimeMode !== "native" || run.status !== "running" || run.finishedAt !== null + || !run.runnerInstanceId || !run.nativeSessionId || run.nativeSessionId !== input.execution.session.normalizedSessionId + || isNativeRunnerOwnershipHeld(run) || run.resultJson?.nativeCancellation != null + || run.resultJson?.startupCancellation != null + || !isUnusedNativeSessionBootstrap(run, (await nativeSessionProviderEvidence(tx, [run.id])).has(run.id))) return false; + if (!await readNativeSessionResumptionAgent(tx as unknown as Db, { issue, run, coordinator })) return false; + let admitted: NativeExecutionInput; + try { admitted = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput); } + catch { return false; } + const executionSha256 = nativeSha256(input.execution); + if (nativeSha256(admitted) !== (input.preparedExecutionAdmission?.admissionSha256 ?? executionSha256) + || input.preparedExecutionAdmission && executionSha256 !== input.preparedExecutionAdmission.preparedSha256) return false; + const identity = { runnerInstanceId: run.runnerInstanceId, environmentLeaseId: binding.executionWorkspaceId, + runId: binding.runId, normalizedSessionId: run.nativeSessionId, turnId: `turn-${binding.runId}`, itemId: `item-${binding.runId}` }; + const canonical = await loadNativeUnclaimedBootstrapCheckpoint(tx, binding, identity); + if (!canonical) return false; + return inspectUnclaimedBootstrapState(input.localStateContents, identity, + { version: canonical.runnerVersion, digest: canonical.runnerDigest })?.fingerprint === canonical.fingerprint; + }); +} diff --git a/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts b/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts new file mode 100644 index 00000000000..e4240f16db8 --- /dev/null +++ b/server/src/services/native-runtime/native-ingress-bootstrap-retry.integration.test.ts @@ -0,0 +1,281 @@ +import { execFileSync, type ChildProcess } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { copyFile, mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import https, { createServer as createHttpsServer } from "node:https"; +import { createServer as createTcpServer } from "node:net"; +import { join, resolve } from "node:path"; +import { getCACertificates, setDefaultCACertificates } from "node:tls"; +import { eq, sql } from "drizzle-orm"; +import WebSocket, { WebSocketServer, type RawData } from "ws"; +import { beforeAll, expect, it, vi } from "vitest"; +import { completionContracts, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; +import type { RunnerIngressEndpoint } from "@paperclipai/adapter-utils/runner-connectivity"; +import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentRunnerIngressError } from "@paperclipai/plugin-sdk"; +import { canonicalNativeRuntimeContextDigest, createNativeSessionBackend, createRunnerdCodexTransport, parseNativeExecutionInput, + type RunnerProcessHandle } from "../../vendor/paperclip-runner/index.js"; +import { nativeRestartFixture, describeEmbeddedPostgres, stopOwnedProcessGroup } from "../../__tests__/helpers/native-restart.js"; +import { canonicalNativeJson, nativeSha256 } from "./canonical.js"; +import { executePaperclipNativeSession, nativeSessionWorkspaceScope, resolveNativeBilling } from "./native-session-executor.js"; +import { createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js"; +import { claimNativeSessionResumptions, reconcileNativeFinalizations } from "./native-finalization-reconciler.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { releaseMaterializedNativeRuntimeSkills, spawnRunner } from "../../vendor/paperclip-runner/testing.js"; +import { materializeAsset } from "./runtime-context.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { createNativeUsageDelivery } from "./native-usage-delivery.js"; +import { accountRunCost } from "../run-cost-accounting.js"; + +describeEmbeddedPostgres("native ingress bootstrap retry through the real Runner", () => { + const environment = nativeRestartFixture(); + const runnerWorkspace = resolve(import.meta.dirname, "../../../../packages/paperclip-runner/runner"); + const runnerBinary = join(runnerWorkspace, "target/release/paperclip-runnerd"); + const fakeProvider = join(runnerWorkspace, "target/release/fake-codex-app-server"); + + beforeAll(() => { + execFileSync(process.execPath, [resolve(runnerWorkspace, "../scripts/build-verified-provider-entrypoints.mjs")], + { cwd: resolve(runnerWorkspace, ".."), stdio: "pipe", timeout: 30_000 }); + execFileSync("cargo", ["build", "--release", "--locked", "-p", "paperclip-runner-core", + "--bin", "paperclip-runnerd", "--bin", "fake-codex-app-server"], + { cwd: runnerWorkspace, stdio: "ignore", timeout: 600_000 }); + }, 660_000); + + it.each(["codex", "opencode"] as const)("reacquires %s ingress and commits exactly one result on the automatic second attempt", async provider => { + const fixture = await environment.seedRun(`INGRESS-BOOTSTRAP-RETRY-${provider}`); + const { db, runId, issueId, native } = fixture; + const { companyId, agentId, runtimeRoot } = environment.current; + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const sandbox = join(runtimeRoot, `ingress-sandbox-${provider}`); + const stateBase = join(runtimeRoot, `ingress-controller-${provider}`); + const codexHome = join(runtimeRoot, `ingress-codex-home-${provider}`); + const filesystem = join(runtimeRoot, `ingress-filesystem-${provider}`); + await mkdir(sandbox, { recursive: true }); + await mkdir(codexHome, { recursive: true }); + await mkdir(join(filesystem, "codex-home"), { recursive: true }); + await db.insert(executionWorkspaces).values({ id: runId, companyId, sourceIssueId: issueId, + mode: "isolated", strategyType: "directory", name: "Owned retry fixture", cwd: sandbox }); + await db.update(issues).set({ executionWorkspaceId: runId }).where(eq(issues.id, issueId)); + const runtimeContext = nativeRuntimeContextFixture(); + const skillContent = "---\nname: retry-fixture\ndescription: Controlled sealed retry skill.\n---\nKeep the admitted task and configuration.\n"; + runtimeContext.instructions.bundle = await materializeAsset([{ path: "AGENTS.md", mode: 0o444, + content: Buffer.from("Complete only the controlled fixture task.\n") }]); + if (provider === "codex") runtimeContext.skills = [{ key: "company/retry-fixture", runtimeName: "retry-fixture", + versionId: "fixture-v1", bundle: await materializeAsset([{ path: "SKILL.md", mode: 0o444, content: Buffer.from(skillContent) }]) }]; + runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext); + const execution = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext, + binding: { companyId, agentId, issueId, runId, executionWorkspaceId: runId }, + provider: provider === "codex" ? { kind: "codex", model: "fixture-model", approvalPolicy: "never" } + : { kind: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", permissionMode: "allow" }, + task: { identifier: "fixture", title: issue!.title, description: null, + prompt: "Complete the controlled fixture task.", workMode: "standard" }, + workspace: { cwd: sandbox, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract!.id, sha256: contract!.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue!, { revision: contract!.revision }) }, interactionResponses: [], credentialBindings: [], + }); + await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) + || ${JSON.stringify({ nativeExecutionInput: execution })}::jsonb` }).where(eq(heartbeatRuns.id, runId)); + const root = join(stateBase, nativeSha256({ schema: "paperclip.native-session-scope.v2", companyId, agentId, + workspace: nativeSessionWorkspaceScope(execution), provider: { driverKind: execution.session.driverKind, + identity: { kind: provider } }, normalizedSessionId: native.normalizedSessionId })); + const digest = (file: string) => `sha256:${createHash("sha256").update(readFileSync(file)).digest("hex")}`; + const opencode = join(runtimeRoot, "fake-opencode-ingress"); + const providerCalls = join(sandbox, "provider-calls.log"); + if (provider === "opencode") { + const script = resolve(runnerWorkspace, "../test/fixtures/fake-opencode-server.mjs"); + execFileSync("cc", ["-x", "c", "-o", opencode, "-"], { input: + `#include \n#include \n#include \nint main(int argc, char **argv) { FILE *log = fopen(${JSON.stringify(providerCalls)}, "a"); if (!log) return 126; fputs("process-start\\n", log); fclose(log); char **args = calloc(argc + 2, sizeof(char *)); args[0] = ${JSON.stringify(process.execPath)}; args[1] = ${JSON.stringify(script)}; for (int i = 1; i < argc; i++) args[i + 1] = argv[i]; execv(args[0], args); return 127; }` }); + } + const proxy = resolve(runnerWorkspace, "../dist/cli/opencode-app-server-proxy.cjs"); + const identity = { runnerInstanceId: native.runnerInstanceId, environmentLeaseId: runId, runId, + normalizedSessionId: native.normalizedSessionId, turnId: `turn-${runId}`, itemId: `item-${runId}` }; + const assertSealedSkills = async () => { + if (provider !== "codex") return; + const skill = join(root, "codex-home", "skills", "retry-fixture", "SKILL.md"); + expect(await readFile(skill, "utf8")).toBe(skillContent); + expect((await stat(skill)).mode & 0o222).toBe(0); + expect((await stat(join(root, "codex-home", "skills", "retry-fixture"))).mode & 0o222).toBe(0); + }; + const resultFile = join(sandbox, "fixture-result.json"); + await writeFile(resultFile, canonicalNativeJson({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", + summary: "Controlled retry completed.", completionClaim: { contractRevision: execution.completionContract.contract.revision, + objectiveSatisfied: true, criteria: execution.completionContract.contract.criteria.map(criterion => ({ + criterionId: criterion.id, status: "satisfied", evidenceRefs: [] })), remainingWork: [] }, + evidence: [], verification: [], artifacts: [], attentionRequests: [] })); + const key = join(sandbox, "preview.key"), certificate = join(sandbox, "preview.crt"); + execFileSync("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", key, "-out", certificate, + "-days", "1", "-subj", "/CN=localhost", "-addext", "subjectAltName=IP:127.0.0.1,DNS:localhost"], { stdio: "ignore" }); + const previousCertificates = getCACertificates("default"); + const certificates = [...previousCertificates, await readFile(certificate, "utf8")]; + setDefaultCACertificates(certificates); + const previousHttpsAgent = https.globalAgent; + const fixtureHttpsAgent = new https.Agent({ ca: certificates }); + https.globalAgent = fixtureHttpsAgent; + const portProbe = createTcpServer(); + await new Promise(done => portProbe.listen(0, "127.0.0.1", done)); + const address = portProbe.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture port"); + const runnerPort = address.port; + await new Promise(done => portProbe.close(() => done())); + let healthy = false, acquisitions = 0, previewUrl = ""; + const preview = createHttpsServer({ key: await readFile(key), cert: await readFile(certificate) }, (request, response) => { + if (request.url !== "/preview") { response.writeHead(404).end(); return; } + acquisitions++; + response.writeHead(healthy ? 200 : 502, { "content-type": "application/json" }); + response.end(JSON.stringify(healthy ? { url: previewUrl, token: "fixture-preview" } : { message: "fixture upstream unavailable" })); + }); + const sockets = new Set(); + const websocket = new WebSocketServer({ noServer: true }); + preview.on("upgrade", (request, socket, head) => { + if (request.url !== `/api/runner/v1/connect/${runId}` || request.headers["x-fixture-preview-token"] !== "fixture-preview") { + socket.end("HTTP/1.1 403 Forbidden\r\n\r\n"); return; + } + websocket.handleUpgrade(request, socket, head, front => { + const back = new WebSocket(`ws://127.0.0.1:${runnerPort}${request.url}`, { perMessageDeflate: false }); + const pending: Array<{ data: RawData; binary: boolean }> = []; + sockets.add(front); sockets.add(back); + front.on("message", (data, binary) => { + if (back.readyState === WebSocket.OPEN) back.send(data, { binary }); + else pending.push({ data, binary }); + }); + back.on("open", () => { for (const message of pending.splice(0)) back.send(message.data, { binary: message.binary }); }); + back.on("message", (data, binary) => { if (front.readyState === WebSocket.OPEN) front.send(data, { binary }); }); + front.on("close", () => { sockets.delete(front); back.terminate(); }); + back.on("close", () => { sockets.delete(back); front.terminate(); }); + front.on("error", error => { diagnostics.push(`fixture front: ${error.message}`); back.terminate(); }); + back.on("error", error => { diagnostics.push(`fixture back: ${error.message}`); front.terminate(); }); + }); + }); + preview.on("tlsClientError", error => { diagnostics.push(`fixture TLS: ${error.message}`); }); + await new Promise(done => preview.listen(0, "127.0.0.1", done)); + const previewAddress = preview.address(); + if (!previewAddress || typeof previewAddress === "string") throw new Error("Missing preview address"); + previewUrl = `wss://127.0.0.1:${previewAddress.port}/api/runner/v1/connect/${runId}`; + const bundles: ReturnType[] = []; + const handles: RunnerProcessHandle[] = []; + const diagnostics: string[] = []; + let usageDelivery: Awaited> | undefined; + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", stateBase); + const endpoint: () => Promise = async () => { + const response = await fetch(`https://127.0.0.1:${previewAddress.port}/preview`); + if (!response.ok) { + const error = new PluginEnvironmentRunnerIngressError({ phase: "private_preview", httpStatus: response.status }); + throw new JsonRpcCallError({ code: -32000, message: error.message, data: environmentRunnerIngressErrorData(error) }); + } + return { kind: "authenticated_websocket" as const, websocketUrl: previewUrl, + secretHeaders: [{ name: "X-Fixture-Preview-Token", value: "fixture-preview" }], generation: "fixture", + refresh: endpoint, close: async () => {} }; + }; + const target = { kind: "remote", transport: "sandbox", remoteCwd: sandbox, leaseId: randomUUID(), + effectiveCapabilities: { runnerWebSocketIngress: true }, getRunnerIngressEndpoint: endpoint } as never; + const backend = () => createNativeSessionBackend(execution, { + runnerInstanceId: native.runnerInstanceId, environment: { CODEX_HOME: codexHome, PATH: process.env.PATH }, + codexTransportFactory: () => { + let started!: () => void; + const processStarted = new Promise(done => { started = done; }); + const bundle = createRunnerdCodexTransport({ + provider, runnerBinary, codexCommand: fakeProvider, codexArgs: ["--state-file", join(sandbox, "provider.json"), + "--terminal-result-file", resultFile, "--call-log", providerCalls, "--record-process-start"], + stateDirectory: root, runnerStateDirectory: join(sandbox, "runner"), runnerFilesystemRoot: filesystem, + runtimeContext, runnerRuntimeContext: runtimeContext, closeGraceMs: 1_000, + onDiagnostic: message => { diagnostics.push(message); }, + onAccountingEvent: async event => { await usageDelivery?.capture(event); }, + environment: { CODEX_HOME: codexHome, PATH: process.env.PATH, OPENROUTER_API_KEY: "synthetic-fixture-key" }, sourceCodexHome: codexHome, + ...(provider === "opencode" ? { opencodeCommand: opencode, opencodeCommandSha256: digest(opencode), + opencodeProxyPath: proxy, opencodeProxySha256: digest(proxy), providerNodeCommand: process.execPath, + providerNodeCommandSha256: digest(process.execPath), opencodePermissionMode: "allow" as const } : {}), + prpIdentity: identity, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + controlPlaneRegistration: async authority => { + const controller = await currentNativeControllerIdentity(); + return createNativeRunnerExecutionDriver({ target, binding: execution.binding, + runnerIngressAuthorized: true, + prepareRemote: async () => { await copyFile(join(root, "runtime-context.json"), join(filesystem, "runtime-context.json")); }, + processStarted, + onLog: async (_stream, message) => { diagnostics.push(message); }, + controllerPersistence: { db, bootId: controller.bootId } }).controlPlaneRegistration!(authority); + }, + runnerProcessLauncher: spec => { + const handle = spawnRunner({ ...spec.nativeRunner!, runnerBinaryPath: spec.command, environment: spec.environment, + connection: { mode: "listen", listenAddress: "0.0.0.0", listenPort: runnerPort, + listenPath: `/api/runner/v1/connect/${runId}` } }); + (handle.child as ChildProcess).once("spawn", started); + handles.push(handle); return handle; + }, + }); + bundles.push(bundle); return bundle.transport; + }, + }); + const execute = (leaseOwner?: string) => executePaperclipNativeSession({ db, execution, + runnerInstanceId: native.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: target, + backend: backend(), leaseOwner, turnTimeoutMs: 15_000 }); + try { + await expect(execute()).rejects.toThrow("HTTP 502"); + expect(acquisitions).toBe(1); + expect(handles).toHaveLength(0); + await assertSealedSkills(); + const saved = await loadNativeRunnerCheckpoint(db, execution.binding); + expect(saved?.records.size).toBe(4); + const savedBootstrapIds = [...saved!.records].filter(([part]) => part.startsWith("command:")) + .map(([, value]) => JSON.parse(value).commandId); + expect(savedBootstrapIds).toHaveLength(2); + const [beforeRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(beforeRetry.usageJson?.accountingProviderWorkStarted).not.toBe(false); + expect(beforeRetry.costAccountedAt).toBeNull(); + const [failed] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(failed).toMatchObject({ phase: "retryable_failure", attempt: 1, failureDetail: { recoveryMode: "bootstrap_retry" } }); + const [claim] = await claimNativeSessionResumptions({ db, runnerInstanceId: "fixture-resumption", runIds: [runId], + now: new Date(failed!.nextAttemptAt!.getTime() + 1), deferIneligible: false }); + expect(claim).toBeDefined(); + usageDelivery = await createNativeUsageDelivery(db, { companyId, runId, runnerInstanceId: native.runnerInstanceId, + provider: execution.provider, billing: resolveNativeBilling(execution.provider) }); + healthy = true; + let timer: ReturnType | undefined; + const retry = execute(claim!.leaseOwner); + try { + await expect(Promise.race([retry, new Promise((_done, reject) => { + timer = setTimeout(() => reject(new Error(`Controlled retry deadline exceeded: ${diagnostics.slice(-12).join("\n")}`)), 20_000); + })])).resolves.toMatchObject({ exitCode: 0, timedOut: false }); + } finally { if (timer) clearTimeout(timer); } + expect(acquisitions).toBe(2); + expect(handles).toHaveLength(1); + await assertSealedSkills(); + const completed = await loadNativeRunnerCheckpoint(db, execution.binding); + const commands = [...completed!.records].filter(([part]) => part.startsWith("command:")).map(([, value]) => JSON.parse(value)); + expect(commands.filter(command => command.type === "run.prepare")).toHaveLength(1); + expect(commands.filter(command => command.type === "session.open")).toHaveLength(1); + expect(commands.filter(command => ["run.prepare", "session.open"].includes(command.type)) + .map(command => command.commandId)).toEqual(savedBootstrapIds); + expect((await readFile(providerCalls, "utf8")).split("\n").filter(line => line === "process-start")).toHaveLength(1); + const results = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId)); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ schemaStatus: "accepted" }); + await reconcileNativeFinalizations(db, [runId]); + const [settled] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(settled).toMatchObject({ phase: "committed", attempt: 2 }); + const [completedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(completedRun.usageJson?.accountingProviderWorkStarted).not.toBe(false); + expect(completedRun.usageJson?.accountingReceiptReady).toBe(true); + expect(Number(completedRun.usageJson?.inputTokens)).toBeGreaterThan(0); + expect(await accountRunCost(db, runId)).toBe(true); + expect(await accountRunCost(db, runId)).toBe(false); + const [accountedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(accountedRun.costAccountedAt).toBeInstanceOf(Date); + } finally { + for (const bundle of bundles) await bundle.transport.close().catch(() => undefined); + for (const handle of handles) await stopOwnedProcessGroup(handle.child.pid ?? null, sandbox); + if (provider === "codex") await releaseMaterializedNativeRuntimeSkills(join(root, "codex-home", "skills")); + for (const socket of sockets) socket.terminate(); + websocket.close(); preview.closeAllConnections(); + await new Promise(done => preview.close(() => done())); + https.globalAgent = previousHttpsAgent; + fixtureHttpsAgent.destroy(); + setDefaultCACertificates(previousCertificates); + vi.unstubAllEnvs(); + } + }, 60_000); +}); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 8c3a4a9c3cd..e23de011265 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -1,7 +1,7 @@ import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm"; import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; -import { inspectDurablePrpCheckpointHeader, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js"; +import { inspectDurablePrpCheckpointHeader, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; @@ -226,6 +226,22 @@ export async function loadNativeRunnerCheckpoint(db: Pick, binding return checkpoint; } +export async function loadNativeUnclaimedBootstrapCheckpoint( + db: Pick, binding: Binding, identity: DurableRecoveryIdentity, +) { + try { + if (binding.runId !== identity.runId) return null; + const rows = await db.select().from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), + eq(nativeRunnerCheckpoints.runId, binding.runId))).limit(5); + if (rows.length !== 4 || rows.reduce((bytes, row) => bytes + Buffer.byteLength(JSON.stringify(row.material)), 0) + > 2 * NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES) return null; + const cipher = createLocalEncryptedCipher("read"); + const records = new Map(rows.map(row => [row.part, decryptPart(cipher, binding, row.part, row.material)] as const)); + return inspectUnclaimedBootstrapCheckpoint(records, identity); + } catch { return null; } +} + export async function restoreNativeRunnerCheckpoint(db: Db, binding: Binding, root: string, normalizedSessionId?: string) { const checkpoint = await loadNativeRunnerCheckpoint(db, binding); if (!checkpoint) return null; diff --git a/server/src/services/native-runtime/native-runner-execution-driver.test.ts b/server/src/services/native-runtime/native-runner-execution-driver.test.ts index 51c79577e02..f7579b6e0e0 100644 --- a/server/src/services/native-runtime/native-runner-execution-driver.test.ts +++ b/server/src/services/native-runtime/native-runner-execution-driver.test.ts @@ -1,7 +1,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { RunnerIngressEndpoint } from "@paperclipai/adapter-utils/runner-connectivity"; -import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js"; +import pino from "pino"; +import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentRunnerIngressError } from "@paperclipai/plugin-sdk"; +import { NativeRunnerIngressAcquisitionError, prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js"; const transport = vi.hoisted(() => ({ register: vi.fn(), @@ -38,6 +40,24 @@ beforeEach(() => { }); describe("shared Runner execution driver", () => { + it.each(["ingress", "preparation", "unknown"])("limits ingress evidence and public error serialization at %s", async phase => { + const failure = { phase: "private_preview", httpStatus: 502 } as const; + const raw = new JsonRpcCallError({ code: -32000, message: "fixture-private-provider-response", + data: phase === "unknown" ? undefined : environmentRunnerIngressErrorData(new PluginEnvironmentRunnerIngressError(failure)) }); + const acquire = vi.fn(async () => { throw raw; }); + const driver = createNativeRunnerExecutionDriver({ binding, runnerIngressAuthorized: true, + target: { ...sandbox(), getRunnerIngressEndpoint: acquire } as AdapterExecutionTarget, + prepareRemote: async () => { if (phase === "preparation") throw raw; }, + }); + const error = await driver.controlPlaneRegistration!(authority).catch(error => error); + if (phase === "ingress") { + expect(error).toBeInstanceOf(NativeRunnerIngressAcquisitionError); + expect(error.failure).toEqual(failure); + expect(JSON.stringify(pino.stdSerializers.err(error))).not.toContain("fixture-private"); + } else expect(error).toBe(raw); + expect(transport.connect).not.toHaveBeenCalled(); + expect(transport.register).not.toHaveBeenCalled(); + }); it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => { const prepare = vi.fn(); const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" }; diff --git a/server/src/services/native-runtime/native-runner-execution-driver.ts b/server/src/services/native-runtime/native-runner-execution-driver.ts index 0929d99d82a..89a5352c000 100644 --- a/server/src/services/native-runtime/native-runner-execution-driver.ts +++ b/server/src/services/native-runtime/native-runner-execution-driver.ts @@ -7,6 +7,14 @@ import { connectRunnerPrpIngress } from "../../realtime/runner-prp-outbound.js"; import type { NativeRunTrace } from "./native-run-trace.js"; import type { Db } from "@paperclipai/db"; import { nativeRunnerCheckpointRegistration } from "./native-runner-checkpoint.js"; +import { readEnvironmentRunnerIngressFailure, type PluginEnvironmentRunnerIngressFailure } from "@paperclipai/plugin-sdk"; + +export class NativeRunnerIngressAcquisitionError extends Error { + constructor(readonly failure: PluginEnvironmentRunnerIngressFailure) { + super(`Runner connection endpoint could not be acquired (HTTP ${failure.httpStatus}).`); + this.name = "NativeRunnerIngressAcquisitionError"; + } +} type ConnectionRegistration = NonNullable; type RunnerIdentity = Parameters[1]; @@ -177,21 +185,25 @@ export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDr ); // Provider endpoint acquisition happens only after runnerd is staged // and its listener capability has been verified. - transport = await measureNativeRunnerSpan( - input.trace, - "runner.ingress.acquire", - () => - resolvePaperclipRunnerTransport({ - target: input.target, - runId: - attachmentIdentity?.runId ?? - input.binding.runId, - localConnectUrl: "ws://127.0.0.1/unused", - runnerPublicUrl: input.runnerPublicUrl, - runnerCaBundlePath: input.runnerCaBundlePath, - runnerIngressAuthorized: true, - }), - ); + try { + transport = await measureNativeRunnerSpan( + input.trace, + "runner.ingress.acquire", + () => + resolvePaperclipRunnerTransport({ + target: input.target, + runId: attachmentIdentity?.runId ?? input.binding.runId, + localConnectUrl: "ws://127.0.0.1/unused", + runnerPublicUrl: input.runnerPublicUrl, + runnerCaBundlePath: input.runnerCaBundlePath, + runnerIngressAuthorized: true, + }), + ); + } catch (error) { + const failure = readEnvironmentRunnerIngressFailure(error); + if (failure) throw new NativeRunnerIngressAcquisitionError(failure); + throw error; + } } const selectedAtMs = Date.now(); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 1ce9366b15b..b7ced95d40c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -15335,7 +15335,7 @@ describe("runnerd provider runtime wiring", () => { runnerInstanceId: "runner-next", runnerExecutionTarget: remote as never })) .rejects.toThrow("runner_state_identity_mismatch: prior_owner_active"); const projected = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); - expect(projected).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v1", commands: [] }); + expect(projected).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v2", commands: [] }); expect(projected.runAttachTemplate).toBeNull(); const template = { provider: prior.provider, normalizedSessionId: identity.normalizedSessionId }; controller.persistRunAttachTemplate(template); @@ -15508,7 +15508,7 @@ describe("runnerd provider runtime wiring", () => { if (["suspended", "backup", "codex", "file-projection", "legacy-file-no-checkpoint"].includes(scenario)) { await expect(resume).rejects.toBe(admitted); const restored = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); - expect(restored).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v1", identity }); + expect(restored).toMatchObject({ schema: "paperclip.runner.durable.control-plane-state.v2", identity }); } else { const reason = ["missing-checkpoint", "foreign-task"].includes(scenario) ? "native_runner_checkpoint_missing" : ["checkpoint-binding-error", "file-checkpoint-binding-error"].includes(scenario) ? "native_runner_checkpoint_binding_mismatch" diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index da07565e4a5..77466c6e776 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -31,7 +31,7 @@ import { type ProviderPackInstallation, verifyRemoteProviderPack as verifyRemote import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; export { renewNativeSessionExecutionLease } from "./native-execution-lease.js"; -import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { writeExecutionControl } from "../execution-control-notifications.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { createRunUsageRecorder } from "../usage-receipts.js"; @@ -75,6 +75,7 @@ import { parseCodexCliVersion } from "./codex-runtime-compatibility.js"; import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js"; import { createNativeProviderFailureObservation } from "./native-provider-failure.js"; import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js"; +import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; import { createNativeDeliverableTextObservation } from "./native-deliverable-text.js"; import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js"; import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js"; @@ -122,7 +123,7 @@ import { } from "node:fs"; import { tmpdir, userInfo } from "node:os"; import { DatabaseSync } from "node:sqlite"; -import { loadNativeRunnerCheckpointHeader, nativeRunnerCheckpointPersistence, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { loadNativeRunnerCheckpointHeader, loadNativeUnclaimedBootstrapCheckpoint, nativeRunnerCheckpointPersistence, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; import { REMOTE_CONTROLLER_LEASE_MS, REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runner-recovery-contract.js"; import type { NativeTaskSessionPublicationContext } from "./native-task-session-publication.js"; import { basename, dirname, join, posix, resolve } from "node:path"; @@ -133,6 +134,7 @@ import type { import type { NativeFinalizationResult } from "@paperclipai/shared"; import type { DurablePrpControlPlane, + DurableRecoveryIdentity, HarnessRuntimeRequestResolution, NativeExecutionInput, NativeRuntimeContextSnapshot, @@ -179,7 +181,7 @@ import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; -import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver, resolveRemoteRunnerTransportMode } from "./native-runner-execution-driver.js"; +import { NativeRunnerIngressAcquisitionError, prepareNativeRemoteRunner, createNativeRunnerExecutionDriver, resolveRemoteRunnerTransportMode } from "./native-runner-execution-driver.js"; import type { Db } from "@paperclipai/db"; import { and, @@ -484,8 +486,10 @@ const ACPX_RUNTIME_CHECKPOINT_EXCLUDED_ENTRIES = [ "home/.npm", "data/pnpm/store", ] as const; -const RUNNERD_CONTROL_PLANE_STATE_SCHEMA = - "paperclip.runner.durable.control-plane-state.v1"; +const RUNNERD_CONTROL_PLANE_STATE_SCHEMAS = new Set([ + "paperclip.runner.durable.control-plane-state.v1", + "paperclip.runner.durable.control-plane-state.v2", +]); const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1"; const CODEX_PROVIDER_STATE_SCHEMA = "paperclip.runner.codex-provider-state.v1"; const ACPX_PROVIDER_STATE_SCHEMA = "paperclip.runner.acpx-provider-state.v3"; @@ -2953,7 +2957,7 @@ export async function verifyStoppedNativeSessionForContinuation( !Array.isArray(snapshot.control.committedEvents)) return null; // An incomplete provider launch can own a process that never emitted its // session identity. It cannot be certified from an earlier owner's receipt. - if (snapshot.control.schema !== "paperclip.runner.durable.control-plane-state.v1" || + if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(snapshot.control.schema)) || snapshot.runner.schema !== RUNNERD_STATE_SCHEMA || snapshot.provider.schema !== (execution.provider.kind === "codex" ? "paperclip.runner.codex-provider-state.v1" : ACPX_PROVIDER_STATE_SCHEMA) || !["turn_active", "prepared", "suspended"].includes(String(snapshot.provider.lifecycle)) || @@ -5165,7 +5169,7 @@ function readSettledRunnerdProjection(root: string, identity: Record command.type === "semantic_tool.result" && command.status === "completed") .map(command => record(command.payload).sourceEventId)); - if (control.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA || control.warmTransition != null || + if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(control.schema)) || control.warmTransition != null || control.stagedRunAttachment != null || !isSettledRemoteWarmControlState(control, identity) || (control.committedEvents as unknown[]).map(record).some(event => (event.eventType === "semantic_tool.input" || event.eventType === "mcp_app.tool_input") && @@ -5268,6 +5272,7 @@ async function migrateRunnerdStateRootForExecution(input: { onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; restartRecovery?: NativeRestartRecoveryClaim; runnerExecutionTarget?: AdapterExecutionTarget | null; + bootstrapRetryLeaseOwner?: string; }): Promise { const scoped = scopedRunnerdStateRoot(input.execution); await restoreWarmRunnerdController(input); @@ -5365,6 +5370,13 @@ async function migrateRunnerdStateRootForExecution(input: { return; } if (durableIdentityMatchesExecution(identity, input.execution)) { + if (!input.restartRecovery && !input.retainedWarmOwnerRunId && input.bootstrapRetryLeaseOwner + && input.runnerExecutionTarget?.kind === "remote" + && runnerdAuthorityLifecycle(scoped, identity) === "absent" + && await admitUnclaimedNativeBootstrapRetry({ db: input.db, execution: input.execution, + leaseOwner: input.bootstrapRetryLeaseOwner, preparedExecutionAdmission: input.preparedExecutionAdmission, + localStateContents: readBoundedNativeFile(resolve(scoped, "control-plane", "control-plane-state.json"), + 8 * 1024 * 1024, "native_runner_checkpoint_invalid").toString("utf8") })) return; if ( runnerdAuthorityLifecycleWithVerifiedBackup({ root: scoped, @@ -5755,7 +5767,7 @@ function localProcessDefinitelyGone(value: unknown, group = false): boolean { function runnerdStateIsIncompleteBootstrap(state: Record): boolean { const commands = Array.isArray(state.commands) ? state.commands.map(record) : []; const committedEvents = Array.isArray(state.committedEvents) ? state.committedEvents : []; - return state.schema === RUNNERD_CONTROL_PLANE_STATE_SCHEMA && + return RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema)) && state.connectionCount === 0 && committedEvents.length === 0 && commands.every(command => command.status === "pending" && (command.type === "run.prepare" || command.type === "session.open")); @@ -5820,7 +5832,7 @@ function readRunnerdDurableIdentity( if (!staged) throw new Error("native_runner_staged_attachment_binding_mismatch"); return { ...staged.sourceIdentity }; } - if (state.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA && + if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema)) && !(allowExternal && state.schema === "paperclip.runner.external-control-plane-state.v1")) return null; return record(state.identity); } catch { @@ -7018,6 +7030,7 @@ export function nativeSessionFailureDisposition( sourceFailureCode?: ReturnType, ) { const permanentFailure = + sourceFailureCode === "runner_ingress_access_rejected" || sourceFailureCode === "native_provider_runtime_missing" || sourceFailureCode === "native_provider_model_rejected" || sourceFailureCode === "native_provider_approval_required" || @@ -7101,6 +7114,8 @@ export function nativeSessionFailureSourceCode( | typeof NATIVE_CURRENT_WAKE_COMMENTS_UNREAD | typeof NATIVE_CURRENT_WAKE_COMMENTS_CHANGED_AFTER_READ | "provider_session_open_rejected" + | "runner_ingress_access_rejected" + | "runner_ingress_acquisition_failed" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; @@ -7120,6 +7135,10 @@ export function nativeSessionFailureSourceCode( return "native_event_replay_conflict"; if (error instanceof NativeSessionCleanupQuarantinedError) return "native_session_cleanup_quarantined"; + if (error instanceof NativeRunnerIngressAcquisitionError) { + return error.failure.httpStatus === 401 || error.failure.httpStatus === 403 + ? "runner_ingress_access_rejected" : "runner_ingress_acquisition_failed"; + } const message = error instanceof Error ? error.message : String(error); if (/classification=runtime_package_missing\)/.test(message)) return "native_provider_runtime_missing"; @@ -7217,14 +7236,22 @@ export async function nativeProviderRecoveryEvidence(input: { db: Db; runId: string; sourceFailureCode: ReturnType; + bootstrap?: { + binding: { companyId: string; issueId: string; runId: string }; + identity: DurableRecoveryIdentity; + transientIngressFailure: boolean; + }; }): Promise<{ recoveryMode: NativeRecoveryMode; providerSessionEstablished: boolean; providerEventsExist: boolean; checkpointExists: boolean; + unclaimedBootstrap?: { fingerprint: string }; }> { const run = await input.db - .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson, companyId: heartbeatRuns.companyId, + nativeIssueId: heartbeatRuns.nativeIssueId, nativeSessionId: heartbeatRuns.nativeSessionId, + runnerInstanceId: heartbeatRuns.runnerInstanceId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.runId)) .limit(1) @@ -7251,6 +7278,12 @@ export async function nativeProviderRecoveryEvidence(input: { const providerEventsExist = durableEvents.some((event) => PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType), ); + const unclaimedBootstrap = !checkpointExists && !providerEventsExist && input.bootstrap + && input.bootstrap.binding.runId === input.runId && run?.companyId === input.bootstrap.binding.companyId + && run.nativeIssueId === input.bootstrap.binding.issueId && run.nativeSessionId === input.bootstrap.identity.normalizedSessionId + && run.runnerInstanceId === input.bootstrap.identity.runnerInstanceId + ? await loadNativeUnclaimedBootstrapCheckpoint(input.db, input.bootstrap.binding, input.bootstrap.identity) + : null; if ( checkpointExists && providerSessionEstablished && @@ -7278,12 +7311,14 @@ export async function nativeProviderRecoveryEvidence(input: { "native_runner_process_exited", "provider_session_open_rejected", ]).has(input.sourceFailureCode); - if (!checkpointExists && !providerEventsExist && definitelyPreSession) { + if (!checkpointExists && !providerEventsExist && (definitelyPreSession + || unclaimedBootstrap && input.bootstrap?.transientIngressFailure)) { return { recoveryMode: "bootstrap_retry", providerSessionEstablished: false, providerEventsExist: false, checkpointExists: false, + ...(unclaimedBootstrap ? { unclaimedBootstrap: { fingerprint: unclaimedBootstrap.fingerprint } } : {}), }; } return { @@ -7292,6 +7327,7 @@ export async function nativeProviderRecoveryEvidence(input: { providerSessionEstablished || providerEventsExist, providerEventsExist, checkpointExists, + ...(unclaimedBootstrap ? { unclaimedBootstrap: { fingerprint: unclaimedBootstrap.fingerprint } } : {}), }; } @@ -8715,6 +8751,7 @@ async function executePaperclipNativeSessionWithinScope( onLog: input.onLog, restartRecovery: input.restartRecovery, runnerExecutionTarget: input.runnerExecutionTarget, + bootstrapRetryLeaseOwner: input.leaseOwner, }); } const durableRunnerBinding = input.useRunnerd @@ -10449,10 +10486,22 @@ async function executePaperclipNativeSessionWithinScope( : providerUsageLimitObserved ? "native_provider_usage_limit" : classifiedFailureCode; + const bootstrap = input.useRunnerd && input.runnerExecutionTarget?.kind === "remote" + && !input.restartRecovery && !retainedTransition && !remoteWarmRecovery + ? { + binding: input.execution.binding, + identity: { runnerInstanceId: effectiveRunnerInstanceId, + environmentLeaseId: durableRunnerBinding?.environmentLeaseId ?? input.execution.binding.executionWorkspaceId, + runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), + turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` }, + transientIngressFailure: sourceFailureCode === "runner_ingress_acquisition_failed" && error instanceof NativeRunnerIngressAcquisitionError + && (error.failure.httpStatus === 429 || error.failure.httpStatus >= 500), + } : undefined; const recoveryEvidence = await nativeProviderRecoveryEvidence({ db: input.db, runId: input.execution.binding.runId, sourceFailureCode, + bootstrap, }); const disposition = nativeSessionFailureDisposition( attempt, @@ -10473,6 +10522,9 @@ async function executePaperclipNativeSessionWithinScope( const startupRetryExhausted = recoveryEvidence.recoveryMode === "bootstrap_retry" && failureCode === "native_session_retry_exhausted"; + const startupExhaustedAction = sourceFailureCode === "runner_ingress_acquisition_failed" + ? `Paperclip could not acquire the computer's connection endpoint after ${attempt} attempts. Automatic retries are stopped. Inspect the computer provider's service status before retrying.` + : null; const startupRetry = recoveryEvidence.recoveryMode === "bootstrap_retry" && (phase === "retryable_failure" || startupRetryExhausted) @@ -10541,6 +10593,7 @@ async function executePaperclipNativeSessionWithinScope( message, ...(providerFailure ? { providerFailure } : {}), ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}), + ...(error instanceof NativeRunnerIngressAcquisitionError ? { runnerIngressFailure: error.failure } : {}), originalFailureCode: error instanceof NativeProviderTerminalFailure ? error.providerCode @@ -10565,6 +10618,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "runner_remote_provider_verification_unavailable" ? "Provider files could not be verified after two read attempts. Inspect the retained workspace and command transport before retrying the original run." + : sourceFailureCode === "runner_ingress_access_rejected" + ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE @@ -10578,7 +10633,7 @@ async function executePaperclipNativeSessionWithinScope( : ownershipUnverified ? "Inspect the retained runner's executable and authenticated connection. Do not replace its provider until ownership is safely resolved." : startupRetryExhausted - ? nativeStartupRetryExhaustedAction(attempt, providerCooldownAt) + ? startupExhaustedAction ?? nativeStartupRetryExhaustedAction(attempt, providerCooldownAt) : exhausted ? "Inspect the persisted native session after its bounded resume budget was exhausted." : recoveryEvidence.recoveryMode === @@ -10640,6 +10695,8 @@ async function executePaperclipNativeSessionWithinScope( ), eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), + eq(nativeRunFinalizations.controllerGeneration, controllerGeneration), + isNull(nativeRunFinalizations.resultId), eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid), eq( @@ -10653,10 +10710,27 @@ async function executePaperclipNativeSessionWithinScope( .then((rows) => rows[0] ?? null); if (!updated) throw new Error("native_session_lease_lost"); const [runBeforeWrite] = await tx - .select({ status: heartbeatRuns.status }) + .select({ status: heartbeatRuns.status, resultJson: heartbeatRuns.resultJson, + runnerInstanceId: heartbeatRuns.runnerInstanceId, nativeSessionId: heartbeatRuns.nativeSessionId, + costAccountedAt: heartbeatRuns.costAccountedAt }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.execution.binding.runId)) .for("update"); + if (recoveryEvidence.unclaimedBootstrap && bootstrap) { + const currentEvidence = await nativeProviderRecoveryEvidence({ db: tx as unknown as Db, + runId: input.execution.binding.runId, sourceFailureCode, bootstrap }); + if (!runBeforeWrite || ["cancelled", "succeeded"].includes(runBeforeWrite.status) + || Object.keys(record(runBeforeWrite.resultJson?.nativeCancellation)).length > 0 + || runBeforeWrite.runnerInstanceId !== bootstrap.identity.runnerInstanceId + || runBeforeWrite.nativeSessionId !== bootstrap.identity.normalizedSessionId + || currentEvidence.unclaimedBootstrap?.fingerprint !== recoveryEvidence.unclaimedBootstrap.fingerprint + || currentEvidence.recoveryMode !== recoveryEvidence.recoveryMode) { + throw new Error("native_runner_bootstrap_evidence_changed"); + } + } + const unusedTerminal = !!recoveryEvidence.unclaimedBootstrap && phase === "terminal_failure" + && !runBeforeWrite?.costAccountedAt; + if (unusedTerminal) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.execution.binding.companyId); if (!ownershipUnverified) await notifyExternalAgentSettlement(tx, input.execution.binding); if (!ownershipUnverified) await notifyInstructionCollection(tx, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId }); if (!ownershipUnverified) await notifyCancelledWorkspacePreservation(tx, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId }); @@ -10675,6 +10749,10 @@ async function executePaperclipNativeSessionWithinScope( : {}), nativePhase: phase, nativePhaseUpdatedAt: now, + ...(unusedTerminal ? { + usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`, + costAccountingPending: true, + } : {}), error: message, resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb @@ -10759,6 +10837,8 @@ async function executePaperclipNativeSessionWithinScope( ? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler." : sourceFailureCode === "native_session_cleanup_quarantined" ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE + : sourceFailureCode === "runner_ingress_access_rejected" + ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE @@ -10772,7 +10852,7 @@ async function executePaperclipNativeSessionWithinScope( : ownershipUnverified ? "Resolve the retained runner's authentication or executable compatibility before an explicit recovery; do not blindly restart, cancel, or replace its provider session." : startupRetryExhausted - ? nativeStartupRetryExhaustedAction(attempt, providerCooldownAt) + ? startupExhaustedAction ?? nativeStartupRetryExhaustedAction(attempt, providerCooldownAt) : exhausted ? "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped." : recoveryEvidence.recoveryMode === "bootstrap_retry" diff --git a/server/src/services/native-runtime/native-startup-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-retry.integration.test.ts index e335d4e13a3..6892000df69 100644 --- a/server/src/services/native-runtime/native-startup-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-retry.integration.test.ts @@ -1,5 +1,9 @@ import { randomUUID } from "node:crypto"; -import { eq, sql } from "drizzle-orm"; +import { mkdtempSync, rmSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, @@ -11,6 +15,7 @@ import { issueRecoveryActions, issues, nativeRunFinalizations, + nativeRunnerCheckpoints, } from "@paperclipai/db"; import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -27,6 +32,12 @@ import { executePaperclipNativeSession } from "./native-session-executor.js"; import { buildNativeCompletionContract } from "./completion-contracts.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; +import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js"; +import { createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentRunnerIngressError } from "@paperclipai/plugin-sdk"; +import { accountRunCost } from "../run-cost-accounting.js"; type SessionOpenStatus = "failed" | "rejected" | "indeterminate"; @@ -272,6 +283,150 @@ describe("native provider startup retry", () => { }); } + it.each([ + { name: "transient 502", status: 502, retry: true }, + { name: "credential 401", status: 401, retry: false }, + { name: "permission 403", status: 403, retry: false }, + { name: "invalid request 400", status: 400, retry: false }, + { name: "untyped HTML 502", status: 502, retry: false, raw: true }, + { name: "previously issued credential", status: 502, retry: false, checkpoint: "issued" }, + { name: "legacy checkpoint", status: 502, retry: false, checkpoint: "legacy" }, + { name: "truncated checkpoint", status: 502, retry: false, checkpoint: "truncated" }, + { name: "provider progress", status: 502, retry: false, checkpoint: "progress" }, + { name: "checkpoint change during settlement", status: 502, retry: false, race: "checkpoint" }, + { name: "Stop during settlement", status: 502, retry: false, race: "stop" }, + { name: "new controller claim during settlement", status: 502, retry: false, race: "claim" }, + { name: "exhausted transient 502", status: 502, retry: false, finalAttempt: true }, + ])("settles ingress failure safely: $name", async options => { + const seeded = await seedRun(await createAgent()); + if (options.finalAttempt) await db.update(nativeRunFinalizations).set({ attempt: 2 }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + const root = mkdtempSync(join(tmpdir(), "native-ingress-bootstrap-")); + const server = createServer((_request, response) => { + response.writeHead(options.status, { "content-type": "text/html" }); + response.end(`${options.status} Bad Gateway`); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test server missing address"); + const identity = { + runnerInstanceId: seeded.runnerInstanceId, + environmentLeaseId: seeded.execution.binding.executionWorkspaceId, + runId: seeded.runId, + normalizedSessionId: seeded.execution.session.normalizedSessionId!, + turnId: `turn-${seeded.runId}`, + itemId: `item-${seeded.runId}`, + }; + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const prepare = { observeCredentials: true, authorizedTools: {}, provider: { provider: "codex" } }; + core.persistRunAttachTemplate(prepare); + core.queueCommand("run.prepare", prepare); + core.queueCommand("session.open", { reuse: "same_session" }); + if (options.checkpoint === "issued") { + core.issueBootstrapTicket(); + core.store.state.tickets = {}; + } + if (options.checkpoint === "legacy") { + core.store.state.schema = "paperclip.runner.durable.control-plane-state.v1"; + delete core.store.state.bootstrapHistory; + } + const launch = vi.fn(); + let settlementArmed = false; + let settlementTransactions = 0; + const transaction = db.transaction.bind(db); + const settlementRace = options.race ? vi.spyOn(db, "transaction").mockImplementation(async (...args) => { + if (settlementArmed && ++settlementTransactions === 2) { + settlementArmed = false; + if (options.race === "checkpoint") { core.issueBootstrapTicket(); await core.flushCheckpoint(); } + if (options.race === "stop") await db.update(heartbeatRuns).set({ resultJson: { + nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId, issueId: seeded.issueId, + runId: seeded.runId, scope: "run", intentAuditId: randomUUID(), dispatchState: "pending" }, + } }).where(eq(heartbeatRuns.id, seeded.runId)); + if (options.race === "claim") await db.update(nativeRunFinalizations).set({ + controllerGeneration: sql`${nativeRunFinalizations.controllerGeneration} + 1`, leaseOwner: "successor-controller", + }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + } + return transaction(...args); + }) : undefined; + const getRunnerIngressEndpoint = vi.fn(async () => { + const response = await fetch(`http://127.0.0.1:${address.port}/private-preview`); + if (options.checkpoint === "truncated") await db.delete(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, companyId), eq(nativeRunnerCheckpoints.runId, seeded.runId), + eq(nativeRunnerCheckpoints.part, "command:2"))); + if (options.checkpoint === "progress") await db.insert(heartbeatRunEvents).values({ + companyId, runId: seeded.runId, agentId: seeded.agentId, seq: 900, eventType: "turn.started", stream: "system", level: "info", + message: "Provider activity exists", payload: {}, + }); + if (!response.ok) { + const cause = new Error(await response.text()); + if (options.raw) throw cause; + const error = new PluginEnvironmentRunnerIngressError({ phase: "private_preview", httpStatus: response.status }); + settlementArmed = true; + throw new JsonRpcCallError({ code: -32000, message: error.message, data: environmentRunnerIngressErrorData(error) }); + } + throw new Error("expected the controlled ingress rejection"); + }); + provider.execute.mockReset().mockImplementation(async () => { + const controller = await currentNativeControllerIdentity(); + const driver = createNativeRunnerExecutionDriver({ + target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", leaseId: randomUUID(), + effectiveCapabilities: { runnerWebSocketIngress: true }, getRunnerIngressEndpoint } as never, + binding: seeded.execution.binding, runnerIngressAuthorized: true, prepareRemote: async () => {}, + controllerPersistence: { db, bootId: controller.bootId }, + processStarted: new Promise(() => {}), process: { runnerProcessLauncher: launch }, + }); + await driver.controlPlaneRegistration!(core); + throw new Error("ingress failure must precede process launch"); + }); + try { + await expect(executePaperclipNativeSession({ db, execution: seeded.execution, + runnerInstanceId: seeded.runnerInstanceId, useRunnerd: true, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner: {} } as never, + backend: { descriptor: async () => { throw new Error("unexpected descriptor"); }, + openSession: async () => { throw new Error("unexpected provider session"); } }, + })).rejects.toThrow(options.race === "claim" ? "native_session_lease_lost" : options.race + ? "native_runner_bootstrap_evidence_changed" : options.raw ? "502 Bad Gateway" : `HTTP ${options.status}`); + expect(getRunnerIngressEndpoint).toHaveBeenCalledOnce(); + expect(launch).not.toHaveBeenCalled(); + if (options.checkpoint !== "truncated") { + const saved = await loadNativeRunnerCheckpoint(db, seeded.execution.binding); + expect(saved?.recoveryEvidence.authenticatedConnectionCount).toBe(0); + expect(saved?.records.size).toBe(4); + } + const state = await read(seeded); + if (options.race) { + expect(state.coordinator.phase).not.toBe("retryable_failure"); + expect(state.coordinator.phase).not.toBe("terminal_failure"); + expect(state.coordinator.leaseOwner).not.toBeNull(); + expect(state.issue.executionRunId).toBe(seeded.runId); + expect(state.run.status).toBe("running"); + expect(state.run.costAccountedAt).toBeNull(); + expect(state.run.usageJson?.accountingProviderWorkStarted).not.toBe(false); + return; + } + expect(state.coordinator).toMatchObject({ phase: options.retry ? "retryable_failure" : "terminal_failure", attempt: options.finalAttempt ? 3 : 1, + failureDetail: { recoveryMode: options.retry || options.finalAttempt ? "bootstrap_retry" : "ambiguous_state", + providerSessionEstablished: options.checkpoint === "progress", providerEventsExist: options.checkpoint === "progress", checkpointExists: false } }); + if (options.retry) expect(state.coordinator.nextAttemptAt).toBeInstanceOf(Date); + else expect(state.coordinator.nextAttemptAt).toBeNull(); + expect(state.issue.status).toBe(options.retry ? "in_progress" : "blocked"); + expect(state.run.costAccountedAt).toBeNull(); + expect(state.coordinator.resultId).toBeNull(); + if (options.retry || options.checkpoint) { + expect(state.run.usageJson?.accountingProviderWorkStarted).not.toBe(false); + } else { + expect(state.run.usageJson?.accountingProviderWorkStarted).toBe(false); + expect(await accountRunCost(db, seeded.runId)).toBe(true); + expect((await read(seeded)).run.costAccountedAt).toBeInstanceOf(Date); + } + } finally { + settlementRace?.mockRestore(); + await core.stop(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + rmSync(root, { recursive: true, force: true }); + } + }); + it("retries a rejected session.open with growing backoff instead of blocking the task", async () => { const seeded = await seedRun(await createAgent()); diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index d28a01baba6..de94cf68272 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -96,6 +96,8 @@ export const providerFailureFromError = runner.providerFailureFromError; export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint; export const captureLegacyDurablePrpCheckpoint = runner.captureLegacyDurablePrpCheckpoint; export const inspectDurablePrpCheckpointHeader = runner.inspectDurablePrpCheckpointHeader; +export const inspectUnclaimedBootstrapCheckpoint = runner.inspectUnclaimedBootstrapCheckpoint; +export const inspectUnclaimedBootstrapState = runner.inspectUnclaimedBootstrapState; export const RUNNER_PRP_PROTOCOL_MIN = runner.RUNNER_PRP_PROTOCOL_MIN; export const RUNNER_PRP_PROTOCOL_MAX = runner.RUNNER_PRP_PROTOCOL_MAX; export const runnerCodexDynamicToolsFit = runner.runnerCodexDynamicToolsFit; diff --git a/server/src/vendor/paperclip-runner/testing.ts b/server/src/vendor/paperclip-runner/testing.ts index 895546253a9..4e037061408 100644 --- a/server/src/vendor/paperclip-runner/testing.ts +++ b/server/src/vendor/paperclip-runner/testing.ts @@ -32,6 +32,7 @@ export const releaseMaterializedNativeRuntimeSkills: RunnerTestingModule["releas runnerTesting.releaseMaterializedNativeRuntimeSkills; export const resolveRunnerdCodexSkillInputs: RunnerTestingModule["resolveRunnerdCodexSkillInputs"] = runnerTesting.resolveRunnerdCodexSkillInputs; +export const spawnRunner: RunnerTestingModule["spawnRunner"] = runnerTesting.spawnRunner; export const startOpenCodeMcpBridge: RunnerTestingModule["startOpenCodeMcpBridge"] = diff --git a/tests/runner-e2e/journal-evidence.test.ts b/tests/runner-e2e/journal-evidence.test.ts index 01b57f13e2d..acaf0a84d9c 100644 --- a/tests/runner-e2e/journal-evidence.test.ts +++ b/tests/runner-e2e/journal-evidence.test.ts @@ -16,12 +16,12 @@ function call(index: number) { } describe("large journal boundary evidence", () => { - it("requires one correctly owned journal over the boundary and an independent completed-call count", async () => { + it.each([1, 2])("requires one correctly owned v%s journal over the boundary and an independent completed-call count", async version => { const root = await mkdtemp(path.join(tmpdir(), "journal-evidence-")); const directory = path.join(root, "a".repeat(64), "control-plane"); await mkdir(directory, { recursive: true }); const filename = path.join(directory, "control-plane-state.json"); - const state = { schema: "paperclip.runner.durable.control-plane-state.v1", identity: { runId: "expected-run" }, commands: [] }; + const state = { schema: `paperclip.runner.durable.control-plane-state.v${version}`, identity: { runId: "expected-run" }, commands: [] }; const input = { stateRoot: root, runId: "expected-run", minimumBytes: 1024, minimumCompletedStimulusCalls: 2 }; try { await writeFile(filename, JSON.stringify(state)); @@ -33,6 +33,9 @@ describe("large journal boundary evidence", () => { expect(Object.keys(evidence)).toEqual(["journalBytes", "minimumBytes", "completedStimulusCalls", "minimumCompletedStimulusCalls"]); expect(evidence.completedStimulusCalls).toBe(2); await expect(largeJournalEvidence({ ...input, runId: "another-run" })).rejects.toThrow("boundary not reached"); + await writeFile(filename, JSON.stringify({ ...state, schema: "paperclip.runner.durable.control-plane-state.v999", + committedEvents: [...call(1), ...call(2)].map(event => ({ envelope: { payload: event } })) })); + await expect(largeJournalEvidence(input)).rejects.toThrow("boundary not reached"); } finally { await rm(root, { recursive: true, force: true }); } }); diff --git a/tests/runner-e2e/journal-evidence.ts b/tests/runner-e2e/journal-evidence.ts index 1884af20860..97fee4f64ff 100644 --- a/tests/runner-e2e/journal-evidence.ts +++ b/tests/runner-e2e/journal-evidence.ts @@ -56,7 +56,8 @@ export async function largeJournalEvidence(input: { identity?: { runId?: string }; committedEvents?: unknown[]; }; - if (state.schema === "paperclip.runner.durable.control-plane-state.v1" && state.identity?.runId === input.runId) { + if (["paperclip.runner.durable.control-plane-state.v1", "paperclip.runner.durable.control-plane-state.v2"] + .includes(String(state.schema)) && state.identity?.runId === input.runId) { const events = (state.committedEvents ?? []).map(entry => object(object(entry).envelope).payload); matches.push({ bytes: metadata.size, calls: completedJournalStimulusCalls(events, input.runId) }); }