import { computerService } from "../../modules/computers/index.js"; import { readNativeComputerWorkspaceReference } from "./native-workspace-sync.js"; import { adapterExecutionTargetIsCommandBacked, type AdapterComputerExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { configuredEnvironment } from "../../vendor/paperclip-runner/index.js"; import { agents } from "@paperclipai/db"; import { isCloudManagedInstance } from "../cloud-instance.js"; import { dotRunnerBroker } from "../dot-runner-broker.js"; import { CURSOR_DISTRIBUTION_PINS, QUALIFIED_ACPX_PROFILES, QUALIFIED_ACPX_VERSION } from "../../vendor/paperclip-runner/index.js"; import { isProviderMode } from "../../vendor/paperclip-runner/index.js"; import { bundledRemoteProviderPackManifestPath, bundledRemoteRunnerBinary } from "../../vendor/paperclip-runner/index.js"; import { nativeRetryCancellationEligible, rethrowNativeCancellationLockConflict, assertCancellationRequest, cancellationIntentId as callerCancellationIntentId, cancellationRequestId } from "./native-cancellation-request.js"; import { readNativePlanWait } from "./native-plan-wait.js"; import { nativeProviderLifecycle } from "./provider-lifecycle.js"; import { NativePermissionDeclinedError, readCompletedNativePermissionDecline } from "./native-permission-decline.js"; import { activeIssueInteractionCondition } from "../issue-question-context.js"; import { resolveAcpxQualification } from "./acpx-qualification.js"; import { readLocalAiCredentialFile } from "../local-ai-credential-file.js"; import { prepareGrokRunnerCredentials } from "./grok-runner-credentials.js"; import { copyBackGrokAuth } from "@paperclipai/adapter-grok-local/server"; import { inferOpenAiCompatibleBiller, type AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; import { isSupportedRemoteCodexVersion, parseCodexCliVersion, REMOTE_CODEX_SUPPORTED_RANGE, } from "./codex-runtime-compatibility.js"; import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js"; import { createNativeProviderFailureObservation } from "./native-provider-failure.js"; import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js"; import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js"; import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } from "./managed-native-credentials.js"; import { createLocalNativeQuestionBridge } from "./local-native-question-bridge.js"; import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js"; import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server"; import { codexCliVersionAtLeast, minimumCodexCliVersionForModel, normalizeCodexModel, } from "@paperclipai/adapter-codex-local"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js"; import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js"; import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js"; import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; import { resolveConnectorAssignments, isConnectorSkill } from "../connector-runtime.js"; import { boundedExecutionCleanup, EXECUTION_CONTROL_DEADLINE_MS, } from "../execution-control-deadline.js"; import { createHash, randomUUID } from "node:crypto"; import { execFile, execFileSync } from "node:child_process"; import { chmodSync, createReadStream, copyFileSync, closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, opendirSync, readFileSync, readdirSync, readlinkSync, readSync, renameSync, rmSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { DatabaseSync } from "node:sqlite"; import { basename, dirname, join, posix, resolve } from "node:path"; import type { AdapterExecutionResult, AdapterRuntimeEvent, } from "../../adapters/index.js"; import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared"; import type { HarnessRuntimeRequestResolution, NativeExecutionInput, NativeRuntimeContextSnapshot, NativeSession, NativeSessionBackend, PaperclipQuestionSet, PersistedNativeSession, PrpEvent, PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { NativeProviderTerminalFailure, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, completeRetainedNativeSessionCleanup, completeTerminatedLocalNativeSessionCleanup, acpxRuntimeSessionDirectoryName, createNativeSessionBackend, createRunnerdCodexTransport, executeNativeSession, applyNativeSessionGoalControl, inspectWarmRunTransition, parseNativeExecutionInput, parsePaperclipQuestionSet, resolveSourceCodexHome, readRunnerdArtifactBinding, retainedRunnerdMaintenanceIsIdle, settleRetainedRunnerdSession, validatePrpEvent, validatePrpStructuredRunResult, type RunnerProcessHandle, type RunnerProcessLaunchSpec, type NativeSessionGoalControl, } from "../../vendor/paperclip-runner/index.js"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js"; import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import { resolvePaperclipRunnerTransport, type PaperclipRunnerTransport, } from "@paperclipai/adapter-utils/runner-connectivity"; import type { Db } from "@paperclipai/db"; import { and, desc, eq, gt, inArray, isNull, like, notInArray, or, sql, } from "drizzle-orm"; import { agentWakeupRequests, documentRevisions, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueDocuments, issueThreadInteractions, issues, nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { nativeSha256 } from "./canonical.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { createAssignedMcpTools, getAssignedMcpGateway } from "./assigned-mcp-tools.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js"; import { NativeChatAttachmentReadScope } from "./chat-attachment-read.js"; import { assertCurrentWakeCommentsRead, resolveCurrentWakeCommentsBinding, } from "./current-wake-comments.js"; import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { registerRunnerPrpAuthority } from "../../realtime/runner-prp-ws.js"; import { connectRunnerPrpIngress } from "../../realtime/runner-prp-outbound.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; import { persistActivity, publishActivity } from "../activity-log.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import { documentService } from "../documents.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; import { issueService } from "../issues.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeAuthoritativeIssueStatus, type NativeStatusDecision, } from "./status-arbiter.js"; import { conflict, HttpError } from "../../errors.js"; import { redactSensitiveText } from "../../redaction.js"; import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js"; import { createNativeRunTrace, isNativeRunRootHistoricalSpan, nativeRunPreparationStarts, type NativeRunHistoricalSpan, type NativeRunSpanScope, type NativeRunTrace, } from "./native-run-trace.js"; import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { registerLiveRunnerGoalController } from "../runner-goal-control-broker.js"; import { applyRunnerGoalPrpEvent } from "../runner-goals.js"; import { readProcessStartedAt } from "../hot-restart.js"; import { NativeRunnerOwnershipUnverifiedError, isNativeRunnerOwnershipHeld, NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE, NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT, } from "./native-runner-ownership.js"; import { currentNativeControllerIdentity, nextNativeProviderAttempt, type NativeControllerIdentity, type NativeRestartRecoveryClaim, } from "./native-restart-recovery.js"; type ActiveNativeSession = { session: NativeSession; cancelRequested: boolean; restartDetach?: Promise; }; class NativeResultPendingFinalizationError extends Error { constructor() { super("native_result_pending_finalization"); this.name = "NativeResultPendingFinalizationError"; } } export class NativeCancellationPendingRecoveryError extends Error { constructor() { super("native_cancellation_pending_recovery"); this.name = "NativeCancellationPendingRecoveryError"; } } export class NativeControllerDetachedForRestartError extends Error { constructor() { super("native_controller_detached_for_restart"); this.name = "NativeControllerDetachedForRestartError"; } } const activeNativeSessions = new Map(); // Shutdown can race provider startup before onSession publishes its handle. // Retain the request for the remainder of this controller's lifetime so that // the late publication detaches before it can dispatch another turn. const nativeRunsDetachingForRestart = new Set(); // A restart must not cut authority rotation between archiving the previous // runner state and authenticating its replacement. Keep the startup owner // alive until onSession can detach it, or until startup itself has settled. type NativeSessionStartup = { promise: Promise; resolve: (session: ActiveNativeSession | null) => void; stopRequested?: boolean; cancellationSettled?: Promise; governedWait?: boolean; settled: Promise; settle: () => void; }; const nativeSessionStartups = new Map(); function detachActiveNativeSessionForRestart(active: ActiveNativeSession) { active.restartDetach ??= Promise.resolve().then(() => active.session.detachControllerForRestart!()); return active.restartDetach; } async function waitForNativeSessionStartup( runId: string, timeoutError: () => Error = () => new Error("native_restart_startup_not_ready"), ) { const startup = nativeSessionStartups.get(runId); if (!startup) return null; let timer: ReturnType | undefined; try { return await Promise.race([ startup.promise, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(timeoutError()), 30_000); }), ]); } finally { if (timer !== undefined) clearTimeout(timer); } } export async function detachNativeSessionsForRestart( runIds: readonly string[], ): Promise<{ detachedRunIds: string[]; inactiveRunIds: string[]; unsupportedRunIds: string[]; }> { const detachedRunIds: string[] = []; const inactiveRunIds: string[] = []; const unsupportedRunIds: string[] = []; const settlementDeadline = Date.now() + 20_000; for (const runId of new Set(runIds)) { // A governed stop already revoked new work and is collecting terminal // accounting. Give that exact owner a bounded chance to persist it before // fencing callbacks and relinquishing the runner. Timeout is not success. const settling = nativeSessionStartups.get(runId); if (settling?.governedWait) { let timer: ReturnType | undefined; try { await Promise.race([settling.settled, new Promise(resolve => { timer = setTimeout(resolve, Math.max(0, settlementDeadline - Date.now())); })]); } finally { clearTimeout(timer); } } nativeRunsDetachingForRestart.add(runId); const active = activeNativeSessions.get(runId) ?? await waitForNativeSessionStartup(runId); if (!active) { inactiveRunIds.push(runId); continue; } if (active.session.detachControllerForRestart === undefined) { unsupportedRunIds.push(runId); continue; } await detachActiveNativeSessionForRestart(active); detachedRunIds.push(runId); } return { detachedRunIds, inactiveRunIds, unsupportedRunIds }; } const MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES = 64 * 1024 * 1024; const MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES = 64 * 1024 * 1024; const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000; // Identity is read from the complete control-plane state, including its PRP // event window. Match the transport's bounded reader so verbose valid turns do // not lose continuation authority merely because their history exceeds 64 MiB. const NATIVE_CONTROL_PLANE_STATE_MAX_BYTES = 256 * 1024 * 1024; const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024; const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024; const CODEX_HOME_NON_PERSISTENT_ENTRIES = [ "tmp", ".tmp", "auth.json", "config.toml", ] as const; const RUNNERD_CONTROL_PLANE_STATE_SCHEMA = "paperclip.runner.durable.control-plane-state.v1"; const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1"; const CODEX_PROVIDER_STATE_SCHEMA = "paperclip.runner.codex-provider-state.v1"; const ACPX_PROVIDER_STATE_SCHEMA = "paperclip.runner.acpx-provider-state.v3"; const MANAGED_PROVIDER_STATE_SCHEMA = "paperclip.runner.managed-provider-state.v1"; const RUNNERD_STATE_LIFECYCLES = new Set([ "connecting", "ready", "backpressure", "recoverable_failure", "unrecoverable", "suspended", "stopped", "revoked", ]); const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([ "succeeded", "interrupted", "failed", "cancelled", "timed_out", ]); const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000; const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000; const NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS = 2_000; const NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX = 256; type NativeRuntimeRequestResolution = { runId: string; fingerprint: string; commandId: string; pending: Promise; completedAt: number | null; }; const nativeRuntimeRequestResolutions = new Map< string, NativeRuntimeRequestResolution >(); async function verifiedRecoveryProcessIsAlive(input: { pid: number; startedAt: string; }): Promise { try { process.kill(input.pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException | undefined)?.code !== "EPERM") return false; } try { const observed = await readProcessStartedAt(input.pid); return ( observed !== null && new Date(observed).getTime() === new Date(input.startedAt).getTime() ); } catch { return false; } } async function signalVerifiedRecoveryProcess( input: { pid: number; startedAt: string }, signal: NodeJS.Signals, ): Promise { if (!(await verifiedRecoveryProcessIsAlive(input))) return false; try { return process.kill(input.pid, signal); } catch { return false; } } function pruneNativeRuntimeRequestResolutionCache(): void { const completed = [...nativeRuntimeRequestResolutions.entries()] .filter(([, resolution]) => resolution.completedAt !== null) .sort( ([, left], [, right]) => (left.completedAt ?? 0) - (right.completedAt ?? 0), ); for ( let index = 0; index < completed.length - NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX; index += 1 ) { nativeRuntimeRequestResolutions.delete(completed[index]![0]); } } function clearNativeRuntimeRequestResolutions(runId: string): void { for (const [key, resolution] of nativeRuntimeRequestResolutions) { if (resolution.runId === runId) { nativeRuntimeRequestResolutions.delete(key); } } } export type NativeInstructionWorkingCopy = { /** Present only for a complete registered agent_files materialization. */ runId?: string; preparationKey?: string; root?: string; checkpointWarm?: () => Promise; hasChanges: () => Promise; collectStopped: () => Promise; retirementFailed?: () => Promise; }; type WarmNativeSession = { computerTarget?: AdapterComputerExecutionTarget; runId: string; instructionWorkingCopy?: NativeInstructionWorkingCopy; // Keep the admitted physical root independent of the per-run collection // capability, which instruction preparation adopts before final admission. instructionRoot?: string; instructionPreparationRunId?: string; agentId: string; instructionCopy?: { runId: string; root?: string; targetIdentity: string; collectStopped: () => Promise }; preparingRunId?: string; managedAiCredentialIdentity?: string; credentialRunId?: string; githubAccess?: NativeGitHubAccess; githubAuthenticationMode?: string; networkAccess: boolean; session: NativeSession; ownerToken: symbol; configDigest: string; configuredEnvironmentDigest: string; ownerScope: string; companyId: string; environmentId: string | null; busy: boolean; closeOnReleaseReason?: string; idleTimer: ReturnType | null; lastActivityAt: string; }; /** A scoped preparation failure is retryable only after exact retirement fences * queued commands. A process claim or invoked launcher keeps normal recovery. */ export async function retireUnlaunchedComputerAttempt( target: AdapterExecutionTarget | null | undefined, launchAttempted: boolean, ): Promise { if (target?.kind !== "remote" || target.transport !== "computer" || launchAttempted) return; const process = await target.inspectProcess(); if (process.running || process.claim !== null) return; if (!(await target.retire())) throw new NativeSessionCleanupQuarantinedError(); } class ComputerOwnerSupersededError extends Error { constructor() { super("computer_owner_retirement_superseded"); } } async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) { let closeFailure: unknown; let closeCompleted = false; const close = async () => { try { await entry.session.close({ reason }); closeCompleted = true; } catch (error) { closeFailure = error; throw error; } }; // Reserve the exact generation before protocol close. A winning admission // must never receive a stale controller's suspend/kill command. The module // always hard-retires descendants after the bounded graceful window. if (entry.computerTarget && !(await entry.computerTarget.retire({ beforeStop: close }))) { throw new ComputerOwnerSupersededError(); } const stopping = entry.githubAccess?.stop(); try { try { if (!entry.computerTarget) await close(); if (closeFailure || !closeCompleted) { await entry.session.detachControllerForRestart?.(); throw closeFailure ?? new NativeSessionCleanupQuarantinedError(); } } catch (error) { try { await entry.instructionWorkingCopy?.retirementFailed?.(); } catch (receiptError) { throw new AggregateError([error, receiptError], "Warm instruction retirement and receipt both failed"); } throw error; } await entry.instructionWorkingCopy?.collectStopped(); if (entry.instructionCopy?.runId !== preserveInstructionsForRunId) await entry.instructionCopy?.collectStopped(); } finally { await stopping; } } const warmNativeSessions = new Map(); // Closing a remote owner saves its checkpoint asynchronously. A new turn must // not inspect or quarantine that owner's state until the save has finished. const closingWarmNativeSessions = new Map>(); /** Reserve the existing physical copy before heartbeat composes AGENT_HOME and * task guidance. The final execution/configuration fence still owns admission. */ export async function claimWarmNativeInstructionCopy(input: { priorExecution: NativeExecutionInput; companyId: string; agentId: string; executionWorkspaceId: string; workspace: NativeExecutionInput["workspace"]; environmentId: string | null; runId: string; preparationKey: string; forceRetirement?: boolean; adopt: (previousRunId: string) => Promise; }): Promise<(() => Promise) | null> { const prior = input.priorExecution; if (prior.binding.companyId !== input.companyId || prior.binding.agentId !== input.agentId || JSON.stringify(nativeSessionWorkspaceScope(prior)) !== JSON.stringify(nativeSessionWorkspaceScope({ binding: { ...prior.binding, runId: input.runId, executionWorkspaceId: input.executionWorkspaceId }, workspace: input.workspace, })) || prior.workspace.cwd !== input.workspace.cwd) return null; const scope = nativeSessionScopeKey(prior); const entry = warmNativeSessions.get(scope); if (!entry || !entry.instructionWorkingCopy?.runId || entry.environmentId !== input.environmentId) return null; if (entry.busy) throw new Error("native_session_supervisor_busy"); entry.busy = true; if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; const retire = async (reason: string) => { if (warmNativeSessions.get(scope) !== entry) return; entry.closeOnReleaseReason = reason; try { await closeWarmNativeSession(entry, reason); if (warmNativeSessions.get(scope) === entry) warmNativeSessions.delete(scope); } finally { entry.busy = false; } }; try { const preparationChanged = input.forceRetirement === true || entry.instructionWorkingCopy.preparationKey !== input.preparationKey; // The prior capability may close over an expired remote lease. Rebind // collection authority first; adoption itself grants no reuse permission. const adopted = await input.adopt(entry.instructionWorkingCopy.runId); if (!adopted) { await retire("warm instruction materialization cannot be handed off"); throw new Error("native_instruction_materialization_handoff_unavailable"); } if (adopted.runId !== input.runId || adopted.preparationKey !== input.preparationKey) throw new Error("native_instruction_materialization_handoff_mismatch"); entry.instructionWorkingCopy = adopted; entry.instructionPreparationRunId = input.runId; if (entry.closeOnReleaseReason !== undefined || preparationChanged || await adopted.hasChanges()) { await retire("warm instruction materialization changed before preparation"); return null; } return async () => { if (entry.instructionPreparationRunId === input.runId) await retire("warm instruction preparation abandoned"); }; } catch (error) { if (entry.closeOnReleaseReason === undefined) await retire("warm instruction preparation failed"); throw error; } } function instructionTargetIdentity(target?: AdapterExecutionTarget | null): string { return JSON.stringify(target?.kind === "remote" ? { environmentId: target.environmentId, cwd: target.remoteCwd, providerLeaseId: target.transport === "sandbox" ? target.sandboxLeaseAcquisition?.providerLeaseId : target.transport === "computer" ? [target.resourceAuthority.computerId, target.resourceAuthority.ownerId] : target.spec, } : { kind: "local", environmentId: target?.environmentId }); } /** Reserve the actual live owner before handing its directory to another run. * A persisted conversation alone is not proof that a working tree is reusable. */ export async function reserveWarmNativeInstructionDirectory(input: { companyId: string; agentId: string; previousRunId: string; runId: string; target?: AdapterExecutionTarget | null; canReuse: () => Promise; }): Promise<{ reuseRunId: string; adopt: (root: string, collectStopped: () => Promise) => void; release: () => Promise } | null> { for (const [id, entry] of warmNativeSessions) { if (entry.companyId !== input.companyId || entry.agentId !== input.agentId || entry.instructionCopy?.runId !== input.previousRunId) continue; if (entry.busy || entry.preparingRunId) throw new Error("native_session_supervisor_busy"); if (input.target?.kind === "remote" && input.target.transport === "computer" && entry.computerTarget?.resourceAuthority.computerId === input.target.resourceAuthority.computerId && entry.computerTarget.resourceAuthority.ownerId === input.target.resourceAuthority.ownerId && entry.computerTarget.resourceAuthority.generation < input.target.resourceAuthority.generation) { entry.computerTarget = input.target; } if (entry.idleTimer) clearTimeout(entry.idleTimer); entry.idleTimer = null; entry.preparingRunId = input.runId; const retire = async (reason: string) => { entry.closeOnReleaseReason = reason; try { await closeWarmNativeSession(entry, reason); if (warmNativeSessions.get(id) === entry) warmNativeSessions.delete(id); } finally { // Failed containment remains registered and cannot be adopted. The next // admission retries retirement instead of launching over a live owner. entry.preparingRunId = undefined; } }; let reusable = false; const targetProven = input.target?.kind !== "remote" || input.target.transport !== "sandbox" || Boolean(input.target.sandboxLeaseAcquisition?.providerLeaseId); try { reusable = targetProven && !entry.closeOnReleaseReason && entry.instructionCopy.targetIdentity === instructionTargetIdentity(input.target) && await input.canReuse(); } finally { if (!reusable) { await retire("managed agent files changed or environment replaced"); } } if (!reusable) return null; return { reuseRunId: input.previousRunId, adopt: (root, collectStopped) => { entry.instructionCopy = { runId: input.runId, root, targetIdentity: instructionTargetIdentity(input.target), collectStopped }; }, release: async () => { if (warmNativeSessions.get(id) === entry && entry.preparingRunId === input.runId) { await retire("managed warm turn preparation ended without attachment"); } } }; } return null; } /** * Close idle native sessions before an operator destroys their remote * environment. A warm runner owns a long-lived sandbox command stream, so the * provider cannot safely delete that sandbox until the session has closed the * stream. Busy sessions are reported instead of interrupted; the environment * delete guard can then fail closed while their heartbeat run is still live. */ export async function closeWarmNativeSessionsForEnvironment(input: { environmentId: string; reason: string; }): Promise<{ closed: number; busy: number; failed: number }> { return closeIdleWarmNativeSessions(input); } /** Cancel one attempt without closing other agents on its computer. */ export async function closeWarmNativeSessionsForRun(input: { runId: string; reason: string }) { return closeIdleWarmNativeSessions(input); } /** Suspend idle owners and persist their remote backup before a controller * exits. Active turns keep their separate authenticated restart handoff. */ export async function closeIdleWarmNativeSessionsForRestart(): Promise<{ closed: number; busy: number; failed: number; }> { return closeIdleWarmNativeSessions({ reason: "controller restart", closeBusyOnRelease: true, }); } async function closeIdleWarmNativeSessions(input: { environmentId?: string; runId?: string; reason: string; closeBusyOnRelease?: boolean; }): Promise<{ closed: number; busy: number; failed: number }> { let closed = 0; let busy = 0; let failed = 0; for (const [sessionId, entry] of [...warmNativeSessions]) { if (input.runId !== undefined && entry.runId !== input.runId) continue; if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) { continue; } if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) { // A busy turn can complete while another idle session is checkpointing. // Fence that entry now so its eventual release cannot leave a new idle // owner behind after the shutdown sweep has already passed it. if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason; busy += 1; continue; } if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); // Keep a fenced owner until both retirement and stopped collection succeed. // Failure must not make a live directory eligible for a replacement owner. entry.busy = true; entry.closeOnReleaseReason = input.reason; const closing = Promise.resolve().then(() => closeWarmNativeSession(entry, input.reason), ); closingWarmNativeSessions.set(sessionId, closing); try { await closing; if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId); closed += 1; } catch { if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); failed += 1; } finally { entry.busy = false; if (closingWarmNativeSessions.get(sessionId) === closing) { closingWarmNativeSessions.delete(sessionId); } } } return { closed, busy, failed }; } function readBoundedNativeFile( path: string, maxBytes: number, errorCode: string, ): Buffer { const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); try { const before = fstatSync(descriptor); if (!before.isFile() || before.size > maxBytes) throw new Error(errorCode); const output = Buffer.allocUnsafe(before.size); let offset = 0; while (offset < output.length) { const bytesRead = readSync( descriptor, output, offset, output.length - offset, offset, ); if (bytesRead === 0) break; offset += bytesRead; } const after = fstatSync(descriptor); if ( offset !== before.size || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ino !== before.ino ) { throw new Error("native_state_file_changed"); } return output; } finally { closeSync(descriptor); } } const NATIVE_PROVIDER_HOST_ENV_KEYS = [ "PATH", "HOME", "USER", "LOGNAME", "SHELL", "TMPDIR", "TEMP", "TMP", "CODEX_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", "SystemRoot", "PATHEXT", ] as const; async function measureNativeRunnerSpan( trace: NativeRunTrace | undefined, name: string, fn: () => Promise, options: | string | { parentName?: string; attributes?: Record; } = {}, ): Promise { return trace ? trace.measure( name, fn, typeof options === "string" ? { parentName: options } : options, ) : fn(); } /** * Provider bootstrap needs a small amount of host process context even when * the agent has no configured env. In particular, an empty environment makes * a bare `codex` command unresolvable. Agent-configured values remain * authoritative and may intentionally override the host defaults, while the * server-selected workspace remains an immutable containment boundary. */ export function buildNativeProviderEnvironment( configured: NodeJS.ProcessEnv, host: NodeJS.ProcessEnv = process.env, assignedWorkspaceCwd?: string, ): NodeJS.ProcessEnv { const inherited = Object.fromEntries( NATIVE_PROVIDER_HOST_ENV_KEYS.flatMap((key) => { const value = host[key]; return typeof value === "string" && value.length > 0 ? [[key, value]] : []; }), ); const environment = { ...inherited, ...configured }; if (assignedWorkspaceCwd?.trim()) { environment.PAPERCLIP_WORKSPACE_CWD = assignedWorkspaceCwd; } return environment; } /** Missing candidate bindings must not turn the controller's ambient credentials into explicit input. */ export function resolveNativeProviderEnvironment( provider: NativeExecutionInput["provider"], configured: NodeJS.ProcessEnv | undefined, host: NodeJS.ProcessEnv = process.env, ): NodeJS.ProcessEnv { if (configured !== undefined) return configured; if (provider.kind === "acpx" && ["pi", "cursor", "copilot"].includes(provider.agent)) { return buildNativeProviderEnvironment({}, host); } return host; } type PlanSynchronization = { eventId: string; planId: string; providerRevision: number; status: | "synchronized" | "already_synchronized" | "conflict" | "invalid" | "approval_failed"; baseRevisionId: string | null; digest: string; documentRevision: number | null; currentRevisionId: string | null; confirmationId: string | null; }; type RuntimeQuestionFallback = { kind: "ask_user_questions"; idempotencyKey: string; sourceRunId: string; title: string | null; summary: string | null; continuationPolicy: "wake_assignee"; payload: { version: 1; title?: string; submitLabel?: string; supersedeOnUserComment: false; runtimeRequestId: string; questionSet: PaperclipQuestionSet; questions: Array<{ id: string; prompt: string; helpText?: string; selectionMode: "single" | "multi"; required: boolean; options: Array<{ id: string; label: string; description?: string; freeText?: boolean; }>; }>; }; }; /** Translate non-replayable live-input expirations into one durable interaction. */ export function runtimeQuestionFallbackFromEvent( event: Pick, ): RuntimeQuestionFallback | null { if (event.eventType !== "runtime_request.expired") return null; const payload = record(event.payload); if ( !["durable_handoff", "provider_process_lost"].includes( String(payload.reason), ) || payload.replayAllowed !== false ) return null; const request = record(payload.request); if ( payload.requestKind !== "runtime" || payload.requestType !== "input" || request.schema !== "paperclip.runtime_request.v2" || request.requestKind !== "runtime" || request.type !== "input" || typeof request.requestId !== "string" || payload.requestId !== request.requestId || typeof request.turnId !== "string" || typeof request.itemId !== "string" ) return null; let questionSet: PaperclipQuestionSet; try { questionSet = parsePaperclipQuestionSet(request.input); } catch { return null; } const questions = questionSet.questions.map((question) => ({ id: question.id, prompt: question.prompt, ...(question.helpText ? { helpText: question.helpText } : {}), selectionMode: question.answerMode === "multi_select" ? ("multi" as const) : ("single" as const), required: question.required, options: question.answerMode === "text" ? [ { id: "__paperclip_text__", label: question.textValidation?.inputType === "integer" ? "Enter an integer" : question.textValidation?.inputType === "number" ? "Enter a number" : "Enter your answer", freeText: true, }, ] : (question.options ?? []).map((option) => ({ id: option.id, label: option.label, ...(option.description ? { description: option.description } : {}), })), })); return { kind: "ask_user_questions", idempotencyKey: `runtime-input-durable:v1:${event.runId}:${request.requestId}`, sourceRunId: event.runId, title: questionSet.title?.slice(0, 240) ?? null, summary: questionSet.description?.slice(0, 1000) ?? null, continuationPolicy: "wake_assignee", payload: { version: 1, ...(questionSet.title ? { title: questionSet.title.slice(0, 240) } : {}), ...(questionSet.submitLabel ? { submitLabel: questionSet.submitLabel.slice(0, 120) } : {}), supersedeOnUserComment: false, runtimeRequestId: request.requestId, questionSet, questions, }, }; } /** * Materialize the durable replacement for a non-replayable runtime question. * * The interaction service enforces the fallback's stable idempotency key, so * this is safe both immediately after the event commit and while recovering an * exact duplicate whose original post-commit callback did not finish. */ export async function materializeRuntimeQuestionFallback(input: { db: Db; binding: { companyId: string; issueId: string; runId: string; agentId: string; }; event: Pick; }): Promise<{ fallback: RuntimeQuestionFallback; interaction: { id: string }; } | null> { const fallback = runtimeQuestionFallbackFromEvent(input.event); if (!fallback) return null; const interaction = await issueThreadInteractionService(input.db).create( { id: input.binding.issueId, companyId: input.binding.companyId, }, fallback as never, { agentId: input.binding.agentId, runId: input.binding.runId, systemId: "native-runtime-question-handoff", }, ); return { fallback, interaction }; } export function runtimeInputLifecycleMetric( event: Pick, ): { outcome: | "normalized" | "rejected" | "resolved" | "expired" | "durable_handoff" | "provider_loss_handoff" | "cancelled"; adapter: string; requestId: string | null; } | null { const payload = record(event.payload); const request = record(payload.request); if ( event.eventType === "runtime_request.created" && request.type === "input" ) { const origin = record(request.origin); return { outcome: "normalized", adapter: typeof origin.adapter === "string" ? origin.adapter : "unknown", requestId: typeof request.requestId === "string" ? request.requestId : null, }; } if ( event.eventType === "harness.diagnostic" && payload.code === "runtime_input_rejected" ) { return { outcome: "rejected", adapter: typeof payload.adapter === "string" ? payload.adapter : "unknown", requestId: null, }; } const terminalOutcome = event.eventType === "runtime_request.resolved" ? "resolved" : event.eventType === "runtime_request.expired" && payload.reason === "durable_handoff" ? "durable_handoff" : event.eventType === "runtime_request.expired" && payload.reason === "provider_process_lost" ? "provider_loss_handoff" : event.eventType === "runtime_request.expired" ? "expired" : event.eventType === "runtime_request.cancelled" ? "cancelled" : null; const requestType = payload.requestType ?? request.type; if (!terminalOutcome || requestType !== "input") return null; const origin = record(request.origin); return { outcome: terminalOutcome, adapter: typeof payload.adapter === "string" ? payload.adapter : typeof origin.adapter === "string" ? origin.adapter : "unknown", requestId: typeof payload.requestId === "string" ? payload.requestId : typeof request.requestId === "string" ? request.requestId : null, }; } export function providerPlanMarkdown(payload: Record): string { const completedMarkdown = typeof payload.markdown === "string" ? payload.markdown.trim() : ""; if (completedMarkdown) return completedMarkdown.slice(0, 256_000); const explanation = typeof payload.explanation === "string" ? payload.explanation.trim() : ""; const steps = Array.isArray(payload.steps) ? payload.steps : []; const lines = steps.slice(0, 256).flatMap((value) => { if (!value || typeof value !== "object" || Array.isArray(value)) return []; const step = value as Record; const body = typeof step.body === "string" ? step.body.trim().slice(0, 4_000) : ""; if (!body) return []; const status = step.status === "completed" ? "x" : " "; const suffix = step.status === "blocked" ? " _(blocked)_" : step.status === "in_progress" ? " _(in progress)_" : ""; return [`- [${status}] ${body}${suffix}`]; }); return [explanation, lines.join("\n")] .filter(Boolean) .join("\n\n") .slice(0, 256_000); } export function semanticProviderPlanMarkdown( result: Record, ): string | null { const artifacts = Array.isArray(result.artifacts) ? result.artifacts : []; for (const value of artifacts) { const artifact = record(value); if ( artifact.kind !== "native_provider_plan" || typeof artifact.ref !== "string" ) continue; const match = artifact.ref.match( /\s*([\s\S]*?)\s*<\/proposed_plan>/i, ); const completedMarkdown = match?.[1]?.trim(); if (completedMarkdown) return completedMarkdown.slice(0, 256_000); const embedded = artifact.ref.match( /^native-provider-plan:([^\n]+)\n([\s\S]+)$/i, ); if (embedded) { const title = embedded[1]! .replace(/^(?:DOT-\d+-)?/i, "") .replace(/-v\d+$/i, "") .replace(/-/g, " ") .trim(); const body = embedded[2]!.trim(); if (title && body) { return [`# ${title.charAt(0).toUpperCase()}${title.slice(1)}`, "", body] .join("\n") .slice(0, 256_000); } } if (/^\s*1\.\s+/.test(artifact.ref)) { const numberedPlan = artifact.ref .split(/\s+\|\s+(?=\d+\.\s+)/) .join("\n") .trim(); if (numberedPlan) return `# Plan\n\n${numberedPlan}`.slice(0, 256_000); } // Some qualified Codex builds use the artifact reference itself as a // compact, human-readable plan. Accept only an explicitly numbered form; // arbitrary opaque artifact references must never become plan documents. const inlineNumbered = artifact.ref.trim(); if (/\(1\)\s+.+\(2\)\s+/s.test(inlineNumbered)) { const body = inlineNumbered .replace(/^DOT-\d+\s+plan:\s*/i, "") .replace(/^\(1\)\s*/, "1. ") .replace(/;\s*\((\d+)\)\s*/g, "\n$1. ") .trim(); if (body) return `# Plan\n\n${body}`.slice(0, 256_000); } const compact = artifact.ref.match(/^native-provider-plan:([^#]+)#(.+)$/i) ?? artifact.ref.match(/^native-plan:\/\/[^/]+\/([^#]+)#(.+)$/i); if (!compact) continue; const humanize = (slug: string) => slug .replace( /\b(GET|POST|PUT|PATCH|DELETE)-([a-z0-9][a-z0-9-]*)/gi, (_whole, method: string, path: string) => `${method.toUpperCase()} /${path}`, ) .replace(/-/g, " ") .replace(/\bjson\b/gi, "JSON") .replace(/\bapi\b/gi, "API") .replace(/\s+/g, " ") .trim(); const title = humanize(compact[1]!.replace(/-v\d+$/i, "")); const steps = compact[2]!.split(";").flatMap((encoded) => { const parsed = encoded.match(/^\d+-(.+)$/); const sentence = humanize(parsed?.[1] ?? encoded); return sentence ? [sentence.charAt(0).toUpperCase() + sentence.slice(1)] : []; }); if (!title || steps.length === 0) continue; return [ `# ${title.charAt(0).toUpperCase()}${title.slice(1)}`, "", ...steps.map((step, index) => `${index + 1}. ${step}`), ] .join("\n") .slice(0, 256_000); } const hasNativePlanArtifact = artifacts.some( (value) => record(value).kind === "native_provider_plan", ); const summary = typeof result.summary === "string" ? result.summary.trim() : ""; const summaryPlan = hasNativePlanArtifact ? summary.match( /(?:^|:\s*)(1\)\s+[\s\S]+;\s*2\)\s+[\s\S]+;\s*3\)\s+[\s\S]+)$/, ) : null; if (summaryPlan) { const body = summaryPlan[1]! .replace(/^1\)\s*/, "1. ") .replace(/;\s*(\d+)\)\s*/g, "\n$1. ") .trim(); if (body) return `# Plan\n\n${body}`.slice(0, 256_000); } return null; } /** * Convert a server-owned pending interaction into the semantic wait that a * provider omitted. This is not a fabricated final response: it records that * the current turn intentionally yielded to a durable governance surface. */ export function nativeGovernedWaitResult(input: { interaction: { id: string; title: string | null; summary: string | null }; completionContract: NativeExecutionInput["completionContract"]["contract"]; }): PrpStructuredRunResult { const interactionRef = `interaction:${input.interaction.id}`; const label = input.interaction.title?.trim() || input.interaction.summary?.trim() || "the requested response"; return { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: `Waiting for ${label}.`, completionClaim: { contractRevision: input.completionContract.revision, objectiveSatisfied: false, criteria: input.completionContract.criteria.map((criterion) => ({ criterionId: criterion.id, status: "unknown", evidenceRefs: [interactionRef], })), remainingWork: [ { description: "Resume after the durable interaction is resolved.", blocksCompletion: true, }, ], }, evidence: [{ ref: interactionRef }], verification: [], attentionRequests: [], artifacts: [{ kind: "issue_thread_interaction", ref: interactionRef }], continuation: { kind: "response_wake", summary: "Resume from the resolved interaction response without repeating prior work.", idempotencyKey: `interaction-response:${input.interaction.id}`, }, }; } /** A completed chat reply yields to the next message without claiming task completion. */ export function nativeConversationReplyResult(input: { conversation: boolean; terminalEvent: PrpEvent; replyEvent: PrpEvent | null; completionContract: NativeExecutionInput["completionContract"]["contract"]; }): PrpStructuredRunResult | null { const reply = input.replyEvent; const payload = record(reply?.payload); const text = typeof payload.text === "string" ? payload.text.trim() : ""; if (!input.conversation || input.terminalEvent.eventType !== "turn.completed" || !reply || reply.eventType !== "item.completed" || payload.kind !== "agentMessage" || payload.channel !== "final" || !text || reply.runId !== input.terminalEvent.runId || reply.turnId !== input.terminalEvent.turnId || reply.normalizedSessionId !== input.terminalEvent.normalizedSessionId) return null; const ref = `run-event:${reply.sourceEventId}`; return { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: text.slice(0, 12_000), completionClaim: { contractRevision: input.completionContract.revision, objectiveSatisfied: false, criteria: input.completionContract.criteria.map((criterion) => ({ criterionId: criterion.id, status: "unknown", evidenceRefs: [ref], })), remainingWork: [], }, evidence: [{ ref }], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Wait for the next user message in this conversation.", idempotencyKey: `conversation-reply:${reply.sourceEventId}`, }, }; } /** * Bridge an asynchronous durable-interaction lookup to the runner package's * synchronous governed-wait boundary. Observations are single-use and bound * to one exact source event so a delayed or replayed lookup cannot leak into a * later provider event. */ export function createGovernedWaitEventObservation( resolvePending: () => Promise, ) { const pendingTools = new Set(); let providerLost = false; let generation = 0; let observation: { sourceInstanceId: string; sourceEventId: string; sourceSeq: number; result: PrpStructuredRunResult; } | null = null; return { async observe(event: PrpEvent, eligible: boolean): Promise { const currentGeneration = ++generation; observation = null; const payload = record(event.payload); providerLost ||= event.eventType === "runtime_request.expired" && payload.reason === "provider_process_lost"; const kind = payload.kind; const tool = ["dynamicToolCall", "mcpToolCall", "commandExecution"].includes(String(kind)); if (event.itemId) { if (tool && event.eventType === "item.started") pendingTools.add(event.itemId); // Terminal error events can omit kind; the tracked ID owns cleanup. if (event.eventType === "item.completed" || event.eventType === "item.failed") { pendingTools.delete(event.itemId); } } // Usage/model messages can arrive while the tool creating the card is // still awaiting its response. Parking then interrupts that in-flight // response and cannot produce a durable suspension checkpoint. if (event.eventType === "item.completed" && ( pendingTools.size > 0 || (!tool && !(kind === "agentMessage" && payload.channel === "final")) )) return; // A replacement question preserves human input after a provider crash; // it does not authorize a successful suspension of the failed execution. if (providerLost || !eligible) return; const result = await resolvePending(); if (generation !== currentGeneration || result === null) return; // If the interaction is answered after this read, parking remains the // fail-closed outcome: the durable answer owns the response-wake path. // Continuing provider work on a possibly stale authorization does not. observation = { sourceInstanceId: event.sourceInstanceId, sourceEventId: event.sourceEventId, sourceSeq: event.sourceSeq, result, }; }, consume(event: PrpEvent): PrpStructuredRunResult | null { generation += 1; const current = observation; observation = null; if ( current === null || current.sourceInstanceId !== event.sourceInstanceId || current.sourceEventId !== event.sourceEventId || current.sourceSeq !== event.sourceSeq ) { return null; } return current.result; }, }; } export function nativeToolsRefreshWaitResult(input: { wakeId: string; key: string; completionContract: NativeExecutionInput["completionContract"]["contract"]; }): PrpStructuredRunResult { const ref = `wakeup:${input.wakeId}`; return { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Continuing with the newly installed connection tools.", completionClaim: { contractRevision: input.completionContract.revision, objectiveSatisfied: false, criteria: input.completionContract.criteria.map((criterion) => ({ criterionId: criterion.id, status: "unknown", evidenceRefs: [ref], })), remainingWork: [ { description: "Continue in the queued session with updated tools.", blocksCompletion: true, }, ], }, evidence: [{ ref }], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "same_agent", summary: "Use the updated connection tools in a fresh session.", idempotencyKey: input.key, }, }; } /** * Partial item-verdict responses deliberately leave their original durable * interaction pending. They are already authority-checked before entering the * closed native envelope, so that exact interaction may park the continuation * run without requiring the model to recreate a second request. */ export function continuingPendingInteractionIds( execution: NativeExecutionInput, ): string[] { return execution.interactionResponses .filter( (response) => response.kind === "request_item_verdicts" && response.response.status === "pending", ) .map((response) => response.interactionId); } export async function synchronizeCompletedProviderPlan(input: { db: Db; execution: NativeExecutionInput; event: { sourceEventId: string; turnId?: string; eventType: string; payload: Record; }; }): Promise { if ( input.event.eventType !== "plan.updated" || input.event.payload.complete !== true ) return null; if ( !("executionMode" in input.execution) || input.execution.executionMode !== "plan" ) return null; const planningContext = input.execution.planningContext; if (!planningContext) return null; const planId = typeof input.event.payload.planId === "string" ? input.event.payload.planId : ""; const providerRevision = Number.isSafeInteger(input.event.payload.revision) ? Number(input.event.payload.revision) : 0; const body = providerPlanMarkdown(input.event.payload); const digest = createHash("sha256").update(body).digest("hex"); if (!planId || providerRevision < 1 || !body) { return { eventId: input.event.sourceEventId, planId, providerRevision, status: "invalid", baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: null, currentRevisionId: null, confirmationId: null, }; } const provenance = `runner-plan-sync:v2 run=${input.execution.binding.runId} turn=${input.event.turnId ?? "unknown"} provider=${input.execution.provider.kind} plan=${planId} revision=${providerRevision} digest=${digest}`; const existingRevision = await input.db .select({ revisionNumber: documentRevisions.revisionNumber, id: documentRevisions.id, }) .from(documentRevisions) .innerJoin( issueDocuments, eq(issueDocuments.documentId, documentRevisions.documentId), ) .where( and( eq(issueDocuments.issueId, input.execution.binding.issueId), eq(issueDocuments.key, "plan"), eq(documentRevisions.changeSummary, provenance), ), ) .orderBy(desc(documentRevisions.revisionNumber)) .limit(1) .then((rows) => rows[0] ?? null); const documents = documentService(input.db); let revision = existingRevision; let status: PlanSynchronization["status"] = existingRevision ? "already_synchronized" : "synchronized"; if (!revision) { const latest = await documents.getIssueDocumentByKey( input.execution.binding.issueId, "plan", ); if (latest?.latestRevisionId && latest.body === body) { const sameRunRevision = await input.db .select({ id: documentRevisions.id, revisionNumber: documentRevisions.revisionNumber, createdByRunId: documentRevisions.createdByRunId, }) .from(documentRevisions) .where(eq(documentRevisions.id, latest.latestRevisionId)) .limit(1) .then((rows) => rows[0] ?? null); if (sameRunRevision?.createdByRunId === input.execution.binding.runId) { revision = sameRunRevision; status = "already_synchronized"; } } } try { if (!revision) { const write = await documents.upsertIssueDocument({ issueId: input.execution.binding.issueId, key: "plan", title: "Plan", format: "markdown", body, baseRevisionId: planningContext.baseRevisionId, changeSummary: provenance, createdByAgentId: input.execution.binding.agentId, createdByRunId: input.execution.binding.runId, }); revision = { revisionNumber: write.document.latestRevisionNumber, id: write.document.latestRevisionId!, }; } } catch (error) { if (!(error instanceof HttpError) || error.status !== 409) throw error; const latest = await documents.getIssueDocumentByKey( input.execution.binding.issueId, "plan", ); return { eventId: input.event.sourceEventId, planId, providerRevision, status: "conflict", baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: latest?.latestRevisionNumber ?? null, currentRevisionId: latest?.latestRevisionId ?? null, confirmationId: null, }; } const current = await documents.getIssueDocumentByKey( input.execution.binding.issueId, "plan", ); if (!current || !revision?.id || current.latestRevisionId !== revision.id) { return { eventId: input.event.sourceEventId, planId, providerRevision, status: "conflict", baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: current?.latestRevisionNumber ?? null, currentRevisionId: current?.latestRevisionId ?? null, confirmationId: null, }; } let confirmationId: string; let confirmationPending = false; try { const confirmation = await issueThreadInteractionService(input.db).create( { id: input.execution.binding.issueId, companyId: input.execution.binding.companyId, }, { kind: "request_confirmation", idempotencyKey: `runner-plan-approval:v1:${input.execution.binding.runId}:${planId}:${providerRevision}:${digest}`, sourceRunId: input.execution.binding.runId, title: `Review plan revision ${revision.revisionNumber}`, summary: "Review the synchronized Paperclip plan.", continuationPolicy: "wake_assignee", payload: { version: 1, prompt: `Approve plan revision ${revision.revisionNumber}?`, detailsMarkdown: "The completed provider plan has been synchronized to the canonical Plan document.", acceptLabel: "Approve plan", rejectLabel: "Request changes", rejectRequiresReason: true, supersedeOnUserComment: false, target: { type: "issue_document", issueId: input.execution.binding.issueId, documentId: current.id, key: "plan", revisionId: revision.id, revisionNumber: revision.revisionNumber, label: `Plan v${revision.revisionNumber}`, }, }, } as never, { agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, }, ); confirmationId = confirmation.id; confirmationPending = confirmation.status === "pending"; } catch { return { eventId: input.event.sourceEventId, planId, providerRevision, status: "approval_failed", baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: revision.revisionNumber, currentRevisionId: revision.id, confirmationId: null, }; } if (confirmationPending) { try { const currentIssue = await input.db .select({ status: issues.status }) .from(issues) .where(eq(issues.id, input.execution.binding.issueId)) .limit(1) .then((rows) => rows[0] ?? null); if (currentIssue && currentIssue.status !== "in_review") { await issueService(input.db).update(input.execution.binding.issueId, { status: "in_review", actorAgentId: input.execution.binding.agentId, }); } } catch { const settledIssue = await input.db .select({ status: issues.status }) .from(issues) .where(eq(issues.id, input.execution.binding.issueId)) .limit(1) .then((rows) => rows[0] ?? null); if (settledIssue?.status !== "in_review") { return { eventId: input.event.sourceEventId, planId, providerRevision, status: "approval_failed", baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: revision.revisionNumber, currentRevisionId: revision.id, confirmationId, }; } } } return { eventId: input.event.sourceEventId, planId, providerRevision, status, baseRevisionId: planningContext.baseRevisionId, digest, documentRevision: revision.revisionNumber, currentRevisionId: revision.id, confirmationId, }; } class SessionToolAuthorityEpoch { readonly runId: string; #authority: PaperclipRunnerToolAuthority; #revoked = false; constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) { this.runId = runId; this.#authority = authority; } revoke(): void { this.#revoked = true; this.#authority.close(); } #assertCurrent(): void { if (this.#revoked) { throw new Error("native_tool_authority_epoch_revoked"); } } definitions() { this.#assertCurrent(); return this.#authority.definitions(); } async execute(call: Parameters[0]) { this.#assertCurrent(); return this.toolTrace ? await this.toolTrace.execute(call, () => this.#authority.execute(call)) : await this.#authority.execute(call); } } const sessionToolAuthorityEpochs = new Map(); const initializingSessionToolAuthorities = new Set(); const executingRunnerdSessionScopes = new Map(); const executingNativeOwnerScopes = new Map(); function nativeSessionKey(execution: NativeExecutionInput): string { return ( execution.session.normalizedSessionId ?? `session-${execution.binding.runId}` ); } /** Persistent computer artifacts belong to an agent session, independent of a shared repo cwd. */ export function remoteRunnerStorageRoots( target: Extract, sessionKey: string, ) { const home = target.transport === "computer" ? target.fileAuthority?.agentHome : target.remoteCwd; if (!home || !posix.isAbsolute(home)) throw new Error("runner_remote_storage_home_missing"); const base = posix.join(home, ".paperclip-runtime", "paperclip-runner"); const sessionRoot = posix.join(base, "sessions", createHash("sha256").update(sessionKey).digest("hex")); return { runtimeRoot: target.transport === "computer" ? sessionRoot : base, sessionRoot }; } export function nativeSessionWorkspaceScope(execution: { binding: Pick; workspace: NativeExecutionInput["workspace"] }) { if ("access" in execution.workspace) return { kind: "none" as const }; // Projectless local runs use the heartbeat run id as a durable placeholder // rather than fabricating an execution_workspaces row. Do not let that // per-run placeholder break continuity for the same provider session; the // immutable workspace descriptor is the stable identity in that case. const transientWorkspace = execution.binding.executionWorkspaceId === execution.binding.runId; return transientWorkspace ? { kind: "transient" as const, cwd: execution.workspace.cwd, repoUrl: execution.workspace.repoUrl, repoRef: execution.workspace.repoRef, branchName: execution.workspace.branchName, } : { kind: "managed" as const, executionWorkspaceId: execution.binding.executionWorkspaceId, }; } function nativeProviderSessionScope(execution: NativeExecutionInput) { switch (execution.provider.kind) { case "openai_dot": return { kind: "openai_dot", bindingId: execution.provider.binding.bindingId, generation: execution.provider.binding.bindingGeneration }; case "claude_managed": return { kind: execution.provider.kind, profileId: execution.provider.managedProfile.profileId, }; case "aws_agentcore": return { kind: execution.provider.kind, profileId: execution.provider.agentCoreProfile.profileId, }; case "acpx": return { kind: execution.provider.kind, agent: execution.provider.agent, profile: execution.provider.profile, }; case "codex": case "opencode": // These local providers have no separate persisted profile id. Company, // agent, and workspace bind their credential/session context; mutable // model and permission settings remain in nativeSessionConfigDigest so // they rotate an incompatible warm session within the same scope. return { kind: execution.provider.kind }; } } function nativeSessionScopeKey(execution: NativeExecutionInput): string { return canonicalJson({ schema: "paperclip.native-session-scope.v2", companyId: execution.binding.companyId, agentId: execution.binding.agentId, workspace: nativeSessionWorkspaceScope(execution), provider: { driverKind: execution.session.driverKind, identity: nativeProviderSessionScope(execution), }, normalizedSessionId: nativeSessionKey(execution), }); } // A plan acceptance or explicit reset can rotate the provider/session identity // while retaining the same task sandbox and its fixed ingress port. function nativeSessionOwnerScope( execution: NativeExecutionInput, environmentId: string | null, ): string { return canonicalJson({ companyId: execution.binding.companyId, agentId: execution.binding.agentId, issueId: execution.binding.issueId, workspace: nativeSessionWorkspaceScope(execution), environmentId, }); } async function retireSupersededWarmNativeSessions( ownerScope: string, nextSessionScope: string, ): Promise { for (const [scope, entry] of warmNativeSessions) { if (scope === nextSessionScope || entry.ownerScope !== ownerScope) continue; if (entry.busy || executingRunnerdSessionScopes.has(scope)) { throw new Error("native_session_supervisor_busy"); } if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; entry.busy = true; entry.closeOnReleaseReason = "warm native session identity changed"; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason); if (warmNativeSessions.get(scope) === entry) warmNativeSessions.delete(scope); } finally { // A failed close retains the owner and prevents launch. A later attempt // must retry retirement, never adopt this partially closed transport. entry.busy = false; } } } function legacyCompanyNativeSessionScopeKey( execution: NativeExecutionInput, ): string { return JSON.stringify([ execution.binding.companyId, nativeSessionKey(execution), ]); } function runnerdStateBase(): string { return ( process.env.PAPERCLIP_RUNNER_STATE_DIR ?? resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "durable-sessions", ) ); } function scopedRunnerdStateRoot(execution: NativeExecutionInput): string { return resolve( runnerdStateBase(), createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex"), ); } function scrubRunnerdQuarantineLaunchState(root: string): void { const codexHome = resolve(root, "codex-home"); const codexHomeStats = lstatSync(codexHome, { throwIfNoEntry: false }); if (!codexHomeStats) return; if (!codexHomeStats.isDirectory() || codexHomeStats.isSymbolicLink()) { throw new Error("runner_state_directory_unsafe"); } for (const name of CODEX_HOME_NON_PERSISTENT_ENTRIES) { const entry = resolve(codexHome, name); const stats = lstatSync(entry, { throwIfNoEntry: false }); if (!stats) continue; // Remove symlinks themselves, never their targets. Real temporary // directories are safe to remove recursively inside the verified home. rmSync(entry, { recursive: stats.isDirectory() && !stats.isSymbolicLink(), force: true, }); } } function quarantineRunnerdStateRoot( root: string, reason: "identity_indeterminate" | "identity_mismatch", ): string { const stateBase = resolve(runnerdStateBase()); const resolvedRoot = resolve(root); const stateKey = basename(resolvedRoot); // Callers pass only roots derived by this module. Keep that boundary explicit // so a future call site cannot turn quarantine into an arbitrary filesystem // move, and never follow or move a symlink in place of the state directory. if ( dirname(resolvedRoot) !== stateBase || !/^[a-f0-9]{64}$/.test(stateKey) || !isSafeNativeStateDirectory(resolvedRoot) ) { throw new Error("runner_state_directory_unsafe"); } // Quarantine retains durable session history for diagnosis and recovery, but // launch credentials and transient files are re-materializable and must not // survive after this state loses authority. scrubRunnerdQuarantineLaunchState(resolvedRoot); const quarantineRoot = resolve(stateBase, "quarantine"); mkdirSync(quarantineRoot, { recursive: true, mode: 0o700 }); if (!isSafeNativeStateDirectory(quarantineRoot)) { throw new Error("runner_state_directory_unsafe"); } const destination = resolve( quarantineRoot, `${stateKey}.${reason}.${Date.now()}.${process.pid}.${randomUUID()}`, ); renameSync(resolvedRoot, destination); return destination; } function legacyCompanyRunnerdStateRoot( execution: NativeExecutionInput, ): string { return resolve( runnerdStateBase(), createHash("sha256") .update(legacyCompanyNativeSessionScopeKey(execution)) .digest("hex"), ); } function legacyRunnerdStateRoot(execution: NativeExecutionInput): string { return resolve( runnerdStateBase(), createHash("sha256").update(nativeSessionKey(execution)).digest("hex"), ); } function migrateLegacyRunnerdStateRoot(input: { legacy: string; scoped: string; execution: NativeExecutionInput; verifiedPriorRunId?: string; }): string | null { if (!existsSync(input.legacy)) return null; if (!isSafeNativeStateDirectory(input.legacy)) { throw new Error("runner_state_directory_unsafe"); } const legacyIdentity = readRunnerdDurableIdentity(input.legacy); const exactRun = durableIdentityMatchesExecution( legacyIdentity, input.execution, ); const verifiedSettledPriorRun = Boolean( input.verifiedPriorRunId && legacyIdentity?.runId === input.verifiedPriorRunId && durableIdentityMatchesSession(legacyIdentity, input.execution), ); if (!exactRun && !verifiedSettledPriorRun) { // A legacy path does not encode the full session scope. A mismatch may be // valid live state owned by another agent/workspace, so refusing the claim // is safe but moving that ambiguous directory is not. throw new Error("runner_state_identity_mismatch: legacy_owner_unverified"); } if ( exactRun && legacyIdentity && ["absent", "indeterminate"].includes( runnerdAuthorityLifecycle( input.legacy, legacyIdentity as RunnerdDurableIdentity, ), ) ) { quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate"); throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate"); } try { renameSync(input.legacy, input.scoped); } catch (error) { // Another in-process recovery may have won the atomic rename. The scoped // directory is authoritative once it exists; otherwise preserve the // original migration failure rather than silently starting empty state. if (!isSafeNativeStateDirectory(input.scoped)) throw error; const scopedIdentity = readRunnerdDurableIdentity(input.scoped); const exactScopedRun = durableIdentityMatchesExecution( scopedIdentity, input.execution, ); const sameVerifiedPriorRun = Boolean( input.verifiedPriorRunId && scopedIdentity && scopedIdentity.runId === input.verifiedPriorRunId && scopedIdentity.runnerInstanceId === legacyIdentity?.runnerInstanceId && scopedIdentity.environmentLeaseId === legacyIdentity?.environmentLeaseId && durableIdentityMatchesSession(scopedIdentity, input.execution), ); if (!exactScopedRun && !sameVerifiedPriorRun) { throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed"); } } return input.scoped; } function runnerdAuthorityLifecycle( root: string, identity: RunnerdDurableIdentity, ): "absent" | "suspended" | "not_suspended" | "indeterminate" { const runnerRoot = resolve(root, "runner"); if (!existsSync(runnerRoot)) return "absent"; if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate"; const statePath = resolve(runnerRoot, "runner-state.json"); if (!existsSync(statePath)) { try { // Older remote transports created this controller-side placeholder even // though runner state was owned by the sandbox. It carries no authority, // so a verified failover backup may be consulted. Any non-empty direct // directory remains indeterminate and therefore blocks fallback. return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate"; } catch { return "indeterminate"; } } try { const state = record( JSON.parse( readBoundedNativeFile( statePath, NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large", ).toString("utf8"), ), ); if ( state.schema !== RUNNERD_STATE_SCHEMA || typeof state.lifecycle !== "string" || !RUNNERD_STATE_LIFECYCLES.has(state.lifecycle) ) { return "indeterminate"; } if ( state.runnerInstanceId === identity.runnerInstanceId && state.environmentLeaseId === identity.environmentLeaseId && state.runId === identity.runId && state.normalizedSessionId === identity.normalizedSessionId ) { if (state.lifecycle === "suspended") return "suspended"; return "not_suspended"; } return "indeterminate"; } catch { return "indeterminate"; } } function runnerdAuthorityLifecycleWithVerifiedBackup(input: { root: string; identity: RunnerdDurableIdentity; execution: NativeExecutionInput; allowVerifiedBackup: boolean; }): "suspended" | "not_suspended" | "indeterminate" { const direct = runnerdAuthorityLifecycle(input.root, input.identity); if (direct !== "absent") return direct; if (!input.allowVerifiedBackup) return "indeterminate"; const backup = verifyNativeHarnessBackup({ root: input.root, execution: input.execution, runnerInstanceId: input.identity.runnerInstanceId, }); if (!backup) return "indeterminate"; const backupLifecycle = runnerdAuthorityLifecycle( backup.root, input.identity, ); return backupLifecycle === "absent" ? "indeterminate" : backupLifecycle; } type PriorRunnerdStateVerification = | "verified" | "retained_warm_runner" | "active" | "authority_indeterminate" | "scope_mismatch" | "terminal_state_indeterminate" | "unavailable"; const CLEANUP_CANONICAL_FILES = [ "control-plane/control-plane-state.json", "runner/runner-state.json", "runner/codex-provider-state.json", ] as const; const CLEANUP_ACTIVATION_FILE = "cleanup-activation.json"; function cleanupStateSnapshot(root: string, providerFile = "codex-provider-state.json") { if ( ![root, resolve(root, "runner"), resolve(root, "control-plane")].every( isSafeNativeStateDirectory, ) ) { throw new Error("native_cleanup_maintenance_unproven"); } const files = [...CLEANUP_CANONICAL_FILES.slice(0, 2), `runner/${providerFile}`]; const bytes = files.map((file) => readBoundedNativeFile( resolve(root, file), file === "control-plane/control-plane-state.json" ? NATIVE_CONTROL_PLANE_STATE_MAX_BYTES : NATIVE_RUNNER_STATE_MAX_BYTES, "native_cleanup_maintenance_unproven", ), ); const [control, runner, provider] = bytes.map((value) => record(JSON.parse(value.toString("utf8"))), ); const fileSha256 = bytes.map((value) => createHash("sha256").update(value).digest("hex"), ) as [string, string, string]; return { control: control!, runner: runner!, provider: provider!, fileSha256, fingerprint: createHash("sha256") .update(JSON.stringify(fileSha256)) .digest("hex"), }; } function cleanupProcessAbsent(pid: unknown): pid is number { if ( process.platform === "win32" || !Number.isSafeInteger(pid) || Number(pid) <= 0 ) return false; return [Number(pid), -Number(pid)].every((target) => { try { process.kill(target, 0); return false; } catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; } }); } function cleanupCanonicalVacancy(root: string) { const stat = lstatSync(root, { throwIfNoEntry: false }); if (!stat) return null; if (!isSafeNativeStateDirectory(root) || readdirSync(root).length !== 0) { throw new Error("native_cleanup_maintenance_unproven"); } return { device: stat.dev, inode: stat.ino, mode: stat.mode, modifiedAt: stat.mtimeMs, }; } function cleanupArchiveRootIdentity(root: string) { if (!isSafeNativeStateDirectory(root)) throw new Error("native_cleanup_maintenance_unproven"); const stat = lstatSync(root); return { device: stat.dev, inode: stat.ino, mode: stat.mode }; } /** Raw runner events and normalized driver events are distinct streams. Bind * the retained raw journal to the server-accepted result, not a guessed shared * event identifier. This is read-only and never interprets a tool as a request. */ export function retainedNativeCleanupJournalMatches(input: { run: Pick< typeof heartbeatRuns.$inferSelect, | "id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "runnerInstanceId" | "completionContractId" | "completionContractSha256" >; execution: NativeExecutionInput; accepted: Pick< typeof nativeRunResults.$inferSelect, | "schemaStatus" | "resultJson" | "turnId" | "canonicalSha256" | "serverFingerprint" >; control: Record; providerSessionId: string; providerAccountSessionId?: string | null; persistedEvents: Array< Pick< typeof heartbeatRunEvents.$inferSelect, | "eventType" | "payload" | "sourceInstanceId" | "sourceEventId" | "sourceSeq" | "sourcePayloadSha256" > >; }): boolean { const { run, accepted, control } = input; const envelope = record(accepted.resultJson); const terminal = record(envelope.terminal); if ( accepted.schemaStatus !== "accepted" || !accepted.turnId || terminal.turnTerminalState !== "completed" || terminal.runTerminalState !== "succeeded" || !Array.isArray(control.committedEvents) || !Array.isArray(control.commands) ) return false; const canonical = { result: envelope.result, terminal: envelope.terminal, turnId: accepted.turnId, }; const fingerprint = nativeSha256({ runId: run.id, completionContractSha256: run.completionContractSha256, canonicalSha256: accepted.canonicalSha256, }); const validPersisted = input.persistedEvents.filter((row) => { const parsed = validatePrpEvent(record(row.payload).prpEvent); return ( parsed.ok && parsed.event.runId === run.id && parsed.event.normalizedSessionId === run.nativeSessionId && parsed.event.sourceInstanceId === row.sourceInstanceId && parsed.event.sourceEventId === row.sourceEventId && parsed.event.sourceSeq === row.sourceSeq && row.sourcePayloadSha256 === nativeSha256(parsed.event) ); }); const identities = validPersisted.filter( (row) => ["session.started", "session.resumed"].includes(row.eventType) && row.sourceInstanceId === run.runnerInstanceId, ); if (identities.length !== 1) return false; const identityEvent = record(record(identities[0]!.payload).prpEvent); const identityPayload = record(identityEvent.payload); if ( identityPayload.providerSessionId !== (input.providerAccountSessionId ?? input.providerSessionId) || identityPayload.driverSessionId !== input.providerSessionId || identityEvent.sourceKind !== "runner" || identityEvent.eventType !== identities[0]!.eventType || identityEvent.sourceEventId !== `${run.runnerInstanceId}:${run.id}:${identityEvent.sourceSeq}` ) return false; const boundDigest = validPersisted.some((row) => { const event = record(record(row.payload).prpEvent); return ( event.sourceKind === "control_plane" && nativeSha256({ binding: { companyId: run.companyId, issueId: run.nativeIssueId, agentId: run.agentId, runId: run.id, sessionId: run.nativeSessionId, sourceInstanceId: run.runnerInstanceId, controlPlaneSourceInstanceId: row.sourceInstanceId, completionContractId: run.completionContractId, completionContractSha256: run.completionContractSha256, }, ...canonical, }) === accepted.canonicalSha256 ); }); if (!( (accepted.canonicalSha256 === `sha256:${nativeSha256(canonical)}` && accepted.serverFingerprint === `sha256:${fingerprint}`) || (accepted.serverFingerprint === fingerprint && boundDigest) )) return false; const events = control.committedEvents.map((entry) => record(record(record(entry).envelope).payload), ); const durableIdentity = record(control.identity); const boundRaw = (event: Record) => validatePrpEvent(event).ok && event.runId === run.id && event.sourceInstanceId === run.runnerInstanceId && event.sourceKind === "runner" && event.normalizedSessionId === run.nativeSessionId && event.turnId === durableIdentity.turnId && event.itemId === durableIdentity.itemId; const commands = control.commands.map(record); const contract = input.execution.completionContract.contract; if ( !commands.some( (command) => ["run.prepare", "run.attach"].includes(String(command.type)) && command.status === "completed" && canonicalJson(record(command.payload).completionContract) === canonicalJson({ revision: contract.revision, criterionIds: contract.criteria.map((criterion) => criterion.id), }), ) ) return false; if ( !commands.some( (command) => command.type === "turn.start" && command.status === "completed" && record(record(command.result).result).providerTurnId === accepted.turnId, ) ) return false; if ( !events.some( (event) => event.eventType === "turn.accepted" && boundRaw(event) && record(event.payload).providerTurnId === accepted.turnId && record(event.payload).providerSessionId === input.providerSessionId, ) ) return false; const submissions = events.filter( (event) => event.eventType === "semantic_tool.input" && record(event.payload).semantic_tool && record(record(event.payload).semantic_tool).operationId === "paperclip_finish", ); if (submissions.length !== 1) return false; const event = submissions[0]!; const semantic = record(record(event.payload).semantic_tool); const correlation = record(semantic.correlation); const result = validatePrpStructuredRunResult(semantic.input); if ( !boundRaw(event) || !result.ok || canonicalJson(result.result) !== canonicalJson(envelope.result) || semantic.schema !== "paperclip.prp.semantic_tool.v1" || semantic.schemaVersion !== 1 || semantic.phase !== "input" || typeof semantic.callId !== "string" || !semantic.callId || record(semantic.content).digest !== `sha256:${nativeSha256(semantic.input)}` || event.runId !== run.id || event.sourceInstanceId !== run.runnerInstanceId || event.normalizedSessionId !== run.nativeSessionId || correlation.runId !== run.id || correlation.normalizedSessionId !== run.nativeSessionId || correlation.turnId !== event.turnId || correlation.itemId !== event.itemId ) return false; return ( events.some((candidate) => { const outcome = record(record(candidate.payload).semantic_tool); return ( candidate.eventType === "semantic_tool.result" && boundRaw(candidate) && candidate.runId === run.id && candidate.sourceInstanceId === run.runnerInstanceId && candidate.normalizedSessionId === run.nativeSessionId && outcome.schema === semantic.schema && outcome.schemaVersion === 1 && outcome.phase === "result" && outcome.operationId === semantic.operationId && outcome.callId === semantic.callId && outcome.outcome === "succeeded" && outcome.code === "semantic_tool_succeeded" && outcome.operationReceiptId === `operation_${semantic.callId}` && canonicalJson(outcome.correlation) === canonicalJson(semantic.correlation) ); }) && commands.some((command) => { const payload = record(command.payload); return ( command.type === "semantic_tool.result" && command.status === "completed" && payload.callId === semantic.callId && payload.operationId === semantic.operationId && payload.isError === false && payload.sourceEventId === event.sourceEventId && payload.sourceEventType === event.eventType && // Current receipts retain a digest; older journals retain the input. // If both exist, both must bind to the accepted semantic input. (payload.inputDigest === undefined ? canonicalJson(payload.input) === canonicalJson(semantic.input) : payload.inputDigest === nativeSha256(semantic.input) && (payload.input === undefined || canonicalJson(payload.input) === canonicalJson(semantic.input))) && canonicalJson(payload.correlation) === canonicalJson(semantic.correlation) && record(record(command.result).result).callId === semantic.callId ); }) ); } /** Durable, content-free evidence for an authenticated maintenance event. * Never feed raw cleanup output into the normal driver/progress namespace. */ export async function appendRetainedNativeCleanupEvent( db: Db, input: { companyId: string; agentId: string; runId: string; nativeSessionId: string; runnerInstanceId: string; requestId: string; event: PrpEvent; }, ): Promise { const parsed = validatePrpEvent(input.event); if ( !parsed.ok || parsed.event.runId !== input.runId || parsed.event.normalizedSessionId !== input.nativeSessionId || parsed.event.sourceInstanceId !== input.runnerInstanceId || parsed.event.sourceKind !== "runner" || !input.requestId ) { throw new Error("native_cleanup_maintenance_unproven"); } const event = parsed.event; const receipt = { schema: "paperclip.native_cleanup_event.v1", requestId: input.requestId, rawSourceInstanceId: event.sourceInstanceId, rawSourceEventId: event.sourceEventId, rawSourceSeq: event.sourceSeq, rawEventType: event.eventType, rawCanonicalSha256: nativeSha256(event), }; await appendHeartbeatRunEvent(db, { companyId: input.companyId, agentId: input.agentId, runId: input.runId, eventType: "native.cleanup.event", stream: "system", level: "info", payload: { nativeCleanupEvent: receipt }, nativeSource: { sourceInstanceId: `${input.runnerInstanceId}:cleanup:${input.requestId}`, sourceEventId: `cleanup:${input.requestId}:${event.sourceEventId}`, sourceSeq: event.sourceSeq, protocolSchemaVersion: 1, canonicalPayload: receipt, }, }); } function cleanupProviderHomeSnapshot(home: string, content: boolean) { const entries: Array<{ path: string; directory: boolean; dev: number; ino: number; size: number; mtimeMs: number; ctimeMs: number; sha256?: string; }> = []; let bytes = 0; const visit = (relative: string, depth: number) => { if (depth > 32 || entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES) throw new Error("native_cleanup_maintenance_unproven"); const path = resolve(home, relative); const metadata = lstatSync(path); if ( metadata.isSymbolicLink() || (!metadata.isFile() && !metadata.isDirectory()) ) throw new Error("native_cleanup_maintenance_unproven"); const entry = { path: relative, directory: metadata.isDirectory(), dev: metadata.dev, ino: metadata.ino, size: metadata.isDirectory() ? 0 : metadata.size, mtimeMs: metadata.mtimeMs, ctimeMs: metadata.ctimeMs, }; entries.push(entry); if (entry.directory) { const directory = opendirSync(path); const children: string[] = []; try { for ( let child = directory.readSync(); child !== null; child = directory.readSync() ) { if ( !relative && (CODEX_HOME_NON_PERSISTENT_ENTRIES as readonly string[]).includes( child.name, ) ) continue; if (children.length + entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES) throw new Error("native_cleanup_maintenance_unproven"); children.push(child.name); } } finally { directory.closeSync(); } for (const name of children.sort()) visit(relative ? `${relative}/${name}` : name, depth + 1); } else { bytes += metadata.size; if (bytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES) throw new Error("native_cleanup_maintenance_unproven"); if (content) { const value = readBoundedNativeFile( path, metadata.size, "native_cleanup_maintenance_unproven", ); (entry as (typeof entries)[number]).sha256 = createHash("sha256") .update(value) .digest("hex"); } } const after = lstatSync(path); if ( after.dev !== metadata.dev || after.ino !== metadata.ino || after.mtimeMs !== metadata.mtimeMs || after.ctimeMs !== metadata.ctimeMs || after.size !== metadata.size ) throw new Error("native_cleanup_maintenance_unproven"); }; visit("", 0); return { entries, bytes, metadataFingerprint: nativeSha256( entries.map(({ sha256: _sha, ...entry }) => entry), ), fingerprint: content ? nativeSha256( entries.map(({ path, directory, size, sha256 }) => ({ path, directory, size, ...(sha256 ? { sha256 } : {}), })), ) : null, }; } function copyCleanupProviderHome( source: string, destination: string, snapshot: ReturnType, ) { for (const entry of snapshot.entries) { const target = resolve(destination, entry.path); if (entry.directory) mkdirSync(target, { mode: 0o700 }); else { const value = readBoundedNativeFile( resolve(source, entry.path), entry.size, "native_cleanup_maintenance_unproven", ); if (createHash("sha256").update(value).digest("hex") !== entry.sha256) throw new Error("native_cleanup_maintenance_unproven"); writeFileSync(target, value, { flag: "wx", mode: 0o600 }); } } if ( cleanupProviderHomeSnapshot(source, false).metadataFingerprint !== snapshot.metadataFingerprint || cleanupProviderHomeSnapshot(destination, true).fingerprint !== snapshot.fingerprint ) throw new Error("native_cleanup_maintenance_unproven"); } export function rebaseRetainedNativeCleanupProviderHome( home: string, canonicalHome: string, threadId: string, destination: "staging" | "canonical", ) { if (resolve(home) === resolve(canonicalHome)) throw new Error("native_cleanup_maintenance_unproven"); const snapshot = cleanupProviderHomeSnapshot(home, true); const rollouts = snapshot.entries.filter( (entry) => !entry.directory && entry.path.startsWith("sessions/") && entry.path.endsWith(`-${threadId}.jsonl`), ); if (rollouts.length !== 1) throw new Error("native_cleanup_maintenance_unproven"); const rollout = rollouts[0]!; const bytes = readBoundedNativeFile( resolve(home, rollout.path), rollout.size, "native_cleanup_maintenance_unproven", ); const newline = bytes.indexOf(10); if (newline < 0 || newline > 64 * 1024) throw new Error("native_cleanup_maintenance_unproven"); const first = record(JSON.parse(bytes.subarray(0, newline).toString("utf8"))); if (first.type !== "session_meta" || record(first.payload).id !== threadId) throw new Error("native_cleanup_maintenance_unproven"); // Only open the NEW private copy. Codex 0.153.4 deliberately does not fall // back to a filesystem scan for paginated threads: SQLite selects the exact // immutable rollout, which can differ after thread/revert. Relocate only // that already-proven path, never choose another rollout or change its mode. const sqlite = resolve(home, "state_5.sqlite"); if ( snapshot.entries.some( (entry) => /^state_\d+\.sqlite$/.test(entry.path) && entry.path !== "state_5.sqlite", ) ) throw new Error("native_cleanup_maintenance_unproven"); if (!existsSync(sqlite)) return; const database = new DatabaseSync(sqlite); try { database.exec("BEGIN IMMEDIATE"); // The pinned schema has insert and timestamp-only triggers. None fire // for this column-only update; unknown/general update triggers deny it. if ( database .prepare( "SELECT sql FROM sqlite_schema WHERE type = 'trigger' AND tbl_name COLLATE NOCASE = 'threads'", ) .all() .some( (trigger) => typeof trigger.sql !== "string" || !/^CREATE\s+TRIGGER\s+[a-z_][a-z0-9_]*\s+AFTER\s+(?:INSERT|UPDATE\s+OF\s+(?:created_at|updated_at))\s+ON\s+threads\s/i.test( trigger.sql, ), ) ) throw new Error("native_cleanup_maintenance_unproven"); // Foreign-key actions can also mutate other tables without an explicit // trigger. Unknown cascading topology is not an exact path relocation. if ( database .prepare( `SELECT 1 FROM sqlite_schema AS s JOIN pragma_foreign_key_list(s.name) AS f WHERE s.type = 'table' AND f."table" COLLATE NOCASE = 'threads' AND f.on_update NOT IN ('NO ACTION', 'RESTRICT') LIMIT 1`, ) .get() ) throw new Error("native_cleanup_maintenance_unproven"); const row = database .prepare("SELECT * FROM threads WHERE id = ?") .get(threadId); if ( !row || typeof row.rollout_path !== "string" || ![ resolve(home, rollout.path), resolve(canonicalHome, rollout.path), ].includes(row.rollout_path) || !["legacy", "paginated"].includes(String(row.history_mode)) || (record(first.payload).history_mode ?? "legacy") !== row.history_mode ) throw new Error("native_cleanup_maintenance_unproven"); const target = resolve( destination === "staging" ? home : canonicalHome, rollout.path, ); if (row.rollout_path !== target) { const changed = database .prepare( "UPDATE threads SET rollout_path = ? WHERE id = ? AND rollout_path = ? AND history_mode = ?", ) .run(target, threadId, row.rollout_path, row.history_mode); if (changed.changes !== 1) throw new Error("native_cleanup_maintenance_unproven"); } const after = database .prepare("SELECT * FROM threads WHERE id = ?") .get(threadId); if (nativeSha256(after) !== nativeSha256({ ...row, rollout_path: target })) throw new Error("native_cleanup_maintenance_unproven"); database.exec("COMMIT"); } catch (error) { if (database.isTransaction) database.exec("ROLLBACK"); throw error; } finally { database.close(); } } /** Physical cleanup for an explicitly authorized NEW conversation turn. Unlike * automatic replacement, this does not certify or replay the interrupted actions. * The caller holds the issue/controller/run locks and preserves their history. * Retain the old durable root permanently; only the exact in-memory cleanup * owner is retired, and the successor must use a fresh normalized session. */ export async function verifyStoppedNativeSessionForContinuation( db: Db, run: typeof heartbeatRuns.$inferSelect, ): Promise<{ evidence: Record; retire: () => boolean } | null> { try { if (run.runtimeMode !== "native" || !["failed", "cancelled", "interrupted", "timed_out"].includes(run.status) || !run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null; const execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); if (!(["codex", "acpx"] as string[]).includes(execution.provider.kind) || execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId || nativeSessionKey(execution) !== run.nativeSessionId || record(run.runnerProfileJson).nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget("local")) return null; const scope = nativeSessionScopeKey(execution); const idle = () => !activeNativeSessions.has(run.id) && !executingRunnerdSessionScopes.has(scope) && !initializingSessionToolAuthorities.has(scope) && !warmNativeSessions.has(scope); if (!idle()) return null; const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id))); if (leases.some(lease => lease.provider !== "local" || !lease.releasedAt || lease.cleanupStatus === "failed")) return null; const stopped = await readNativeLocalProcessStop(db, run.companyId, run.id); if (!stopped) return null; const root = scopedRunnerdStateRoot(execution); const providerFile = runnerProviderStateFilename(execution); const snapshot = cleanupStateSnapshot(root, providerFile); const identity = record(snapshot.control.identity); if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId || !["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key => typeof identity[key] === "string" && identity[key] && snapshot.runner[key] === identity[key]) || !Array.isArray(snapshot.control.committedEvents)) return null; // An incomplete provider launch can own a process that never emitted its // session identity. It cannot be certified from an earlier owner's receipt. if (snapshot.control.schema !== "paperclip.runner.durable.control-plane-state.v1" || snapshot.runner.schema !== RUNNERD_STATE_SCHEMA || snapshot.provider.schema !== (execution.provider.kind === "codex" ? "paperclip.runner.codex-provider-state.v1" : ACPX_PROVIDER_STATE_SCHEMA) || !["turn_active", "prepared", "suspended"].includes(String(snapshot.provider.lifecycle)) || snapshot.provider.startupAttempt != null) return null; const pending = JSON.stringify([snapshot.runner.outbox, snapshot.provider.pendingEvents, snapshot.provider.queuedEvents]); if (/session\.(started|resumed|reconciled)/.test(pending)) return null; const events = snapshot.control.committedEvents.map(entry => record(record(record(entry).envelope).payload)); const providerEvents = events.filter(event => ["session.started", "session.resumed", "session.reconciled"].includes(String(event.eventType))); if (!providerEvents.length) return null; const receipts = await db.select().from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), inArray(heartbeatRunEvents.eventType, ["session.started", "session.resumed", "session.reconciled", "harness.diagnostic"]))); const providerPids = new Set(); for (const event of providerEvents) { const provider = record(event.payload); if (!validatePrpEvent(event).ok || event.sourceKind !== "runner" || event.sourceInstanceId !== run.runnerInstanceId || event.runId !== run.id || event.normalizedSessionId !== run.nativeSessionId || typeof provider.providerSessionId !== "string" || !provider.providerSessionId || !cleanupProcessAbsent(provider.processId) || provider.processId === stopped.processPid) return null; if (event.eventType === "session.reconciled" && (!providerPids.has(Number(provider.previousProcessId)) || !cleanupProcessAbsent(provider.previousProcessId))) return null; const receipt = receipts.find(row => row.sourceEventId === `${run.runnerInstanceId}:${run.id}:${event.sourceSeq}`); const durable = record(record(receipt?.payload).prpEvent); const durableProvider = record(durable.payload); if (!receipt || !validatePrpEvent(durable).ok || durable.sourceInstanceId !== event.sourceInstanceId || durable.runId !== run.id || durable.normalizedSessionId !== run.nativeSessionId || // Normalized session-open receipts omit turn/item IDs. Those belong // to the durable runner identity checked above, not the open event. durable.sourceSeq !== event.sourceSeq || durable.eventType !== event.eventType || receipt.sourcePayloadSha256 !== nativeSha256(durable) || (durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) || (durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null; providerPids.add(provider.processId); } if (receipts.filter(row => record(record(row.payload).prpEvent).eventType !== "harness.diagnostic").length !== providerEvents.length) return null; // ACPX's sidecar is not its agent process. Include separately recorded // owners from both the durable journal and the independent DB/checkpoint. // Extra evidence can only add a stop requirement, never waive one. const owners: Record[] = [record(record(run.runnerProfileJson?.sessionCheckpoint).process), snapshot.provider, record(snapshot.provider.identity), record(snapshot.provider.descriptor)]; for (const event of [...events, ...receipts.map(row => record(record(row.payload).prpEvent))]) { const payload = record(event.payload); if (["session.started", "session.resumed", "session.reconciled"].includes(String(event.eventType))) { owners.push(payload, record(payload.providerDescriptor), record(payload.providerIdentity), record(payload.runtimeIdentity)); } else if (event.eventType === "harness.diagnostic" && payload.providerMethod === "acpx/process") { owners.push({ agentPid: payload.pid }); } } for (const owner of owners) for (const key of [ "processId", "process_id", "processGroupId", "providerPid", "codexPid", "sidecarPid", "agentPid", "agentProcessId", ]) { const pid = owner[key]; if (pid === null || pid === undefined) continue; if (!cleanupProcessAbsent(pid)) return null; providerPids.add(pid); } const unchanged = () => idle() && cleanupProcessAbsent(stopped.processPid) && [...providerPids].every(cleanupProcessAbsent) && cleanupStateSnapshot(root, providerFile).fingerprint === snapshot.fingerprint; return { evidence: { schema: "paperclip.stopped_native_conversation.v1", runId: run.id, nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId, processPid: stopped.processPid, providerPids: [...providerPids], stateFingerprint: snapshot.fingerprint }, retire: () => { try { return unchanged() && completeTerminatedLocalNativeSessionCleanup({ companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId!, }); } catch { return false; } }, }; } catch { return null; } } /** Prove that a crashed local Codex turn ended without external effects. No provider * is launched and no retained state is rewritten. The successor must use a fresh * normalized session; the old directory remains available for investigation. */ export async function verifyStoppedNativeSessionForReplacement( db: Db, run: typeof heartbeatRuns.$inferSelect, ): Promise<{ evidence: Record; retire: () => boolean } | null> { try { if (run.runtimeMode !== "native" || run.status !== "failed" || !run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null; const execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); if (execution.provider.kind !== "codex" || execution.session.driverKind !== "codex_app_server" || execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId || nativeSessionKey(execution) !== run.nativeSessionId || record(run.runnerProfileJson).nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget("local")) return null; const scope = nativeSessionScopeKey(execution); const idle = () => !activeNativeSessions.has(run.id) && !executingRunnerdSessionScopes.has(scope) && !initializingSessionToolAuthorities.has(scope) && !warmNativeSessions.has(scope); if (!idle()) return null; const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id))); if (leases.some(lease => lease.provider !== "local" || !lease.releasedAt)) return null; const stopped = await readNativeLocalProcessStop(db, run.companyId, run.id); if (!stopped) return null; const root = scopedRunnerdStateRoot(execution); const snapshot = cleanupStateSnapshot(root); const identity = record(snapshot.control.identity); if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId || !["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key => typeof identity[key] === "string" && identity[key] && snapshot.runner[key] === identity[key]) || snapshot.runner.lifecycle !== "ready" || snapshot.provider.lifecycle !== "turn_active" || record(snapshot.provider.config).provider !== "codex" || record(snapshot.provider.config).cwd !== execution.workspace.cwd || !Array.isArray(snapshot.control.committedEvents) || !Array.isArray(snapshot.control.commands)) return null; const events = snapshot.control.committedEvents.map(entry => record(record(record(entry).envelope).payload)); const providerEvent = events.filter(event => ["session.started", "session.resumed"].includes(String(event.eventType))).at(-1); const provider = record(providerEvent?.payload); const bound = (event: Record) => validatePrpEvent(event).ok && event.sourceKind === "runner" && event.sourceInstanceId === run.runnerInstanceId && event.runId === run.id && event.normalizedSessionId === run.nativeSessionId && event.turnId === identity.turnId && event.itemId === identity.itemId; if (!providerEvent || !bound(providerEvent) || !cleanupProcessAbsent(provider.processId) || provider.processId === stopped.processPid || typeof provider.providerSessionId !== "string" || snapshot.provider.threadId !== provider.providerSessionId || typeof snapshot.provider.activeProviderTurnId !== "string") return null; const receipts = await db.select().from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), inArray(heartbeatRunEvents.eventType, ["session.started", "session.resumed"]))).limit(2); const receipt = receipts.length === 1 ? receipts[0] : undefined; const durableEvent = record(record(receipt?.payload).prpEvent); // The persisted adapter enriches identity payloads with driverSessionId. Bind // the normalized and provider identities explicitly instead of comparing raw JSON. const durableProvider = record(durableEvent.payload); if (!receipt || !validatePrpEvent(durableEvent).ok || durableEvent.runId !== run.id || durableEvent.sourceInstanceId !== run.runnerInstanceId || durableEvent.normalizedSessionId !== run.nativeSessionId || receipt.sourceEventId !== `${run.runnerInstanceId}:${run.id}:${durableEvent.sourceSeq}` || receipt.sourcePayloadSha256 !== nativeSha256(durableEvent) || (durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) || (durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null; const turnId = snapshot.provider.activeProviderTurnId; if (!snapshot.control.commands.map(record).some(command => command.type === "turn.start" && command.status === "completed" && record(record(command.result).result).providerTurnId === turnId) || !events.some(event => bound(event) && event.eventType === "turn.accepted" && record(event.payload).providerTurnId === turnId && record(event.payload).providerSessionId === provider.providerSessionId)) return null; // Completion bookkeeping may precede the final answer. Its exact accepted // receipt is safe to preserve; arbitrary provider tools still prevent replay. const completedTaskControlCalls: Array<{ callId: string; input: unknown }> = []; for (const event of events.filter(event => event.eventType === "semantic_tool.input")) { const semantic = record(record(event.payload).semantic_tool); const correlation = record(semantic.correlation); if (!bound(event) || semantic.operationId !== "paperclip_finish" || semantic.phase !== "input" || typeof semantic.callId !== "string" || !validatePrpStructuredRunResult(semantic.input).ok || correlation.runId !== run.id || correlation.normalizedSessionId !== run.nativeSessionId || correlation.turnId !== identity.turnId || correlation.itemId !== identity.itemId || record(semantic.content).digest !== `sha256:${nativeSha256(semantic.input)}` || !events.some(resultEvent => { const result = record(record(resultEvent.payload).semantic_tool); return bound(resultEvent) && resultEvent.eventType === "semantic_tool.result" && result.operationId === semantic.operationId && result.callId === semantic.callId && result.outcome === "succeeded" && result.operationReceiptId === `operation_${semantic.callId}` && canonicalJson(result.correlation) === canonicalJson(semantic.correlation); }) || !snapshot.control.commands.map(record).some(command => { const payload = record(command.payload); return command.type === "semantic_tool.result" && command.status === "completed" && payload.callId === semantic.callId && payload.operationId === semantic.operationId && payload.sourceEventId === event.sourceEventId && payload.isError === false && record(payload.result).success === true && canonicalJson(payload.input) === canonicalJson(semantic.input) && canonicalJson(payload.correlation) === canonicalJson(semantic.correlation); })) return null; completedTaskControlCalls.push({ callId: semantic.callId, input: semantic.input }); } if (completedTaskControlCalls.length > 1) return null; const pendingInventory = JSON.stringify([snapshot.runner.outbox, snapshot.provider.pendingEvents, snapshot.provider.queuedEvents]); if (/semantic_tool\.input|mcp_app\.tool_input|runtime\.input\.requested|runtime_request\.created/.test(pendingInventory) || events.some(event => ["mcp_app.tool_input", "runtime.input.requested", "runtime_request.created"].includes(String(event.eventType)))) return null; if (snapshot.control.commands.map(record).some(command => command.status === "pending" && !["turn.stop", "runner.drain", "runner.suspend"].includes(String(command.type)))) return null; const home = cleanupProviderHomeSnapshot(resolve(root, "codex-home"), false); const rollouts = home.entries.filter(entry => !entry.directory && entry.path.startsWith("sessions/") && basename(entry.path).endsWith(`-${provider.providerSessionId}.jsonl`)); if (rollouts.length !== 1 || rollouts[0]!.size > 32 * 1024 * 1024) return null; const rolloutPath = resolve(root, "codex-home", rollouts[0]!.path); const bytes = readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven"); // A partial final write is not a closed transcript. if (!bytes.toString("utf8").endsWith("\n")) return null; const rows = bytes.toString("utf8").trimEnd().split("\n").map(line => JSON.parse(line)); if (!stoppedCodexTurnIsTextOnly({ rows, threadId: provider.providerSessionId, turnId, cwd: execution.workspace.cwd ?? "", completedTaskControlCalls })) return null; const rolloutSha256 = nativeSha256(bytes.toString("utf8")); const evidence = { schema: "paperclip.stopped_text_turn.v1", runId: run.id, nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId, processPid: stopped.processPid, providerPid: provider.processId, providerSessionId: provider.providerSessionId, providerTurnId: turnId, stateFingerprint: snapshot.fingerprint, rolloutSha256, completedTaskControlCallIds: completedTaskControlCalls.map(call => call.callId) }; return { evidence, retire: () => idle() && cleanupProcessAbsent(stopped.processPid) && cleanupProcessAbsent(provider.processId) && cleanupStateSnapshot(root).fingerprint === snapshot.fingerprint && nativeSha256(readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven").toString("utf8")) === rolloutSha256 && completeTerminatedLocalNativeSessionCleanup({ companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId! }), }; } catch { return null; } } /** Exact local cleanup only: the accepted result and original quarantine are * never rewritten. A failed/interrupted maintenance attempt is retained for * inspection, not retried from an older snapshot with unknown process owners. */ export async function reconcileRetainedNativeSessionCleanup( db: Db, input: { companyId: string; runId: string; }, ): Promise<{ status: "settled" | "not_eligible" | "operator_required"; runId: string; }> { const denied = () => new Error("native_cleanup_maintenance_unproven"); const leaseOwner = `native-cleanup:${randomUUID()}`; let reservedScope: string | null = null; const releaseScope = () => { if ( reservedScope && executingRunnerdSessionScopes.get(reservedScope) === leaseOwner ) { executingRunnerdSessionScopes.delete(reservedScope); } }; let claim: { execution: NativeExecutionInput; run: typeof heartbeatRuns.$inferSelect; quarantine: string; root: string; emptyRoot: ReturnType; source: ReturnType; providerHome: ReturnType; sourceArchive: { intent: Record; fromCanonical: boolean; completed: boolean; } | null; copySource: { directory: string; snapshot: ReturnType; requestId: string; } | null; providerPid: number; providerSessionId: string; identity: { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string; turnId: string; itemId: string; }; history: Array>; } | null = null; try { claim = await db.transaction(async (tx) => { const run = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), ), ) .for("update", { noWait: true }) .limit(1) .then((rows) => rows[0]); if ( !run || run.runtimeMode !== "native" || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId || !["succeeded", "failed"].includes(run.status) || !run.finishedAt || !cleanupProcessAbsent(run.processPid) || run.processGroupId !== run.processPid ) return null; const execution = parseNativeExecutionInput( record(run.runnerProfileJson).nativeExecutionInput, ); const failure = record(record(run.resultJson).recoveredExecutionFailure); const errorCode = run.errorCode ?? failure.errorCode; const error = run.error ?? failure.error; if ( !["adapter_failed", "provider_transport_failed"].includes(String(errorCode)) || error !== "provider_transport_failed: runner did not durably suspend before checkpoint" || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId || execution.binding.runId !== run.id || nativeSessionKey(execution) !== run.nativeSessionId || execution.provider.kind !== "codex" || execution.session.driverKind !== "codex_app_server" || record(run.runnerProfileJson).nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget("local") ) return null; const scope = nativeSessionScopeKey(execution); if ( activeNativeSessions.has(run.id) || executingRunnerdSessionScopes.has(scope) || initializingSessionToolAuthorities.has(scope) || warmNativeSessions.has(scope) ) return null; executingRunnerdSessionScopes.set(scope, leaseOwner); reservedScope = scope; const coordinator = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.issueId, run.nativeIssueId), ), ) .for("update", { noWait: true }) .limit(1) .then((rows) => rows[0]); if ( !coordinator || coordinator.phase !== "committed" || !coordinator.resultId || !coordinator.assessmentId || !coordinator.decisionId || coordinator.nextAttemptAt || (coordinator.leaseOwner && coordinator.leaseExpiresAt && coordinator.leaseExpiresAt > new Date()) || coordinator.recoveryHistory.some( (event) => event.kind === "native_cleanup_runner_epoch", ) ) return null; const result = await tx .select() .from(nativeRunResults) .where( and( eq(nativeRunResults.id, coordinator.resultId), eq(nativeRunResults.runId, run.id), eq(nativeRunResults.companyId, run.companyId), eq(nativeRunResults.issueId, run.nativeIssueId), ), ) .for("share", { noWait: true }) .limit(1) .then((rows) => rows[0]); if (!result || result.schemaStatus !== "accepted") return null; const environment = await tx .select({ id: environmentLeases.id }) .from(environmentLeases) .where( and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), inArray(environmentLeases.status, ["active", "pending_cleanup"]), ), ) .limit(1); if (environment.length) return null; const root = scopedRunnerdStateRoot(execution); const maintenanceHistory = coordinator.recoveryHistory.filter( (event) => event.kind === "native_cleanup_maintenance", ); const archiveHistory = coordinator.recoveryHistory.filter( (event) => event.kind === "native_cleanup_source_archive", ); const parent = resolve(runnerdStateBase(), "quarantine"); if (existsSync(parent) && !isSafeNativeStateDirectory(parent)) return null; const entries = existsSync(parent) ? readdirSync(parent) : []; if (entries.length > 4096) return null; const candidates = entries.filter((name) => name.startsWith(`${basename(root)}.identity_indeterminate.`), ); let emptyRoot: ReturnType = null; let sourceDirectory: string; let quarantine: string; let sourceArchive: NonNullable["sourceArchive"] = null; const rootExists = !!lstatSync(root, { throwIfNoEntry: false }); if (rootExists && !isSafeNativeStateDirectory(root)) return null; if (archiveHistory.length) { const prepared = archiveHistory[0]!; const archived = archiveHistory[1]; if ( maintenanceHistory.length || archiveHistory.length > 2 || prepared.version !== 1 || prepared.phase !== "prepared" || prepared.companyId !== run.companyId || prepared.agentId !== run.agentId || prepared.runId !== run.id || prepared.nativeSessionId !== run.nativeSessionId || prepared.runnerInstanceId !== run.runnerInstanceId || prepared.stateKey !== basename(root) || typeof prepared.requestId !== "string" || !prepared.requestId.startsWith("native-cleanup:") || typeof prepared.archiveName !== "string" || prepared.archiveName.length > 192 || basename(prepared.archiveName) !== prepared.archiveName || !prepared.archiveName.startsWith( `${basename(root)}.identity_indeterminate.cleanup.`, ) || typeof prepared.sourceFingerprint !== "string" || !/^[0-9a-f]{64}$/.test(prepared.sourceFingerprint) || typeof prepared.providerHomeFingerprint !== "string" || !/^[0-9a-f]{64}$/.test(prepared.providerHomeFingerprint) || (archived && (archived.phase !== "archived" || Object.entries(prepared).some( ([key, value]) => key !== "phase" && canonicalJson(archived[key]) !== canonicalJson(value), ))) ) return null; quarantine = resolve(parent, prepared.archiveName); if (rootExists) { if (archived || candidates.length || existsSync(quarantine)) return null; sourceDirectory = root; } else { if (candidates.length !== 1 || candidates[0] !== prepared.archiveName) return null; sourceDirectory = quarantine; } sourceArchive = { intent: prepared, fromCanonical: rootExists, completed: !!archived, }; } else if (rootExists && readdirSync(root).length) { if (maintenanceHistory.length || candidates.length) return null; sourceDirectory = root; quarantine = resolve( parent, `${basename(root)}.identity_indeterminate.cleanup.${randomUUID()}`, ); sourceArchive = { intent: {}, fromCanonical: true, completed: false, }; } else { // A refused successor may create only the directory before admission // fails. Inventory that exact empty inode, never replace another owner. emptyRoot = cleanupCanonicalVacancy(root); if (candidates.length !== 1) return null; quarantine = resolve(parent, candidates[0]!); sourceDirectory = quarantine; } const archiveRootIdentity = sourceArchive ? cleanupArchiveRootIdentity(sourceDirectory) : null; if (sourceArchive) { const scopeEntries = readdirSync(runnerdStateBase()); if ( scopeEntries.length > 4096 || scopeEntries.some((name) => name.startsWith(`${basename(root)}.cleanup-`), ) ) return null; } const source = cleanupStateSnapshot(sourceDirectory); const identity = record(source.control.identity); if ( !durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId || ![ "runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId", ].every( (key) => typeof identity[key] === "string" && identity[key] && source.runner[key] === identity[key], ) || source.runner.lifecycle !== "ready" || source.provider.lifecycle !== "turn_active" || record(source.provider.config).provider !== "codex" || record(source.provider.config).command !== "codex" || record(source.provider.config).cwd !== execution.workspace.cwd || !Array.isArray(source.control.committedEvents) ) return null; const providerEvents = source.control.committedEvents .map((entry) => record(record(record(entry).envelope).payload)) .filter((event) => ["session.started", "session.resumed"].includes( String(event.eventType), ), ); const providerEvent = providerEvents.at(-1); const provider = record(providerEvent?.payload); if ( !providerEvent || !validatePrpEvent(providerEvent).ok || providerEvent.sourceKind !== "runner" || providerEvent.turnId !== identity.turnId || providerEvent.itemId !== identity.itemId || !cleanupProcessAbsent(provider.processId) || typeof provider.providerSessionId !== "string" || source.provider.threadId !== provider.providerSessionId || providerEvent.runId !== run.id || providerEvent.sourceInstanceId !== run.runnerInstanceId || providerEvent.normalizedSessionId !== run.nativeSessionId || typeof providerEvent.sourceEventId !== "string" ) return null; const persisted = await tx .select() .from(heartbeatRunEvents) .where( and( eq(heartbeatRunEvents.runId, run.id), eq(heartbeatRunEvents.companyId, run.companyId), or( and( eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), inArray(heartbeatRunEvents.eventType, [ "session.started", "session.resumed", ]), ), sql`${heartbeatRunEvents.payload}->'prpEvent'->>'sourceKind' = 'control_plane'`, ), ), ) .limit(20); if ( persisted.length >= 20 || !retainedNativeCleanupJournalMatches({ run, execution, accepted: result, control: source.control, providerSessionId: provider.providerSessionId, providerAccountSessionId: typeof provider.providerAccountSessionId === "string" ? provider.providerAccountSessionId : null, persistedEvents: persisted, }) ) return null; let copySource: { directory: string; snapshot: ReturnType; requestId: string; } | null = null; const providerHome = cleanupProviderHomeSnapshot( resolve(sourceDirectory, "codex-home"), true, ); if (sourceArchive) { if (archiveHistory.length) { if ( sourceArchive.intent.sourceFingerprint !== source.fingerprint || sourceArchive.intent.providerHomeFingerprint !== providerHome.fingerprint || canonicalJson(sourceArchive.intent.rootIdentity) !== canonicalJson(archiveRootIdentity) ) return null; } else { sourceArchive.intent = { kind: "native_cleanup_source_archive", version: 1, phase: "prepared", requestId: leaseOwner, companyId: run.companyId, agentId: run.agentId, runId: run.id, nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId, stateKey: basename(root), archiveName: basename(quarantine), rootIdentity: archiveRootIdentity, sourceFingerprint: source.fingerprint, providerHomeFingerprint: providerHome.fingerprint, }; } } if (maintenanceHistory.length) { if ( maintenanceHistory.length !== 2 || typeof maintenanceHistory[0]?.requestId !== "string" ) return null; const priorRequestId = maintenanceHistory[0].requestId; const scopeEntries = readdirSync(runnerdStateBase()); if (scopeEntries.length > 4096) return null; const attemptedNames = scopeEntries.filter((name) => name.startsWith(`${basename(root)}.cleanup-`), ); if (attemptedNames.length !== 1) return null; const directory = resolve(runnerdStateBase(), attemptedNames[0]!); if ( !retainedRunnerdMaintenanceIsIdle(directory) || lstatSync(resolve(directory, CLEANUP_ACTIVATION_FILE), { throwIfNoEntry: false, }) ) return null; const attempted = cleanupStateSnapshot(directory); const receipts = await tx .select() .from(heartbeatRunEvents) .where( and( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), eq(heartbeatRunEvents.eventType, "native.cleanup.event"), eq( heartbeatRunEvents.sourceInstanceId, `${run.runnerInstanceId}:cleanup:${priorRequestId}`, ), ), ) .limit(513); if ( receipts.length > 512 || !verifyRetainedMaintenanceNoLaunch({ companyId: run.companyId, agentId: run.agentId, identity: identity as { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string; turnId: string; itemId: string; }, original: source, attempted, requestId: priorRequestId, requestHistory: coordinator.recoveryHistory, receipts, now: new Date(), }) ) return null; copySource = { directory, snapshot: attempted, requestId: priorRequestId, }; } const history = [ ...coordinator.recoveryHistory, ...(sourceArchive ? archiveHistory.length ? [] : [sourceArchive.intent] : [ { kind: "native_cleanup_maintenance", version: 1, phase: "started", requestId: leaseOwner, sourceFingerprint: copySource?.snapshot.fingerprint ?? source.fingerprint, ...(copySource ? { originalFingerprint: source.fingerprint, copiedFromRequestId: copySource.requestId, copiedFromStagingName: basename(copySource.directory), } : {}), startedAt: new Date().toISOString(), }, ]), ]; await tx .update(nativeRunFinalizations) .set({ leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000), recoveryHistory: history, updatedAt: new Date(), }) .where(eq(nativeRunFinalizations.runId, run.id)); return { execution, run, quarantine, root, emptyRoot, source, providerHome, sourceArchive, copySource, providerPid: provider.processId, providerSessionId: provider.providerSessionId, identity: identity as { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string; turnId: string; itemId: string; }, history, }; }); } catch { releaseScope(); return { status: "not_eligible", runId: input.runId }; } if (!claim) { releaseScope(); return { status: "not_eligible", runId: input.runId }; } const owned = claim; let stagingDirectory: string | null = null; let maintenanceStarted = !owned.sourceArchive; const archiveReference = owned.sourceArchive ? { sourceArchiveRequestId: owned.sourceArchive.intent.requestId } : {}; const assertLease = async () => { const lease = await db .select({ runId: nativeRunFinalizations.runId }) .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.companyId, owned.run.companyId), eq(nativeRunFinalizations.phase, "committed"), eq(nativeRunFinalizations.leaseOwner, leaseOwner), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .limit(1); if ( !lease.length || !reservedScope || executingRunnerdSessionScopes.get(reservedScope) !== leaseOwner || !cleanupProcessAbsent(owned.run.processPid) || !cleanupProcessAbsent(owned.providerPid) ) throw denied(); }; const authorize = async () => { await assertLease(); if ( cleanupStateSnapshot(owned.quarantine).fingerprint !== owned.source.fingerprint || cleanupProviderHomeSnapshot( resolve(owned.quarantine, "codex-home"), false, ).metadataFingerprint !== owned.providerHome.metadataFingerprint || (owned.copySource && (!retainedRunnerdMaintenanceIsIdle(owned.copySource.directory) || cleanupStateSnapshot(owned.copySource.directory).fingerprint !== owned.copySource.snapshot.fingerprint || lstatSync( resolve(owned.copySource.directory, CLEANUP_ACTIVATION_FILE), { throwIfNoEntry: false }, ))) || canonicalJson(cleanupCanonicalVacancy(owned.root)) !== canonicalJson(owned.emptyRoot) ) throw denied(); }; const appendMaintenanceHistory = async (entry: Record) => { const history = await db.transaction(async (tx) => { const current = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.companyId, owned.run.companyId), eq(nativeRunFinalizations.phase, "committed"), eq(nativeRunFinalizations.leaseOwner, leaseOwner), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .for("update") .limit(1) .then((rows) => rows[0]); if (!current || current.leaseOwner !== leaseOwner) throw denied(); const next = [ ...current.recoveryHistory, { ...entry, ...(entry.kind === "native_cleanup_source_archive" ? {} : archiveReference), }, ]; await tx .update(nativeRunFinalizations) .set({ recoveryHistory: next, updatedAt: new Date() }) .where(eq(nativeRunFinalizations.runId, owned.run.id)); return next; }); owned.history = history; }; try { if (owned.sourceArchive) { const archive = owned.sourceArchive; await assertLease(); const sourceDirectory = archive.fromCanonical ? owned.root : owned.quarantine; if ( canonicalJson(cleanupArchiveRootIdentity(sourceDirectory)) !== canonicalJson(archive.intent.rootIdentity) || cleanupStateSnapshot(sourceDirectory).fingerprint !== owned.source.fingerprint || cleanupProviderHomeSnapshot( resolve(sourceDirectory, "codex-home"), true, ).fingerprint !== owned.providerHome.fingerprint || (archive.fromCanonical ? existsSync(owned.quarantine) : existsSync(owned.root)) ) throw denied(); if (archive.fromCanonical) { const parent = dirname(owned.quarantine); mkdirSync(parent, { recursive: true, mode: 0o700 }); if (!isSafeNativeStateDirectory(parent)) throw denied(); // Preserve the original inode and every byte. In particular, do not // invoke ordinary quarantine scrubbing on this evidence-only archive. renameSync(owned.root, owned.quarantine); archive.fromCanonical = false; } const archivedHome = cleanupProviderHomeSnapshot( resolve(owned.quarantine, "codex-home"), true, ); if ( cleanupStateSnapshot(owned.quarantine).fingerprint !== owned.source.fingerprint || archivedHome.fingerprint !== owned.providerHome.fingerprint || canonicalJson(cleanupArchiveRootIdentity(owned.quarantine)) !== canonicalJson(archive.intent.rootIdentity) || existsSync(owned.root) ) throw denied(); owned.providerHome = archivedHome; if (!archive.completed) { await appendMaintenanceHistory({ ...archive.intent, phase: "archived", }); archive.completed = true; } await authorize(); await appendMaintenanceHistory({ kind: "native_cleanup_maintenance", version: 1, phase: "started", requestId: leaseOwner, sourceFingerprint: owned.source.fingerprint, startedAt: new Date().toISOString(), }); maintenanceStarted = true; } await authorize(); const copy = mkdtempSync( resolve(runnerdStateBase(), `${basename(owned.root)}.cleanup-`), ); stagingDirectory = copy; chmodSync(copy, 0o700); for (const folder of ["runner", "control-plane"]) mkdirSync(resolve(copy, folder), { mode: 0o700 }); for (const file of CLEANUP_CANONICAL_FILES) { copyFileSync( resolve(owned.copySource?.directory ?? owned.quarantine, file), resolve(copy, file), constants.COPYFILE_EXCL, ); chmodSync(resolve(copy, file), 0o600); } copyCleanupProviderHome( resolve(owned.quarantine, "codex-home"), resolve(copy, "codex-home"), owned.providerHome, ); rebaseRetainedNativeCleanupProviderHome( resolve(copy, "codex-home"), resolve(owned.root, "codex-home"), owned.providerSessionId, "staging", ); await authorize(); await appendMaintenanceHistory({ kind: "native_cleanup_maintenance", version: 1, phase: "staged", requestId: leaseOwner, sourceFingerprint: owned.copySource?.snapshot.fingerprint ?? owned.source.fingerprint, stagingName: basename(copy), providerHomeFingerprint: owned.providerHome.fingerprint, providerHomeBytes: owned.providerHome.bytes, stagedProviderHomeFingerprint: cleanupProviderHomeSnapshot( resolve(copy, "codex-home"), true, ).fingerprint, }); const proof = await settleRetainedRunnerdSession({ requestId: leaseOwner, binding: { companyId: owned.run.companyId, issueId: owned.run.nativeIssueId!, agentId: owned.run.agentId, runId: owned.run.id, sessionId: owned.run.nativeSessionId!, }, identity: owned.identity, backend: { kind: "runner", name: "codex_app_server" }, stateDirectory: copy, activationDirectory: owned.root, sourceFingerprint: owned.copySource?.snapshot.fingerprint ?? owned.source.fingerprint, providerSessionId: owned.providerSessionId, originalRunnerPid: owned.run.processPid!, originalProviderPid: owned.providerPid, // The control-only provider resume still needs normal host discovery // and auth-file lookup. Inherit only the existing host allowlist. environment: buildNativeProviderEnvironment( {}, process.env, owned.execution.workspace.cwd ?? undefined, ), authorize, recordEpoch: async (receipt) => { if (receipt.requestId !== leaseOwner || receipt.stateDirectory !== copy) throw denied(); await appendMaintenanceHistory({ kind: "native_cleanup_runner_epoch", version: 1, ...receipt, }); }, appendEvent: async (event) => { await appendRetainedNativeCleanupEvent(db, { companyId: owned.run.companyId, agentId: owned.run.agentId, runId: owned.run.id, nativeSessionId: owned.run.nativeSessionId!, runnerInstanceId: owned.run.runnerInstanceId!, requestId: leaseOwner, event, }); }, }); // Commit intent before the filesystem handoff. A crash can then be // distinguished from an unattempted quarantine; never replay its source. if ( cleanupProviderHomeSnapshot(resolve(owned.quarantine, "codex-home"), true) .fingerprint !== owned.providerHome.fingerprint ) throw denied(); rebaseRetainedNativeCleanupProviderHome( resolve(copy, "codex-home"), resolve(owned.root, "codex-home"), owned.providerSessionId, "canonical", ); const settledHome = cleanupProviderHomeSnapshot( resolve(copy, "codex-home"), true, ); const emptyRootArchive = owned.emptyRoot ? `${basename(owned.root)}.empty-before-cleanup.${leaseOwner}` : null; const preparedHistory = [ ...owned.history, { kind: "native_cleanup_maintenance", version: 1, phase: "activation_prepared", ...archiveReference, requestId: leaseOwner, sourceFingerprint: proof.sourceFingerprint, settledFingerprint: proof.settledFingerprint, settledProviderHomeFingerprint: settledHome.fingerprint, stagingName: basename(copy), ...(emptyRootArchive ? { emptyRootArchive, emptyRoot: owned.emptyRoot } : {}), nativeSessionId: owned.run.nativeSessionId, runnerInstanceId: owned.run.runnerInstanceId, providerSessionId: owned.providerSessionId, }, ]; writeFileSync( resolve(copy, CLEANUP_ACTIVATION_FILE), JSON.stringify({ schema: "paperclip.native_cleanup_activation.v1", companyId: owned.run.companyId, issueId: owned.run.nativeIssueId, runId: owned.run.id, requestId: leaseOwner, sourceFingerprint: proof.sourceFingerprint, settledFingerprint: proof.settledFingerprint, settledProviderHomeFingerprint: settledHome.fingerprint, }), { flag: "wx", mode: 0o600 }, ); const prepared = await db .update(nativeRunFinalizations) .set({ recoveryHistory: preparedHistory, updatedAt: new Date() }) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.leaseOwner, leaseOwner), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .returning({ runId: nativeRunFinalizations.runId }); if (!prepared.length) throw denied(); owned.history = preparedHistory; await db.transaction(async (tx) => { const current = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.leaseOwner, leaseOwner), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .for("update") .limit(1) .then((rows) => rows[0]); if (!current || current.phase !== "committed") throw denied(); await authorize(); if ( cleanupProviderHomeSnapshot(resolve(copy, "codex-home"), false) .metadataFingerprint !== settledHome.metadataFingerprint ) throw denied(); if (emptyRootArchive) { const archive = resolve(runnerdStateBase(), emptyRootArchive); if (lstatSync(archive, { throwIfNoEntry: false })) throw denied(); // Preserve even an empty predecessor directory as evidence. The // scope reservation and durable prepared intent cover this handoff. renameSync(owned.root, archive); } renameSync(copy, owned.root); await tx .update(nativeRunFinalizations) .set({ leaseOwner: null, leaseExpiresAt: null, recoveryHistory: [ ...owned.history, { kind: "native_cleanup_maintenance", version: 1, phase: "settled", ...archiveReference, requestId: leaseOwner, sourceFingerprint: proof.sourceFingerprint, settledFingerprint: proof.settledFingerprint, settledProviderHomeFingerprint: settledHome.fingerprint, nativeSessionId: owned.run.nativeSessionId, runnerInstanceId: owned.run.runnerInstanceId, providerSessionId: owned.providerSessionId, settledAt: new Date().toISOString(), }, ], updatedAt: new Date(), }) .where(eq(nativeRunFinalizations.runId, owned.run.id)); }); if ( cleanupProviderHomeSnapshot(resolve(owned.root, "codex-home"), true) .fingerprint !== settledHome.fingerprint ) throw denied(); completeRetainedNativeSessionCleanup(proof); rmSync(resolve(owned.root, CLEANUP_ACTIVATION_FILE)); return { status: "settled", runId: input.runId }; } catch { if (stagingDirectory && existsSync(stagingDirectory)) { // Keep the failed journal, not transient copied provider credentials. try { scrubRunnerdQuarantineLaunchState(stagingDirectory); } catch { /* retain fail-closed ownership */ } } await db.transaction(async (tx) => { const current = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.leaseOwner, leaseOwner), ), ) .for("update") .limit(1) .then((rows) => rows[0]); if (!current || current.leaseOwner !== leaseOwner) return; await tx .update(nativeRunFinalizations) .set({ leaseOwner: null, leaseExpiresAt: null, // A timed-out epoch callback may have committed before this lock. // Preserve its evidence rather than replacing it from a stale copy. recoveryHistory: [ ...current.recoveryHistory, { kind: maintenanceStarted ? "native_cleanup_maintenance" : "native_cleanup_source_archive", version: 1, phase: "operator_required", requestId: leaseOwner, ...archiveReference, code: "native_cleanup_maintenance_unproven", }, ], updatedAt: new Date(), }) .where( and( eq(nativeRunFinalizations.runId, owned.run.id), eq(nativeRunFinalizations.leaseOwner, leaseOwner), ), ); }); return { status: "operator_required", runId: input.runId }; } finally { releaseScope(); } } /** Read-only scoped admission fence; never grants provider or recovery authority. */ export async function assertRetainedNativeSourceArchiveSettled( db: Db, input: { companyId: string; issueId: string; stateKey: string }, ): Promise { if (!/^[a-f0-9]{64}$/.test(input.stateKey)) throw new NativeSessionCleanupQuarantinedError(); const owners = await db .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId), sql`exists ( select 1 from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory}) prepared where prepared->>'kind' = 'native_cleanup_source_archive' and prepared->>'phase' = 'prepared' and jsonb_typeof(prepared->'stateKey') = 'string' and prepared->>'stateKey' = ${input.stateKey} and not coalesce(${nativeRunFinalizations.phase} = 'committed' and (select count(*) from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory}) duplicate where duplicate->>'kind' = 'native_cleanup_source_archive' and duplicate->>'phase' = 'prepared' and duplicate->'stateKey' = prepared->'stateKey') = 1 and prepared->'version' = '1'::jsonb and jsonb_typeof(prepared->'requestId') = 'string' and prepared->>'requestId' like 'native-cleanup:%' and jsonb_typeof(prepared->'sourceFingerprint') = 'string' and prepared->>'sourceFingerprint' ~ '^[a-f0-9]{64}$' and ( select settled.value->>'phase' = 'settled' and jsonb_typeof(settled.value->'sourceArchiveRequestId') = 'string' and jsonb_typeof(settled.value->'sourceFingerprint') = 'string' and settled.value->>'sourceFingerprint' = prepared->>'sourceFingerprint' from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory}) with ordinality as settled(value, position) where settled.value->>'kind' = 'native_cleanup_maintenance' and settled.value->>'sourceArchiveRequestId' = prepared->>'requestId' order by settled.position desc limit 1 ), false) )`, ), ) .limit(1); for (const owner of owners) { const history = Array.isArray(owner.recoveryHistory) ? owner.recoveryHistory : []; const intents = history.filter( (entry) => entry.kind === "native_cleanup_source_archive" && entry.phase === "prepared" && entry.stateKey === input.stateKey, ); if (!intents.length) continue; const intent = intents[0]!; const settlement = history.findLast( (entry) => entry.kind === "native_cleanup_maintenance" && entry.sourceArchiveRequestId === intent.requestId, ); if ( intents.length !== 1 || owner.phase !== "committed" || intent.version !== 1 || typeof intent.requestId !== "string" || !intent.requestId.startsWith("native-cleanup:") || typeof intent.sourceFingerprint !== "string" || !/^[a-f0-9]{64}$/.test(intent.sourceFingerprint) || settlement?.phase !== "settled" || settlement.sourceFingerprint !== intent.sourceFingerprint ) throw new NativeSessionCleanupQuarantinedError(); } } async function assertCleanupActivationCommitted( db: Db, root: string, execution: NativeExecutionInput, ): Promise { const path = resolve(root, CLEANUP_ACTIVATION_FILE); if (!lstatSync(path, { throwIfNoEntry: false })) return; const marker = record( JSON.parse( readBoundedNativeFile( path, 4096, "native_cleanup_maintenance_unproven", ).toString("utf8"), ), ); if ( marker.schema !== "paperclip.native_cleanup_activation.v1" || marker.companyId !== execution.binding.companyId || marker.issueId !== execution.binding.issueId || typeof marker.runId !== "string" || typeof marker.requestId !== "string" ) { throw new NativeSessionCleanupQuarantinedError(); } const coordinator = await db .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, marker.runId), eq(nativeRunFinalizations.companyId, execution.binding.companyId), eq(nativeRunFinalizations.issueId, execution.binding.issueId), ), ) .limit(1) .then((rows) => rows[0]); const receipt = coordinator?.recoveryHistory.findLast( (entry) => entry.kind === "native_cleanup_maintenance" && entry.requestId === marker.requestId, ); const snapshot = cleanupStateSnapshot(root); if ( marker.settledProviderHomeFingerprint !== undefined || receipt?.settledProviderHomeFingerprint !== undefined ) { if ( typeof marker.settledProviderHomeFingerprint !== "string" || !/^[0-9a-f]{64}$/.test(marker.settledProviderHomeFingerprint) || receipt?.settledProviderHomeFingerprint !== marker.settledProviderHomeFingerprint || cleanupProviderHomeSnapshot(resolve(root, "codex-home"), true) .fingerprint !== marker.settledProviderHomeFingerprint ) throw new NativeSessionCleanupQuarantinedError(); } if ( coordinator?.phase !== "committed" || receipt?.phase !== "settled" || receipt.sourceFingerprint !== marker.sourceFingerprint || receipt.settledFingerprint !== marker.settledFingerprint || snapshot.fingerprint !== marker.settledFingerprint || snapshot.runner.runId !== marker.runId || snapshot.runner.lifecycle !== "suspended" || snapshot.provider.lifecycle !== "prepared" || !Array.isArray(snapshot.control.committedEvents) ) throw new NativeSessionCleanupQuarantinedError(); const owners = snapshot.control.committedEvents .map((entry) => record(record(record(entry).envelope).payload)) .filter((event) => ["session.started", "session.resumed"].includes(String(event.eventType)), ) .map((event) => record(event.payload).processId); if (!owners.length || !owners.every(cleanupProcessAbsent)) throw new NativeSessionCleanupQuarantinedError(); // A committed receipt survived a crash after rename. Normal admission now // applies its existing exact old-owner/session fences; no process is started here. rmSync(path); } /** Read-only admission evidence for an explicit retry of a terminal failed * run. Never migrate/archive state, release an owner, or contact a provider. * Normal executor admission independently verifies the state again. */ export function nativeFailedRunRetryStateIsSafe(input: { execution: unknown; companyId: string; issueId: string; agentId: string; runId: string; nativeSessionId: string; runnerInstanceId: string; providerSessionId: string | null; providerBackendSessionId: string | null; processPid: number | null; processGroupId: number | null; recoveryMode: "bootstrap_retry" | "exact_checkpoint_resume"; allowVerifiedBackup: boolean; }): boolean { try { const execution = parseNativeExecutionInput(input.execution); if ( execution.binding.companyId !== input.companyId || execution.binding.issueId !== input.issueId || execution.binding.agentId !== input.agentId || execution.binding.runId !== input.runId || nativeSessionKey(execution) !== input.nativeSessionId || !input.runnerInstanceId ) return false; const scope = nativeSessionScopeKey(execution); if ( executingRunnerdSessionScopes.has(scope) || initializingSessionToolAuthorities.has(scope) || warmNativeSessions.has(scope) ) return false; for (const [id, group] of [ [input.processPid, false], [input.processGroupId, true], ] as const) { if (id === null) continue; if (!Number.isSafeInteger(id) || id <= 0) return false; try { process.kill(group ? -id : id, 0); return false; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") return false; } } const root = scopedRunnerdStateRoot(execution); if (!lstatSync(root, { throwIfNoEntry: false })) { if ( input.recoveryMode !== "bootstrap_retry" || lstatSync(legacyRunnerdStateRoot(execution), { throwIfNoEntry: false, }) || lstatSync(legacyCompanyRunnerdStateRoot(execution), { throwIfNoEntry: false, }) ) return false; // A missing root after quarantine is not proof of a clean bootstrap. const quarantine = resolve(runnerdStateBase(), "quarantine"); if (lstatSync(quarantine, { throwIfNoEntry: false })) { if (!isSafeNativeStateDirectory(quarantine)) return false; const directory = opendirSync(quarantine); try { const prefixes = [ root, legacyRunnerdStateRoot(execution), legacyCompanyRunnerdStateRoot(execution), ].map((path) => `${basename(path)}.`); for (let count = 0; ; count++) { const entry = directory.readSync(); if (!entry) break; if ( count >= 4096 || prefixes.some((prefix) => entry.name.startsWith(prefix)) ) return false; } } finally { directory.closeSync(); } } return true; } const identity = readRunnerdDurableIdentity(root); if ( !durableIdentityMatchesSession(identity, execution) || !durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== input.runnerInstanceId ) return false; let stateRoot = root; const direct = runnerdAuthorityLifecycle(root, identity); if (direct === "absent" && input.allowVerifiedBackup) { const backup = verifyNativeHarnessBackup({ root, execution, runnerInstanceId: input.runnerInstanceId, }); if (!backup) return false; stateRoot = backup.root; } if (runnerdAuthorityLifecycle(stateRoot, identity) !== "suspended") return false; const runnerState = record( JSON.parse( readBoundedNativeFile( resolve(stateRoot, "runner", "runner-state.json"), NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large", ).toString("utf8"), ), ); if ( runnerState.schema !== RUNNERD_STATE_SCHEMA || runnerState.lifecycle !== "suspended" || runnerState.runId !== identity.runId || runnerState.runnerInstanceId !== identity.runnerInstanceId || runnerState.normalizedSessionId !== identity.normalizedSessionId || runnerState.environmentLeaseId !== identity.environmentLeaseId || !Array.isArray(runnerState.outbox) || runnerState.outbox.length !== 0 ) return false; const providerFile = resolve( stateRoot, "runner", runnerProviderStateFilename(execution), ); if (input.recoveryMode === "bootstrap_retry") return false; const providerState = record( JSON.parse( readBoundedNativeFile( providerFile, NATIVE_RUNNER_STATE_MAX_BYTES, "runner_provider_state_too_large", ).toString("utf8"), ), ); if ( !Array.isArray(providerState.pendingEvents) || providerState.pendingEvents.length !== 0 || !Array.isArray(providerState.queuedEvents) || providerState.queuedEvents.length !== 0 || Object.keys(record(record(providerState.toolBridge).pending)).length !== 0 || providerState.activeProviderResultFingerprint != null ) return false; const providerIdentity = providerSessionIdentityFromDurableProviderState({ execution, providerState, }); return ( !!input.providerSessionId && providerIdentity.providerSessionId === input.providerSessionId && providerIdentity.providerBackendSessionId === input.providerBackendSessionId ); } catch { return false; } } /** Physical half of retrying a request rejected before provider admission. * The caller separately proves the failed receipt/coordinator has no native * events or result and selects exactly one committed cleanup owner. */ export function nativePreProviderRetryAfterCleanupStateIsSafe(input: { failedExecution: unknown; retiredExecution: unknown; companyId: string; issueId: string; agentId: string; failedRunId: string; retiredRunId: string; nativeSessionId: string; runnerInstanceId: string; providerSessionId: string; providerBackendSessionId: string | null; processPid: number; processGroupId: number; receipt: Record; }): boolean { try { const failed = parseNativeExecutionInput(input.failedExecution); const retired = parseNativeExecutionInput(input.retiredExecution); if ( input.failedRunId === input.retiredRunId || failed.binding.runId !== input.failedRunId || retired.binding.runId !== input.retiredRunId || failed.binding.companyId !== input.companyId || failed.binding.issueId !== input.issueId || failed.binding.agentId !== input.agentId || nativeSessionScopeKey(failed) !== nativeSessionScopeKey(retired) || input.receipt.kind !== "native_cleanup_maintenance" || input.receipt.version !== 1 || input.receipt.phase !== "settled" || input.receipt.nativeSessionId !== input.nativeSessionId || input.receipt.runnerInstanceId !== input.runnerInstanceId || input.receipt.providerSessionId !== input.providerSessionId || typeof input.receipt.requestId !== "string" || input.receipt.requestId.length === 0 || typeof input.receipt.sourceFingerprint !== "string" || !/^[a-f0-9]{64}$/.test(input.receipt.sourceFingerprint) || typeof input.receipt.settledFingerprint !== "string" || !/^[a-f0-9]{64}$/.test(input.receipt.settledFingerprint) || !nativeFailedRunRetryStateIsSafe({ execution: retired, companyId: input.companyId, issueId: input.issueId, agentId: input.agentId, runId: input.retiredRunId, nativeSessionId: input.nativeSessionId, runnerInstanceId: input.runnerInstanceId, providerSessionId: input.providerSessionId, processPid: input.processPid, processGroupId: input.processGroupId, providerBackendSessionId: input.providerBackendSessionId, recoveryMode: "exact_checkpoint_resume", allowVerifiedBackup: false, }) ) return false; const root = scopedRunnerdStateRoot(retired); const markerPath = resolve(root, CLEANUP_ACTIVATION_FILE); if (lstatSync(markerPath, { throwIfNoEntry: false })) { // A crash after receipt commit may leave the activation marker. This // read-only proof accepts only that exact committed handoff; normal // executor admission independently reconciles the marker before launch. const marker = record( JSON.parse( readBoundedNativeFile( markerPath, 4096, "native_cleanup_maintenance_unproven", ).toString("utf8"), ), ); if ( marker.schema !== "paperclip.native_cleanup_activation.v1" || marker.companyId !== input.companyId || marker.issueId !== input.issueId || marker.runId !== input.retiredRunId || marker.requestId !== input.receipt.requestId || marker.sourceFingerprint !== input.receipt.sourceFingerprint || marker.settledFingerprint !== input.receipt.settledFingerprint ) return false; } return ( cleanupStateSnapshot(root).fingerprint === input.receipt.settledFingerprint ); } catch { return false; } } export async function verifyPriorRunnerdStateForSessionScope(input: { db: Db; root: string; identity: RunnerdDurableIdentity; execution: NativeExecutionInput; allowVerifiedBackup: boolean; allowRetainedWarmRunner: boolean; remoteRunnerState?: boolean; runnerExecutionTarget?: AdapterExecutionTarget | null; }): Promise { let priorRun: { status: string; runnerProfileJson: unknown; } | null; try { priorRun = await input.db .select({ status: heartbeatRuns.status, runnerProfileJson: heartbeatRuns.runnerProfileJson, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, input.identity.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), ), ) .limit(1) .then((rows) => rows[0] ?? null); } catch { return "unavailable"; } if (!priorRun) return "authority_indeterminate"; if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(priorRun.status)) { return "active"; } const persistedInput = record( priorRun.runnerProfileJson, ).nativeExecutionInput; if (persistedInput === undefined) return "authority_indeterminate"; try { const priorExecution = parseNativeExecutionInput(persistedInput); const sameScope = priorExecution.binding.runId === input.identity.runId && nativeSessionScopeKey(priorExecution) === nativeSessionScopeKey(input.execution); if (!sameScope) return "scope_mismatch"; const lifecycle = runnerdAuthorityLifecycleWithVerifiedBackup(input); if (lifecycle === "suspended") return "verified"; const directLifecycle = runnerdAuthorityLifecycle( input.root, input.identity, ); if ( input.allowRetainedWarmRunner && (lifecycle === "not_suspended" || // A remote runner keeps runner-state.json in the sandbox rather than // beside the controller's PRP journal. During a genuinely warm handoff // there is intentionally no suspended failover backup yet. The exact // idle in-memory session owner is the authority for this one case; // after a restart that owner is absent and this remains fail-closed. ((input.allowVerifiedBackup || input.remoteRunnerState) && directLifecycle === "absent")) ) { return "retained_warm_runner"; } const target = input.runnerExecutionTarget; if (input.remoteRunnerState && directLifecycle === "absent" && target?.kind === "remote" && target.transport === "computer" && target.environmentId) { const descriptor = record(priorRun.runnerProfileJson).nativeComputerWorkspace; const priorWorkspace = readNativeComputerWorkspaceReference(descriptor); if (!priorWorkspace || priorWorkspace.computerOwner.computerId !== target.resourceAuthority.computerId || priorWorkspace.remoteCwd !== target.remoteCwd || record(descriptor).placementId !== target.fileAuthority.placementId) return "scope_mismatch"; // Persistent runners keep their state remotely. Once the exact prior // owner has completed retirement, its terminal run and fenced process // are authoritative even though no local runner-state.json exists. // This read cannot retire a live owner or accept a newer generation. try { // PRP historically calls the immutable execution-workspace coordinate // environmentLeaseId. Resolve its recorded placement lease separately // from the latest process owner: workspace identity survives warm turns // and idle replacement. Explicit physical leases retain their own lineage. let workspaceRunId: string | null = input.identity.environmentLeaseId === priorExecution.binding.executionWorkspaceId ? input.identity.runId : null; let leaseWorkspace = priorWorkspace; let physicalLeaseId = workspaceRunId ? priorWorkspace.leaseId : input.identity.environmentLeaseId; if (!workspaceRunId) { // Warm attachment advances runId while retaining the process's original // workspace coordinate. Resolve that recorded origin, never an arbitrary // run ID, before checking the physical lease and latest retirement. const origins = await input.db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, runnerProfileJson: heartbeatRuns.runnerProfileJson, }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), eq(sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->>'executionWorkspaceId'`, input.identity.environmentLeaseId), )).limit(2); if (origins.length > 1) return "scope_mismatch"; if (origins.length === 1) { const origin = origins[0]!; const originProfile = record(origin.runnerProfileJson); const originExecution = parseNativeExecutionInput(originProfile.nativeExecutionInput); const originWorkspace = readNativeComputerWorkspaceReference(originProfile.nativeComputerWorkspace); if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(origin.status) || originExecution.binding.runId !== origin.id || originExecution.binding.executionWorkspaceId !== input.identity.environmentLeaseId || nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(priorExecution) || !originWorkspace || originWorkspace.remoteCwd !== priorWorkspace.remoteCwd || record(originProfile.nativeComputerWorkspace).placementId !== record(descriptor).placementId || originWorkspace.computerOwner.computerId !== priorWorkspace.computerOwner.computerId || (originWorkspace.computerOwner.ownerId === priorWorkspace.computerOwner.ownerId && originWorkspace.computerOwner.generation > priorWorkspace.computerOwner.generation)) return "scope_mismatch"; // This coordinate outlives physical runner replacements. Its lease // proves the recorded workspace origin, not ownership of the latest // process. The latest exact generation is verified independently below. workspaceRunId = originExecution.binding.runId; leaseWorkspace = originWorkspace; physicalLeaseId = originWorkspace.leaseId; } } const [physicalLease] = await input.db.select({ providerLeaseId: environmentLeases.providerLeaseId, heartbeatRunId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata, }).from(environmentLeases).where(and( eq(environmentLeases.id, physicalLeaseId), eq(environmentLeases.companyId, input.execution.binding.companyId), eq(environmentLeases.environmentId, target.environmentId), )).limit(1); const leaseMetadata = record(physicalLease?.metadata); const leaseOwner = record(leaseMetadata.computerOwner); if ((workspaceRunId && physicalLease?.heartbeatRunId !== workspaceRunId) || physicalLease?.providerLeaseId !== leaseWorkspace.computerOwner.ownerId || leaseMetadata.agentId !== input.execution.binding.agentId || leaseOwner.computerId !== leaseWorkspace.computerOwner.computerId || leaseOwner.ownerId !== leaseWorkspace.computerOwner.ownerId || !Number.isSafeInteger(leaseOwner.generation) || Number(leaseOwner.generation) < 1 || (workspaceRunId ? Number(leaseOwner.generation) !== leaseWorkspace.computerOwner.generation : Number(leaseOwner.generation) > leaseWorkspace.computerOwner.generation)) return "scope_mismatch"; if (await computerService(input.db).isRetired({ companyId: input.execution.binding.companyId, environmentId: target.environmentId, agentId: input.execution.binding.agentId, runId: input.identity.runId, owner: priorWorkspace.computerOwner, })) return "verified"; } catch { return "unavailable"; } } return "terminal_state_indeterminate"; } catch { return "authority_indeterminate"; } } function hasRetainedWarmTransitionEvidence(root: string): boolean { for (const [directory, filename, maximum] of [ [ "control-plane", "control-plane-state.json", NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, ], ["runner", "runner-state.json", NATIVE_RUNNER_STATE_MAX_BYTES], ] as const) { const path = resolve(root, directory, filename); if (!lstatSync(path, { throwIfNoEntry: false })) continue; let bytes: string; try { bytes = readBoundedNativeFile( path, maximum, "runner_state_too_large", ).toString("utf8"); } catch { // The ordinary verifier still owns unreadable legacy state. Inspect the // other file before deciding whether this is a forward-protocol fence. continue; } try { const state = record(JSON.parse(bytes)); if ( Object.prototype.hasOwnProperty.call(state, "warmTransition") || state.schema === "paperclip.runner.durable.control-plane-state.warm-transition.v1" || state.schema === "paperclip.runner.durable.state.warm-transition.v1" ) return true; } catch { // This is detection only, never admission. A damaged forward receipt // must remain available to its owner rather than become legacy state. if ( bytes.includes("warm-transition.v1") || bytes.includes('"warmTransition"') ) return true; } } return false; } type VerifiedWarmTransitionBinding = { runnerInstanceId: string; environmentLeaseId: string; transitionId: string; stateFingerprint: string; }; function readWarmTransitionSnapshot(root: string) { if ( ![root, resolve(root, "control-plane"), resolve(root, "runner")].every( isSafeNativeStateDirectory, ) ) { throw new Error("native_runner_warm_transition_recovery_unproven"); } const core = readBoundedNativeFile( resolve(root, "control-plane", "control-plane-state.json"), NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven", ); const runner = readBoundedNativeFile( resolve(root, "runner", "runner-state.json"), NATIVE_RUNNER_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven", ); try { return { controlPlaneState: JSON.parse(core.toString("utf8")) as unknown, runnerState: JSON.parse(runner.toString("utf8")) as unknown, stateFingerprint: nativeSha256([ core.toString("base64"), runner.toString("base64"), ]), }; } catch { throw new Error("native_runner_warm_transition_recovery_unproven"); } } /** Forward receipts select a protocol boundary, never a process owner. */ async function verifyWarmTransitionRestart(input: { db: Db; execution: NativeExecutionInput; restartRecovery?: NativeRestartRecoveryClaim; runnerExecutionTarget?: AdapterExecutionTarget | null; }): Promise { const deny = (): never => { throw new Error("native_runner_warm_transition_recovery_unproven"); }; const claim = input.restartRecovery; // A surviving runner needs its own authenticated reattach path. Do not // reinterpret that claim, remote evidence, or an incomplete bootstrap as // permission to launch another process against a retained transition. if ( claim?.kind !== "resume_dead_runner" || input.runnerExecutionTarget?.kind === "remote" || input.execution.provider.kind !== "codex" || input.execution.session.driverKind !== "codex_app_server" || claim.runId !== input.execution.binding.runId ) return deny(); const root = scopedRunnerdStateRoot(input.execution); const snapshot = readWarmTransitionSnapshot(root); const artifact = readRunnerdArtifactBinding(resolvePaperclipRunnerBinary()); const controller = await currentNativeControllerIdentity(); const binding = input.execution.binding; let finalAuthorityCheck: (() => boolean) | undefined; const verified = await input.db.transaction(async (tx) => { const current = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, binding.runId), eq(heartbeatRuns.companyId, binding.companyId), ), ) .for("update") .limit(1) .then((rows) => rows[0]); const coordinator = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, binding.runId), eq(nativeRunFinalizations.companyId, binding.companyId), ), ) .for("update") .limit(1) .then((rows) => rows[0]); const issue = await tx .select() .from(issues) .where( and( eq(issues.id, binding.issueId), eq(issues.companyId, binding.companyId), ), ) .for("update") .limit(1) .then((rows) => rows[0]); const now = new Date(); if ( !current || !coordinator || !issue || issue.executionRunId !== binding.runId || current.agentId !== binding.agentId || current.nativeIssueId !== binding.issueId || current.runtimeMode !== "native" || current.status !== "running" || current.finishedAt !== null || current.nativeSessionId !== nativeSessionKey(input.execution) || !current.runnerInstanceId || isNativeRunnerOwnershipHeld(current) || coordinator.issueId !== binding.issueId || coordinator.phase !== "observed" || coordinator.resultId !== null || coordinator.leaseOwner !== claim.leaseOwner || !coordinator.leaseExpiresAt || coordinator.leaseExpiresAt <= now || coordinator.controllerBootId !== controller.bootId || coordinator.controllerPid !== controller.pid || coordinator.controllerProcessStartedAt?.getTime() !== controller.processStartedAt.getTime() || coordinator.controllerGeneration !== claim.controllerGeneration ) return deny(); const profile = record(current.runnerProfileJson); let frozen: NativeExecutionInput; try { frozen = parseNativeExecutionInput(profile.nativeExecutionInput); } catch { return deny(); } if (nativeSha256(frozen) !== nativeSha256(input.execution)) return deny(); const cancellation = record(record(current.resultJson).nativeCancellation); if ( cancellation.scope === "run" && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)) ) return deny(); let expectedEnvironmentLeaseId = binding.executionWorkspaceId; if (binding.executionWorkspaceId === binding.runId) { // Projectless runs retain the first run's lease while their workspace // placeholder changes. The receipt only SELECTS that origin: its frozen // DB execution, full scope and terminal ownership independently prove it. const selectedLease = record( record(record(snapshot.runnerState).warmTransition).receipt, ).newIdentity; const originId = record(selectedLease).environmentLeaseId; if ( typeof originId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( originId, ) ) return deny(); const origin = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, originId), eq(heartbeatRuns.companyId, binding.companyId), ), ) .for("update") .limit(1) .then((rows) => rows[0]); if ( !origin || origin.agentId !== binding.agentId || origin.nativeIssueId !== binding.issueId || origin.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "timed_out"].includes( origin.status, ) || origin.finishedAt === null || isNativeRunnerOwnershipHeld(origin) || origin.runnerInstanceId !== current.runnerInstanceId || origin.nativeSessionId !== current.nativeSessionId ) return deny(); let originExecution: NativeExecutionInput; try { originExecution = parseNativeExecutionInput( record(origin.runnerProfileJson).nativeExecutionInput, ); } catch { return deny(); } if ( originExecution.binding.runId !== origin.id || originExecution.binding.executionWorkspaceId !== origin.id || originExecution.binding.companyId !== binding.companyId || originExecution.binding.agentId !== binding.agentId || originExecution.binding.issueId !== binding.issueId || nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(input.execution) ) return deny(); expectedEnvironmentLeaseId = origin.id; } const inspectionInput = { ...snapshot, expectedNewIdentity: { runnerInstanceId: current.runnerInstanceId, environmentLeaseId: expectedEnvironmentLeaseId, runId: binding.runId, normalizedSessionId: nativeSessionKey(input.execution), turnId: `turn-${binding.runId}`, itemId: `item-${binding.runId}`, }, expectedRunnerVersion: artifact.version, expectedRunnerDigest: artifact.digest, }; const proof = inspectWarmRunTransition({ ...inspectionInput, now: Date.now(), }); if (!proof) return deny(); const prior = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, proof.receipt.oldIdentity.runId), eq(heartbeatRuns.companyId, binding.companyId), ), ) .for("update") .limit(1) .then((rows) => rows[0]); if ( !prior || prior.agentId !== binding.agentId || prior.nativeIssueId !== binding.issueId || prior.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "timed_out"].includes( prior.status, ) || prior.finishedAt === null || isNativeRunnerOwnershipHeld(prior) || prior.runnerInstanceId !== current.runnerInstanceId || prior.nativeSessionId !== current.nativeSessionId || proof.receipt.oldIdentity.runnerInstanceId !== prior.runnerInstanceId || proof.receipt.oldIdentity.normalizedSessionId !== prior.nativeSessionId || proof.receipt.oldIdentity.environmentLeaseId !== expectedEnvironmentLeaseId || proof.receipt.oldIdentity.turnId !== `turn-${prior.id}` || proof.receipt.oldIdentity.itemId !== `item-${prior.id}` ) return deny(); let previousExecution: NativeExecutionInput; try { previousExecution = parseNativeExecutionInput( record(prior.runnerProfileJson).nativeExecutionInput, ); } catch { return deny(); } if ( previousExecution.binding.runId !== prior.id || previousExecution.binding.companyId !== binding.companyId || previousExecution.binding.agentId !== binding.agentId || previousExecution.binding.issueId !== binding.issueId || nativeSessionScopeKey(previousExecution) !== nativeSessionScopeKey(input.execution) ) return deny(); const history = coordinator.recoveryHistory.at(-1); const checkpoint = record(profile.sessionCheckpoint); const checkpointIdentity = record(checkpoint.identity); const processEvidence = record(checkpoint.process); if ( !history || history.disposition !== claim.kind || history.controllerBootId !== controller.bootId || history.controllerPid !== controller.pid || history.controllerGeneration !== claim.controllerGeneration || history.recoveryRequestId !== claim.recoveryRequestId || history.providerAttempt !== claim.providerAttempt || history.stateRootAction !== "reuse_exact_root" || history.hasCheckpoint !== true || history.checkpointIdentityMatches !== true || history.hasProviderEvidence !== true || checkpointIdentity.runId !== binding.runId || checkpointIdentity.companyId !== binding.companyId || checkpointIdentity.issueId !== binding.issueId || checkpointIdentity.agentId !== binding.agentId || checkpointIdentity.sessionId !== current.nativeSessionId || history.processPid !== prior.processPid || history.processStartedAt !== prior.processStartedAt?.toISOString() || processEvidence.runnerPid !== prior.processPid || processEvidence.runnerProcessGroupId !== prior.processGroupId || prior.processGroupId !== prior.processPid || !cleanupProcessAbsent(prior.processPid) ) return deny(); // Recheck every independently retained provider owner, including process // groups. These are DB/checkpoint facts, not claims in a copied journal. const known = history.knownProviderPids; if ( !Array.isArray(known) || known.length === 0 || known.length > 16 || ![ history.liveProviderPids, history.ambiguousProviderPids, history.recycledProviderPids, ].every((pids) => Array.isArray(pids) && pids.length === 0) ) return deny(); const checkpointPids = new Set(); for (const [pidKey, startedKey] of [ ["providerPid", "providerProcessStartedAt"], ["codexPid", "codexProcessStartedAt"], ["sidecarPid", "sidecarProcessStartedAt"], ["agentPid", "agentProcessStartedAt"], ] as const) { const pid = processEvidence[pidKey]; if (pid === null || pid === undefined) continue; if ( typeof processEvidence[startedKey] !== "string" || !Number.isFinite(Date.parse(processEvidence[startedKey] as string)) || !cleanupProcessAbsent(pid) ) return deny(); checkpointPids.add(pid); } if ( checkpointPids.size === 0 || new Set(known).size !== checkpointPids.size || known.some( (pid) => typeof pid !== "number" || !checkpointPids.has(pid) || !cleanupProcessAbsent(pid), ) ) return deny(); // These fences remain relevant if maintenance evidence appears after // initial construction. Every later registration/launch/auth gate repeats // them; no settled receipt or pending transition bypasses quarantine. await assertRetainedNativeSourceArchiveSettled(tx as unknown as Db, { companyId: binding.companyId, issueId: binding.issueId, stateKey: basename(root), }); await assertCleanupActivationCommitted( tx as unknown as Db, root, input.execution, ); finalAuthorityCheck = () => { if ( readWarmTransitionSnapshot(root).stateFingerprint !== snapshot.stateFingerprint ) return false; const selected = readRunnerdArtifactBinding( resolvePaperclipRunnerBinary(), ); if ( selected.version !== artifact.version || selected.digest !== artifact.digest ) return false; const finalProof = inspectWarmRunTransition({ ...inspectionInput, now: Date.now(), }); return ( coordinator.leaseExpiresAt!.getTime() > Date.now() && finalProof?.receipt.transitionId === proof.receipt.transitionId && finalProof.receipt.leaseExpiresAtUnixMs > Date.now() ); }; if (!finalAuthorityCheck()) return deny(); return { runnerInstanceId: current.runnerInstanceId, environmentLeaseId: expectedEnvironmentLeaseId, transitionId: proof.receipt.transitionId, stateFingerprint: snapshot.stateFingerprint, }; }); // A lock wait or the transaction completion itself can outlive a lease. // Never carry a timestamp sampled before those awaits into admission. if (!finalAuthorityCheck?.()) return deny(); return verified; } async function migrateRunnerdStateRootForExecution(input: { db: Db; execution: NativeExecutionInput; allowVerifiedBackup: boolean; allowRetainedWarmRunner: boolean; allowLocalRecovery: boolean; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; restartRecovery?: NativeRestartRecoveryClaim; runnerExecutionTarget?: AdapterExecutionTarget | null; }): Promise { const scoped = scopedRunnerdStateRoot(input.execution); // A crash can leave a source archive intent before/after its rename. Until // that exact maintenance is settled, absence of a canonical root is not // permission to bootstrap a replacement provider session. await assertRetainedNativeSourceArchiveSettled(input.db, { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, stateKey: basename(scoped), }); if ( input.allowLocalRecovery && !input.allowRetainedWarmRunner && !input.restartRecovery ) { await recoverQuiescentRunnerdState({ ...input, scoped }); } if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) { throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing"); } if (existsSync(scoped)) { if (!isSafeNativeStateDirectory(scoped)) { throw new Error("runner_state_directory_unsafe"); } if (hasRetainedWarmTransitionEvidence(scoped)) { // A copied receipt is not restart/process authority. Keep both valid // unsupported and invalid forward evidence out of destructive legacy // migration until the exact transition admission path proves its owner. const verified = await verifyWarmTransitionRestart(input); return verified; } await assertCleanupActivationCommitted(input.db, scoped, input.execution); const identity = readRunnerdDurableIdentity(scoped); if (!identity) { if (input.restartRecovery?.kind !== "reattach_existing_runner" && input.restartRecovery?.kind !== "reattach_remote_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); } throw new Error("runner_state_identity_mismatch: durable_identity_unreadable"); } if (!durableIdentityMatchesSession(identity, input.execution)) { if (input.restartRecovery?.kind !== "reattach_existing_runner" && input.restartRecovery?.kind !== "reattach_remote_runner") { quarantineRunnerdStateRoot(scoped, "identity_mismatch"); } throw new Error("runner_state_identity_mismatch: session_scope_mismatch"); } if (input.restartRecovery?.kind === "bootstrap_incomplete") { if (runnerdStateProvesIncompleteBootstrap(scoped)) { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); return; } // Database evidence alone cannot distinguish a never-connected runner // from a partially-persisted provider bootstrap. Only the durable PRP // root can authorize a fresh bootstrap; anything else stays fail-closed. throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete"); } if (input.restartRecovery?.kind === "reattach_remote_runner") { await verifyRemoteRunnerReattachment({ claim: input.restartRecovery, target: input.runnerExecutionTarget, identity, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), }); return; } if (durableIdentityMatchesExecution(identity, input.execution)) { if ( runnerdAuthorityLifecycleWithVerifiedBackup({ root: scoped, identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, }) === "indeterminate" ) { if (input.restartRecovery?.kind !== "reattach_existing_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); } throw new Error("runner_state_identity_mismatch: authority_indeterminate"); } } else { const verification = await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: scoped, identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" && verification !== "retained_warm_runner" ) { if (verification !== "active" && verification !== "unavailable") { quarantineRunnerdStateRoot( scoped, verification === "scope_mismatch" ? "identity_mismatch" : "identity_indeterminate", ); } throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`); } } return; } for (const legacy of [ legacyCompanyRunnerdStateRoot(input.execution), legacyRunnerdStateRoot(input.execution), ]) { if (!existsSync(legacy)) continue; const identity = readRunnerdDurableIdentity(legacy); if ( !identity || !durableIdentityMatchesSession(identity, input.execution) ) { // Unlike the full-scope target above, this legacy name can legitimately // belong to another scope. Leave it in place for its owner and fail the // attempted migration visibly. throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch"); } let verifiedPriorRunId: string | undefined; if (!durableIdentityMatchesExecution(identity, input.execution)) { const verification = await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: legacy, identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, allowRetainedWarmRunner: input.allowRetainedWarmRunner, remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer", runnerExecutionTarget: input.runnerExecutionTarget, }); if ( verification !== "verified" && verification !== "retained_warm_runner" ) { if (verification === "terminal_state_indeterminate") { // The database proves this ambiguous legacy path belongs to the same // full session scope and its owner is terminal, so it is now safe to // move aside. Scope mismatches and active/unavailable owners remain // untouched because the legacy name may still belong to them. quarantineRunnerdStateRoot(legacy, "identity_indeterminate"); } throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`); } verifiedPriorRunId = identity.runId; } migrateLegacyRunnerdStateRoot({ legacy, scoped, execution: input.execution, ...(verifiedPriorRunId ? { verifiedPriorRunId } : {}), }); return; } } // A terminal warm run can lose its controller before session.suspend. Older // controllers quarantined even fully settled state in that case. Recover the // exact provider thread, never bootstrap a replacement from an ambiguous root. async function recoverQuiescentRunnerdState(input: { db: Db; execution: NativeExecutionInput; scoped: string; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; }): Promise { if (input.execution.provider.kind !== "codex") return; const quarantine = resolve(runnerdStateBase(), "quarantine"); const scopedExists = existsSync(input.scoped); if (scopedExists && !isSafeNativeStateDirectory(input.scoped)) return; // Never hide corrupt or contradictory current authority behind an older // checkpoint. Only an absent/empty root may recover from quarantine. const currentEmpty = !scopedExists || readdirSync(input.scoped).length === 0; const candidates = currentEmpty ? isSafeNativeStateDirectory(quarantine) ? readdirSync(quarantine) .filter((name) => name.startsWith( `${basename(input.scoped)}.identity_indeterminate.`, ), ) .map((name) => resolve(quarantine, name)) : [] : [input.scoped]; if ( currentEmpty && candidates.filter( (root) => isSafeNativeStateDirectory(root) && readdirSync(root).length > 0, ).length > 1 ) { // Do not roll back to an older valid checkpoint when a newer quarantined // root contains unconfirmed work, even if the newer root is unreadable. throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous"); } const verified: Array<{ root: string; runnerBytes: string; controlBytes: string; providerBytes: string; runner: Record; runId: string; processPid: number; processGroupId: number; }> = []; for (const root of candidates) { if (!isSafeNativeStateDirectory(root)) continue; const identity = readRunnerdDurableIdentity(root); if (!durableIdentityMatchesSession(identity, input.execution)) continue; if (identity.runId === input.execution.binding.runId) continue; try { const readState = (directory: string, name: string) => { if (!isSafeNativeStateDirectory(resolve(root, directory))) throw new Error("unsafe_recovery_state"); return readBoundedNativeFile( resolve(root, directory, name), directory === "control-plane" && name === "control-plane-state.json" ? NATIVE_CONTROL_PLANE_STATE_MAX_BYTES : NATIVE_RUNNER_STATE_MAX_BYTES, "recovery_state_too_large", ).toString("utf8"); }; const runnerBytes = readState("runner", "runner-state.json"); const controlBytes = readState( "control-plane", "control-plane-state.json", ); const providerBytes = readState("runner", "codex-provider-state.json"); const runner = record(JSON.parse(runnerBytes)); const control = record(JSON.parse(controlBytes)); const provider = record(JSON.parse(providerBytes)); const commands = control.commands; const events = control.committedEvents; const terminalEnvelope = record( record(Array.isArray(events) ? events.at(-1) : null).envelope, ); if ( runner.schema !== RUNNERD_STATE_SCHEMA || !["ready", "suspended"].includes(String(runner.lifecycle)) || (!currentEmpty && runner.lifecycle === "suspended") || [ "runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId", ].some((key) => runner[key] !== identity[key]) || !Array.isArray(runner.outbox) || runner.outbox.length !== 0 || runner.pendingTerminalDelivery !== null || !Array.isArray(commands) || commands.some((command) => record(command).status !== "completed") || !Array.isArray(events) || record(events.at(-1)).eventType !== "run.terminal" || terminalEnvelope.runId !== identity.runId || terminalEnvelope.normalizedSessionId !== identity.normalizedSessionId || provider.completedTurnAuthoritative !== true ) continue; const providerIdentity = providerSessionIdentityFromDurableProviderState({ execution: input.execution, providerState: provider, }); if (!providerIdentity.providerSessionId) continue; const prior = await input.db .select({ status: heartbeatRuns.status, runnerProfileJson: heartbeatRuns.runnerProfileJson, processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId, contextSnapshot: heartbeatRuns.contextSnapshot, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, identity.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), ), ) .limit(1) .then((rows) => rows[0] ?? null); if (!prior || !TERMINAL_HEARTBEAT_RUN_STATUSES.has(prior.status)) continue; const profile = record(prior.runnerProfileJson); const previous = parseNativeExecutionInput(profile.nativeExecutionInput); const checkpoint = record(profile.sessionCheckpoint); const checkpointIdentity = record(checkpoint.identity); const current = await input.db .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), ), ) .limit(1) .then((rows) => rows[0] ?? null); const latestCheckpoint = record( record(current?.runnerProfileJson).sessionCheckpoint, ); const latestIdentity = record(latestCheckpoint.identity); if ( previous.binding.runId !== identity.runId || previous.binding.issueId !== input.execution.binding.issueId || nativeSessionScopeKey(previous) !== nativeSessionScopeKey(input.execution) || nativeSessionConfigDigest(previous) !== nativeSessionConfigDigest(input.execution) || record(record(prior.contextSnapshot).paperclipEnvironment).driver !== "local" || checkpointIdentity.runId !== identity.runId || checkpointIdentity.sessionId !== identity.normalizedSessionId || checkpointIdentity.companyId !== previous.binding.companyId || checkpointIdentity.agentId !== previous.binding.agentId || checkpointIdentity.issueId !== previous.binding.issueId || checkpoint.driverKind !== previous.session.driverKind || checkpoint.providerSessionId !== providerIdentity.providerSessionId || latestCheckpoint.providerSessionId !== providerIdentity.providerSessionId || latestIdentity.sessionId !== identity.normalizedSessionId || latestIdentity.companyId !== previous.binding.companyId || latestIdentity.agentId !== previous.binding.agentId || latestIdentity.issueId !== previous.binding.issueId || checkpoint.activeTurnId !== null || !Array.isArray(checkpoint.pendingRuntimeRequests) || checkpoint.pendingRuntimeRequests.length !== 0 || !localProcessDefinitelyGone(prior.processPid) || !localProcessDefinitelyGone(prior.processGroupId, true) ) continue; // Provider processes are normally inside the runner group. Check any // independently recorded child identities too, including detached ones. const processIds = [ ...Object.entries(record(checkpoint.process)) .filter(([key]) => /^(provider|codex|sidecar|agent)Pid$/.test(key)) .map(([, value]) => value), ...events .map( (event) => record(record(record(event).envelope).payload).payload, ) .map((payload) => record(payload).processId) .filter((pid) => pid !== undefined), ]; if (processIds.some((pid) => !localProcessDefinitelyGone(pid))) continue; // No mutation if any evidence changed while the database was read. if ( runnerBytes !== readState("runner", "runner-state.json") || controlBytes !== readState("control-plane", "control-plane-state.json") || providerBytes !== readState("runner", "codex-provider-state.json") ) continue; verified.push({ root, runnerBytes, controlBytes, providerBytes, runner, runId: identity.runId, processPid: prior.processPid!, processGroupId: prior.processGroupId!, }); } catch { // Unreadable, unsafe, unscoped, or unprovable state stays quarantined. } } if (verified.length !== 1) { if ( currentEmpty && candidates.some( (root) => isSafeNativeStateDirectory(root) && readdirSync(root).length > 0, ) ) { // Known provider history is not permission to start a replacement when // recovery cannot prove a unique, settled owner. throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified"); } return; } const candidate = verified[0]!; // Candidate enumeration can await other database reads. Revalidate the // selected evidence and dead owner immediately before the atomic moves. for (const [relativePath, expected, maxBytes] of [ ["runner/runner-state.json", candidate.runnerBytes, NATIVE_RUNNER_STATE_MAX_BYTES], ["runner/codex-provider-state.json", candidate.providerBytes, NATIVE_RUNNER_STATE_MAX_BYTES], ["control-plane/control-plane-state.json", candidate.controlBytes, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES], ] as const) { if ( readBoundedNativeFile( resolve(candidate.root, relativePath), maxBytes, "recovery_state_too_large", ).toString("utf8") !== expected ) { throw new Error("runner_state_identity_mismatch: recovery_evidence_changed"); } } if ( !localProcessDefinitelyGone(candidate.processPid) || !localProcessDefinitelyGone(candidate.processGroupId, true) ) { throw new Error("runner_state_identity_mismatch: recovery_process_not_gone"); } if (candidate.root !== input.scoped) { if (existsSync(input.scoped)) { if ( !isSafeNativeStateDirectory(input.scoped) || readdirSync(input.scoped).length !== 0 ) return; quarantineRunnerdStateRoot(input.scoped, "identity_indeterminate"); } renameSync(candidate.root, input.scoped); } // The provider and its process group are gone and all work is settled. Seal // this old authority so the standard epoch rotation archives it before the // next run, retaining the provider's history and exact thread identity. const statePath = resolve(input.scoped, "runner", "runner-state.json"); const temporary = `${statePath}.${randomUUID()}.tmp`; writeFileSync( temporary, JSON.stringify({ ...candidate.runner, lifecycle: "suspended" }), { encoding: "utf8", mode: 0o600, flag: "wx" }, ); renameSync(temporary, statePath); await input.onLog?.( "stdout", `[paperclip-runner] Automatically recovered settled session from run ${candidate.runId}; preserving the existing provider thread.\n`, ); } function localProcessDefinitelyGone(value: unknown, group = false): boolean { if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 1) return false; try { process.kill(group ? -value : value, 0); return false; } catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; } } export function runnerdStateProvesIncompleteBootstrap(root: string): boolean { try { const statePath = resolve( root, "control-plane", "control-plane-state.json", ); const state = record( JSON.parse( readBoundedNativeFile( statePath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large", ).toString("utf8"), ), ); const commands = Array.isArray(state.commands) ? state.commands.map(record) : []; const committedEvents = Array.isArray(state.committedEvents) ? state.committedEvents : []; const onlyUnconsumedBootstrapCommands = commands.every( (command) => command.status === "pending" && (command.type === "run.prepare" || command.type === "session.open"), ); return ( state.schema === RUNNERD_CONTROL_PLANE_STATE_SCHEMA && state.connectionCount === 0 && committedEvents.length === 0 && onlyUnconsumedBootstrapCommands ); } catch { return false; } } function isSafeNativeStateDirectory(path: string): boolean { if (!existsSync(path)) return false; const stats = lstatSync(path); return stats.isDirectory() && !stats.isSymbolicLink(); } type RunnerdDurableIdentity = Record & { runId: string; normalizedSessionId: string; runnerInstanceId: string; environmentLeaseId: string; }; function readRunnerdDurableIdentity( root: string, ): Record | null { if (!isSafeNativeStateDirectory(root)) return null; const controlPlaneRoot = resolve(root, "control-plane"); if (!isSafeNativeStateDirectory(controlPlaneRoot)) return null; const statePath = resolve(controlPlaneRoot, "control-plane-state.json"); if (!existsSync(statePath)) return null; try { const state = record( JSON.parse( readBoundedNativeFile( statePath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large", ).toString("utf8"), ), ); if (state.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA) return null; return record(state.identity); } catch { return null; } } function durableIdentityMatchesExecution( identity: Record | null, execution: NativeExecutionInput, ): boolean { return Boolean( identity && identity.runId === execution.binding.runId && durableIdentityMatchesSession(identity, execution), ); } function durableIdentityMatchesSession( identity: Record | null, execution: NativeExecutionInput, ): identity is RunnerdDurableIdentity { return Boolean( identity && identity.normalizedSessionId === nativeSessionKey(execution) && typeof identity.runId === "string" && identity.runId.length > 0 && typeof identity.runnerInstanceId === "string" && identity.runnerInstanceId.length > 0 && typeof identity.environmentLeaseId === "string" && identity.environmentLeaseId.length > 0, ); } /** * Pre-v2 state is migrated for an exact active run, or for a suspended prior * run whose persisted, validated execution input proves the same full native * session scope. Ambiguous company/session-only state can never be claimed by * another agent, workspace, or provider profile. */ function runnerdStateRoot(execution: NativeExecutionInput): string { const scoped = scopedRunnerdStateRoot(execution); if (existsSync(scoped)) { if (!isSafeNativeStateDirectory(scoped)) { throw new Error("runner_state_directory_unsafe"); } return scoped; } for (const legacy of [ legacyCompanyRunnerdStateRoot(execution), legacyRunnerdStateRoot(execution), ]) { const migrated = migrateLegacyRunnerdStateRoot({ legacy, scoped, execution, }); if (migrated) return migrated; } return scoped; } function loadRunnerdDurableBinding( execution: NativeExecutionInput, transition?: VerifiedWarmTransitionBinding, ): { runnerInstanceId: string; environmentLeaseId: string; } | null { if (transition) { if ( readWarmTransitionSnapshot(runnerdStateRoot(execution)) .stateFingerprint !== transition.stateFingerprint ) { throw new Error("native_runner_warm_transition_recovery_unproven"); } return transition; } const identity = readRunnerdDurableIdentity(runnerdStateRoot(execution)); // The run id is intentionally different during a continuation. Reuse only // the verified runner/lease binding from the same company-scoped durable // session root; rotateLocalAuthorityEpoch then requires that exact binding // before it can archive the prior per-run authority. if (!durableIdentityMatchesSession(identity, execution)) return null; return { runnerInstanceId: identity.runnerInstanceId, environmentLeaseId: identity.environmentLeaseId, }; } function nativeSessionConfigDigest( execution: NativeExecutionInput, executionTargetKind: "local" | "remote" = "local", legacyProjectlessRunId?: string, ): string { const executionLocation = { executionKind: "local_process", // Use the same workspace identity as the durable session scope. For a // projectless task, executionWorkspaceId is a per-run placeholder, not a // workspace change. Real workspace/provider/policy changes still fence // retained processes and checkpoints through the rest of this digest. workspaceId: execution.binding.executionWorkspaceId === execution.binding.runId ? (legacyProjectlessRunId ?? nativeSessionWorkspaceScope(execution)) : execution.binding.executionWorkspaceId, cwd: execution.workspace.cwd, }; return `sha256:${createHash("sha256") .update( JSON.stringify({ companyId: execution.binding.companyId, ...(execution.binding.agentKeyId ? { agentKeyId: execution.binding.agentKeyId } : {}), normalizedSessionId: nativeSessionKey(execution), executionLocation, provider: execution.provider, driverKind: execution.session.driverKind, lifecyclePolicy: execution.session.lifecyclePolicy, workMode: execution.task.workMode, executionMode: "executionMode" in execution ? execution.executionMode : "default", runtimeContextDigest: "runtimeContext" in execution ? execution.runtimeContext.aggregateDigest : null, // Provider threads retain their tool declarations across resume. Bump // this revision whenever the server-authorized native tool surface // changes so an older thread is rotated instead of falsely resuming // without newly required tools. nativeToolContractFingerprint: nativeToolContractFingerprintForTarget(executionTargetKind), runtimeContract: nativeRuntimeContractForProvider(execution.provider), }), ) .digest("hex")}`; } function hasIdleWarmNativeSessionOwner(input: { execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; }): boolean { if (input.execution.session.lifecyclePolicy.mode !== "warm") return false; const entry = warmNativeSessions.get(nativeSessionScopeKey(input.execution)); if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return false; // A verified idle owner proves the checkpoint belongs to this session even // when its process must later rotate to a new run-scoped broker capability. // Local environments also have an id. Compare the same environment binding // stored at acquisition instead of treating every local target as unbound. const environmentId = input.runnerExecutionTarget?.environmentId ?? null; return ( entry.companyId === input.execution.binding.companyId && entry.environmentId === environmentId && entry.configDigest === nativeSessionConfigDigest( input.execution, input.runnerExecutionTarget?.kind ?? "local", ) ); } function nativeHarnessEnvironmentFingerprint( execution: NativeExecutionInput, ): string { return `sha256:${createHash("sha256") .update( canonicalJson({ companyId: execution.binding.companyId, ...(execution.binding.agentKeyId ? { agentKeyId: execution.binding.agentKeyId } : {}), agentId: execution.binding.agentId, issueId: execution.binding.issueId, normalizedSessionId: nativeSessionKey(execution), workspace: { cwd: execution.workspace.cwd, repoUrl: execution.workspace.repoUrl, repoRef: execution.workspace.repoRef, branchName: execution.workspace.branchName, }, provider: execution.provider, driverKind: execution.session.driverKind, }), ) .digest("hex")}`; } type NativeProviderKind = NativeExecutionInput["provider"]["kind"]; type NativeDriverKind = NativeExecutionInput["session"]["driverKind"]; export interface NativeHarnessPersistenceDirectory { name: "runner" | "codex-home" | "opencode" | "acpx"; location: "runner" | "filesystem"; excludeEntries: readonly string[]; } export interface NativeHarnessPersistenceProfile { providerKind: NativeProviderKind; driverKind: NativeDriverKind; directories: readonly NativeHarnessPersistenceDirectory[]; } export interface NativeHarnessBackupManifest { schema: "paperclip.native-harness-backup.v1"; normalizedSessionId: string; runnerInstanceId: string; providerKind: NativeProviderKind; driverKind: NativeDriverKind; providerSessionIdentity: unknown; sourceProviderLeaseId: string; environmentFingerprint: string; runnerContractVersion: number; directories: Array<{ name: string; sha256: string; bytes: number; }>; completedAt: string; } export function resolveNativeHarnessPersistenceProfile( execution: NativeExecutionInput, ): NativeHarnessPersistenceProfile { const providerDirectory: NativeHarnessPersistenceDirectory | null = execution.provider.kind === "codex" ? { name: "codex-home", location: "filesystem", // Codex session history lives below this home, but these files are // launch-time material: auth.json is copied from the configured // credential source and config.toml can contain the native MCP // bearer token. Re-materialize both for a replacement sandbox // instead of putting credentials into the disaster-recovery copy. excludeEntries: CODEX_HOME_NON_PERSISTENT_ENTRIES, } : execution.provider.kind === "opencode" ? { name: "opencode", location: "filesystem", excludeEntries: [], } : execution.provider.kind === "acpx" ? { name: "acpx", location: "filesystem", // ACPX stores each provider beneath a stable session directory. // Codex creates process-local executable aliases in tmp/arg0; // they may point outside the runtime tree and are neither safe // nor necessary to restore. Credentials and launch-time config // are also re-materialized in the replacement sandbox. Grok // diagnostics can contain auth fields and are not session state. excludeEntries: execution.provider.agent === "grok" ? ["auth.json", "auth-refresh.json", "auth-refresh.json.tmp", "config.toml", "logs"].map((entry) => `acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/grok-home/${entry}`) : execution.provider.agent === "codex" ? CODEX_HOME_NON_PERSISTENT_ENTRIES.map( (entry) => `acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/codex-home/${entry}`, ) : [], } : null; return { providerKind: execution.provider.kind, driverKind: execution.session.driverKind, directories: [ { name: "runner", location: "runner", excludeEntries: [] }, ...(providerDirectory ? [providerDirectory] : []), ], }; } function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; if (value && typeof value === "object") { const entries = Object.entries(value as Record).sort( ([left], [right]) => left.localeCompare(right), ); return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`; } return JSON.stringify(value) ?? "null"; } function runnerProviderStateFilename(execution: NativeExecutionInput): string { switch (execution.provider.kind) { case "openai_dot": return "dot-provider-state.json"; case "codex": case "opencode": return "codex-provider-state.json"; case "acpx": return "acpx-provider-state.json"; case "claude_managed": case "aws_agentcore": return "managed-provider-state.json"; } } /** * The v2 runner owns PRP identity/lifecycle in runner-state.json and keeps * provider recovery identity in a sibling provider state file. Never infer a * resumable provider from the outer PRP journal alone. */ export function providerSessionIdentityFromDurableProviderState(input: { execution: NativeExecutionInput; providerState: unknown; }): Record { const state = record(input.providerState); const expectedSessionId = nativeSessionKey(input.execution); const nonEmptyString = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0; const sha256 = (value: unknown): value is string => typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value); const emptyIdentity = () => ({ providerSessionId: null, providerBackendSessionId: null, providerSessionIdentity: null, }); switch (input.execution.provider.kind) { case "openai_dot": // Dot checkpoints recover the bridge only; there is no native provider thread. return emptyIdentity(); case "acpx": { const descriptor = record(state.descriptor); const identity = record(state.identity); const expectedModel = input.execution.provider.model; const requiredIdentityFields = [ "kind", "normalizedSessionId", "acpxRecordId", "backendSessionId", "agentSessionId", "profileDigest", "workspaceDigest", "requestedModel", "effectiveModel", ] as const; if ( state.schema !== ACPX_PROVIDER_STATE_SCHEMA || state.lifecycle !== "suspended" || state.providerExitUnconfirmed !== false || state.activeTurnId !== null || descriptor.kind !== "acpx" || descriptor.provider !== "acpx" || descriptor.driver !== "acpx_runtime" || descriptor.agent !== input.execution.provider.agent || descriptor.model !== expectedModel || descriptor.normalizedSessionId !== expectedSessionId || identity.kind !== "acpx" || identity.normalizedSessionId !== expectedSessionId || requiredIdentityFields.some( (field) => !nonEmptyString(identity[field]), ) || !sha256(identity.profileDigest) || !sha256(identity.workspaceDigest) || identity.profileDigest !== descriptor.commandDigest || identity.requestedModel !== expectedModel || identity.effectiveModel !== expectedModel || identity.permissionMode !== input.execution.provider.permissionMode || !acpxRecoveryModeMatches(input.execution.provider, descriptor.mode, identity.mode) || !acpxRecoveryPiThinkingMatches(input.execution.provider, descriptor.piThinkingLevel, identity.piThinkingLevel) || !["approve-all", "approve-paperclip", "approve-reads", "deny-all"].includes( String(identity.permissionMode), ) || !Array.isArray(identity.providerLifetimeFenceCandidates) || identity.providerLifetimeFenceCandidates.length !== 3 || new Set(identity.providerLifetimeFenceCandidates).size !== 3 || identity.providerLifetimeFenceCandidates.some( (port) => !Number.isInteger(port) || Number(port) < 49_152 || Number(port) > 65_535, ) ) { return emptyIdentity(); } return { providerSessionId: identity.acpxRecordId ?? null, providerBackendSessionId: identity.backendSessionId ?? null, providerSessionIdentity: structuredClone(identity), }; } case "codex": case "opencode": { const config = record(state.config); const expectedDriver = input.execution.provider.kind === "codex" ? "codex_app_server" : "opencode_server"; if ( state.schema !== CODEX_PROVIDER_STATE_SCHEMA || !["prepared", "session_open", "provider_exited"].includes( String(state.lifecycle), ) || !nonEmptyString(state.threadId) || (state.providerSessionId !== null && state.providerSessionId !== undefined && !nonEmptyString(state.providerSessionId)) || state.activeProviderTurnId !== null || state.ambiguousTurnStartPending === true || config.provider !== input.execution.provider.kind || config.driver !== expectedDriver ) { return emptyIdentity(); } return { providerSessionId: state.threadId ?? null, providerBackendSessionId: state.providerSessionId ?? null, providerSessionIdentity: null, }; } case "claude_managed": case "aws_agentcore": { const descriptor = record(state.descriptor); if ( state.schema !== MANAGED_PROVIDER_STATE_SCHEMA || state.lifecycle !== "suspended" || state.normalizedSessionId !== expectedSessionId || descriptor.kind !== input.execution.provider.kind || !nonEmptyString(state.providerSessionId) || state.activeTurnId !== null ) { return emptyIdentity(); } return { providerSessionId: state.providerSessionId ?? null, providerBackendSessionId: state.providerSessionId ?? null, providerSessionIdentity: null, }; } } } function providerSessionIdentityIsPresent(value: unknown): boolean { const identity = record(value); return ( (identity.providerSessionId !== null && identity.providerSessionId !== undefined) || (identity.providerBackendSessionId !== null && identity.providerBackendSessionId !== undefined) || (identity.providerSessionIdentity !== null && identity.providerSessionIdentity !== undefined) ); } /** Closed diagnostic categories; durable state may contain private provider data. */ export function durableProviderCheckpointFailureReason(execution: NativeExecutionInput, providerState: unknown): string { if (execution.provider.kind !== "acpx") return "identity_binding"; const state = record(providerState); if (state.schema !== ACPX_PROVIDER_STATE_SCHEMA) return "state_schema"; if (state.lifecycle !== "suspended") return "provider_not_suspended"; if (state.providerExitUnconfirmed !== false) return "provider_exit_unconfirmed"; if (state.activeTurnId !== null) return "provider_turn_unsettled"; if (!acpxRecoveryModeMatches(execution.provider, record(state.descriptor).mode, record(state.identity).mode)) return "mode_binding"; return "identity_binding"; } // Recovery consumes observed identities; it must never apply the fresh-config // default to a missing persisted mode. Provider capability checks belong to admission. function acpxRecoveryModeMatches( provider: NativeExecutionInput["provider"], ...observedModes: unknown[] ): boolean { const expected = record(provider).mode; return (expected === undefined || isProviderMode(expected)) && observedModes.every(mode => mode === expected); } function acpxRecoveryPiThinkingMatches(provider: NativeExecutionInput["provider"], ...observed: unknown[]): boolean { const expected = record(provider).piThinkingLevel; if (provider.kind === "acpx" && provider.agent === "pi") return ["off", "low", "high", "max"].includes(String(expected)) && observed.every(value => value === expected); return expected === undefined && observed.every(value => value === undefined); } export function providerSessionIdentityTransitionIsAllowed(input: { execution: NativeExecutionInput; previous: unknown; current: unknown; }): boolean { if (input.execution.provider.kind === "acpx" && !acpxRecoveryModeMatches( input.execution.provider, record(record(input.previous).providerSessionIdentity).mode, record(record(input.current).providerSessionIdentity).mode, )) return false; if (!acpxRecoveryPiThinkingMatches(input.execution.provider, record(record(input.previous).providerSessionIdentity).piThinkingLevel, record(record(input.current).providerSessionIdentity).piThinkingLevel)) return false; if (canonicalJson(input.previous) === canonicalJson(input.current)) { return true; } if ( input.execution.provider.kind !== "acpx" || input.execution.interactionResponses.length === 0 ) { return false; } const previousOuter = record(input.previous); const currentOuter = record(input.current); const previous = record(previousOuter.providerSessionIdentity); const current = record(currentOuter.providerSessionIdentity); if (previous.kind !== "acpx" || current.kind !== "acpx") return false; const stableFields = [ "normalizedSessionId", "profileDigest", "workspaceDigest", "requestedModel", "effectiveModel", "permissionMode", ] as const; if ( current.normalizedSessionId !== nativeSessionKey(input.execution) || stableFields.some( (field) => typeof previous[field] !== "string" || previous[field] !== current[field], ) ) { return false; } return ( typeof previous.acpxRecordId === "string" && typeof previous.backendSessionId === "string" && typeof previous.agentSessionId === "string" && typeof current.acpxRecordId === "string" && typeof current.backendSessionId === "string" && typeof current.agentSessionId === "string" && previousOuter.providerSessionId === previous.acpxRecordId && previousOuter.providerBackendSessionId === previous.backendSessionId && currentOuter.providerSessionId === current.acpxRecordId && currentOuter.providerBackendSessionId === current.backendSessionId ); } function digestBackupDirectory(directory: string): { sha256: string; bytes: number; } { const hash = createHash("sha256"); let bytes = 0; const visit = (current: string, relative: string) => { const entries = readdirSync(current, { withFileTypes: true }).sort( (left, right) => left.name.localeCompare(right.name), ); if (entries.length === 0) hash.update(`directory:${relative}\0`); for (const entry of entries) { const entryPath = resolve(current, entry.name); const entryRelative = relative ? `${relative}/${entry.name}` : entry.name; const stats = lstatSync(entryPath); if (entry.isDirectory()) { hash.update(`directory:${entryRelative}:${stats.mode & 0o777}\0`); visit(entryPath, entryRelative); } else if (entry.isSymbolicLink()) { hash.update(`symlink:${entryRelative}:${readlinkSync(entryPath)}\0`); } else if (entry.isFile()) { const contents = readFileSync(entryPath); bytes += contents.byteLength; hash.update( `file:${entryRelative}:${stats.mode & 0o777}:${contents.byteLength}\0`, ); hash.update(contents); } else { throw new Error( `runner_harness_backup_unsupported_entry:${entryRelative}`, ); } } }; visit(directory, ""); return { sha256: `sha256:${hash.digest("hex")}`, bytes }; } function harnessBackupRoot(root: string): string { return resolve(root, "failover-backups"); } function harnessBackupCandidates(root: string): string[] { const backupRoot = harnessBackupRoot(root); return [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]; } type VerifiedHarnessBackup = { root: string; manifest: NativeHarnessBackupManifest; bytes: number; }; export function shouldRestoreNativeHarnessBackupIntoSandbox(input: { acquisitionOutcome: "created" | "resumed" | "replacement" | null; reusableLeaseConfigured: boolean | null | undefined; backupAvailable: boolean; }): boolean { return ( input.acquisitionOutcome === "created" && input.reusableLeaseConfigured === false && input.backupAvailable ); } function compatibleNativeHarnessBackupManifests(input: { root: string; execution: NativeExecutionInput; runnerInstanceId: string; }): Array<{ root: string; manifest: NativeHarnessBackupManifest }> { const profile = resolveNativeHarnessPersistenceProfile(input.execution); const expectedNames = profile.directories .map((directory) => directory.name) .sort(); const compatible: Array<{ root: string; manifest: NativeHarnessBackupManifest; }> = []; for (const candidateRoot of harnessBackupCandidates(input.root)) { const manifestPath = resolve(candidateRoot, "manifest.json"); if (!existsSync(manifestPath)) continue; let manifest: NativeHarnessBackupManifest; try { manifest = JSON.parse( readFileSync(manifestPath, "utf8"), ) as NativeHarnessBackupManifest; } catch { continue; } if ( manifest.schema !== "paperclip.native-harness-backup.v1" || manifest.normalizedSessionId !== nativeSessionKey(input.execution) || manifest.runnerInstanceId !== input.runnerInstanceId || manifest.providerKind !== profile.providerKind || manifest.driverKind !== profile.driverKind || manifest.environmentFingerprint !== nativeHarnessEnvironmentFingerprint(input.execution) || manifest.runnerContractVersion !== RUNNERD_BINARY_CONTRACT_VERSION || !providerSessionIdentityIsPresent(manifest.providerSessionIdentity) || !Array.isArray(manifest.directories) ) continue; const manifestNames = manifest.directories .map((directory) => directory.name) .sort(); if (canonicalJson(expectedNames) !== canonicalJson(manifestNames)) continue; compatible.push({ root: candidateRoot, manifest }); } return compatible; } export function buildNativeHarnessBackupManifest(input: { backupRoot: string; execution: NativeExecutionInput; runnerInstanceId: string; providerSessionIdentity: unknown; sourceProviderLeaseId: string; completedAt?: string; }): NativeHarnessBackupManifest { if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) { throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing"); } const profile = resolveNativeHarnessPersistenceProfile(input.execution); const directories = profile.directories.map((directory) => { const path = resolve(input.backupRoot, directory.name); if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing"); return { name: directory.name, ...digestBackupDirectory(path) }; }); return { schema: "paperclip.native-harness-backup.v1", normalizedSessionId: nativeSessionKey(input.execution), runnerInstanceId: input.runnerInstanceId, providerKind: profile.providerKind, driverKind: profile.driverKind, providerSessionIdentity: input.providerSessionIdentity, sourceProviderLeaseId: input.sourceProviderLeaseId, environmentFingerprint: nativeHarnessEnvironmentFingerprint( input.execution, ), runnerContractVersion: RUNNERD_BINARY_CONTRACT_VERSION, directories, completedAt: input.completedAt ?? new Date().toISOString(), }; } export function verifyNativeHarnessBackup(input: { root: string; execution: NativeExecutionInput; runnerInstanceId: string; }): VerifiedHarnessBackup | null { for (const { root: candidateRoot, manifest, } of compatibleNativeHarnessBackupManifests(input)) { let bytes = 0; let valid = true; for (const declared of manifest.directories) { const directoryPath = resolve(candidateRoot, declared.name); if (!existsSync(directoryPath)) { valid = false; break; } try { const digest = digestBackupDirectory(directoryPath); if ( digest.sha256 !== declared.sha256 || digest.bytes !== declared.bytes ) { valid = false; break; } bytes += digest.bytes; } catch { valid = false; break; } } if (valid) return { root: candidateRoot, manifest, bytes }; } return null; } function nativeSessionCheckpointDirectory(): string { const directory = resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "sessions", ); mkdirSync(directory, { recursive: true, mode: 0o700 }); chmodSync(directory, 0o700); return directory; } function nativeSessionCheckpointPath(execution: NativeExecutionInput): string { return resolve( nativeSessionCheckpointDirectory(), `${createHash("sha256") .update(nativeSessionScopeKey(execution)) .digest("hex")}.json`, ); } function legacyCompanyNativeSessionCheckpointPath( execution: NativeExecutionInput, ): string { return resolve( nativeSessionCheckpointDirectory(), `${createHash("sha256") .update(legacyCompanyNativeSessionScopeKey(execution)) .digest("hex")}.json`, ); } function legacyNativeSessionCheckpointPath( execution: NativeExecutionInput, ): string { return resolve( nativeSessionCheckpointDirectory(), `${createHash("sha256") .update(nativeSessionKey(execution)) .digest("hex")}.json`, ); } function persistWarmNativeCheckpoint( execution: NativeExecutionInput, configDigest: string, snapshot: PersistedNativeSession, ): void { const path = nativeSessionCheckpointPath(execution); const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`; writeFileSync( temporary, JSON.stringify({ schema: "paperclip.native-session-supervisor.v1", configDigest, updatedAt: new Date().toISOString(), snapshot, }), { encoding: "utf8", mode: 0o600 }, ); renameSync(temporary, path); chmodSync(path, 0o600); } function loadWarmNativeCheckpoint( execution: NativeExecutionInput, configDigest: string, executionTargetKind: "local" | "remote" = "local", ): PersistedNativeSession | null { const scopedPath = nativeSessionCheckpointPath(execution); const path = [ scopedPath, legacyCompanyNativeSessionCheckpointPath(execution), legacyNativeSessionCheckpointPath(execution), ].find((candidate) => existsSync(candidate)); if (!path) return null; const envelope = JSON.parse( readBoundedNativeFile( path, NATIVE_WARM_CHECKPOINT_MAX_BYTES, "native_session_supervisor_checkpoint_too_large", ).toString("utf8"), ) as { schema?: string; configDigest?: string; snapshot?: PersistedNativeSession; }; if ( envelope.schema !== "paperclip.native-session-supervisor.v1" || !envelope.snapshot ) { throw new Error("native_session_supervisor_checkpoint_mismatch"); } const persistedIdentity = record(envelope.snapshot.identity); if ( persistedIdentity.sessionId !== nativeSessionKey(execution) || persistedIdentity.companyId !== execution.binding.companyId || persistedIdentity.agentId !== execution.binding.agentId ) { throw new Error("native_session_supervisor_checkpoint_mismatch"); } // Upgrade old projectless checkpoints using their validated prior run id. // Every provider/model/runtime-context/permission field must still match; // only the old per-run workspace placeholder is normalized away. const legacyDigest = execution.binding.executionWorkspaceId === execution.binding.runId && typeof persistedIdentity.runId === "string" ? nativeSessionConfigDigest( execution, executionTargetKind, persistedIdentity.runId, ) : null; if ( envelope.configDigest !== configDigest && envelope.configDigest !== legacyDigest ) { return null; } const sameRunRecovery = persistedIdentity.runId === execution.binding.runId && persistedIdentity.issueId === execution.binding.issueId; const resumed = sameRunRecovery ? structuredClone(envelope.snapshot) : { ...envelope.snapshot, identity: { runId: execution.binding.runId, sessionId: nativeSessionKey(execution), companyId: execution.binding.companyId, issueId: execution.binding.issueId, agentId: execution.binding.agentId, }, // A warm provider can be rebound only after the previous run settled. // Its provider identity survives, but run-scoped turn, result, and // request authority must not cross into the new heartbeat run. semanticResult: null, terminal: null, activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], governedWait: undefined, }; if (path !== scopedPath || envelope.configDigest !== configDigest) { // Upgrade the validated checkpoint atomically. When moving from a legacy // path, retain that file so an interrupted migration remains replayable. persistWarmNativeCheckpoint(execution, configDigest, resumed); } return resumed; } async function releaseWarmNativeSession( sessionId: string, ownerToken: symbol, idleTimeoutMs: number, failed: boolean, ): Promise { const entry = warmNativeSessions.get(sessionId); // A late completion or cleanup callback from an older execution must never // release a replacement that has since claimed the same logical session. if (!entry || entry.ownerToken !== ownerToken) return; entry.busy = false; entry.lastActivityAt = new Date().toISOString(); if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); if (failed || entry.closeOnReleaseReason !== undefined) { const requireCleanClose = entry.closeOnReleaseReason !== undefined; entry.closeOnReleaseReason ??= "warm native session failed"; entry.busy = true; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason); if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId); } catch (error) { if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); if (requireCleanClose) throw error; } finally { entry.busy = false; } return; } // Durable shared-computer ownership must be retained before completion is acknowledged. await entry.computerTarget?.retainWarm(idleTimeoutMs); entry.idleTimer = setTimeout(() => { const current = warmNativeSessions.get(sessionId); if (current !== entry || current.busy) return; current.busy = true; current.closeOnReleaseReason = "warm native session idle timeout"; void closeWarmNativeSession(current, current.closeOnReleaseReason) .then(() => { if (warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); }) .catch((error) => { if (error instanceof ComputerOwnerSupersededError) { // Admission owns the next generation. Leave this transport available // for its imminent warm reservation instead of closing or deleting it. current.closeOnReleaseReason = undefined; current.idleTimer = null; return; } if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); }) .finally(() => { current.busy = false; }); }, idleTimeoutMs); entry.idleTimer.unref(); } /** Classify only a committed provider terminal, never model prose or tool output. */ export function nativeProviderUsageLimitFromEvent( event: Pick, ): boolean { const payload = record(event.payload); return ( event.sourceKind === "runner" && event.eventType === "turn.failed" && payload.status === "failed" && record(payload.error).codexErrorInfo === "usageLimitExceeded" ); } export function nativeSessionFailureDisposition( attempt: number, now = new Date(), sourceFailureCode?: ReturnType, ) { const permanentFailure = sourceFailureCode === "native_provider_model_rejected" || sourceFailureCode === "native_provider_approval_required" || sourceFailureCode === "native_event_replay_conflict" || sourceFailureCode === "runner_remote_provider_artifact_incompatible" || sourceFailureCode === "runner_remote_artifact_platform_mismatch" || sourceFailureCode === "native_provider_terminal_failed" || sourceFailureCode === "native_current_wake_comments_unread" || sourceFailureCode === "native_current_wake_comments_changed_after_read" || sourceFailureCode === "native_session_cleanup_quarantined" || sourceFailureCode === "native_adopted_runner_authentication_timeout" || sourceFailureCode === "native_provider_usage_limit"; const permissionDeclined = sourceFailureCode === "native_permission_declined"; const exhausted = permanentFailure || permissionDeclined || attempt >= 3; return { phase: exhausted ? ("terminal_failure" as const) : ("retryable_failure" as const), failureCode: permanentFailure || permissionDeclined ? sourceFailureCode! : exhausted ? ("native_session_retry_exhausted" as const) : ("native_session_interrupted" as const), nextAttemptAt: exhausted ? null : new Date(now.getTime() + 30_000), }; } export function nativeSessionRecoveryProjection(input: { phase: "retryable_failure" | "terminal_failure"; failureCode: string; agentId: string; }) { const exhausted = input.phase === "terminal_failure"; return { exhausted, issueStatus: exhausted && input.failureCode !== NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT ? ("blocked" as const) : null, recoveryOwner: exhausted ? { kind: "board" as const } : { kind: "agent" as const, agentId: input.agentId }, recoveryActionOwnerType: exhausted ? ("board" as const) : ("agent" as const), recoveryActionOwnerAgentId: exhausted ? null : input.agentId, recoveryActionCause: input.failureCode, supersedeOnIdentityChange: true as const, }; } export function nativeSessionFailureSourceCode( error: unknown, ): | "native_permission_declined" | "native_provider_terminal_failed" | "native_provider_approval_required" | "native_provider_usage_limit" | "native_session_cleanup_quarantined" | "native_adopted_runner_authentication_timeout" | "runner_remote_provider_artifact_incompatible" | "runner_remote_artifact_platform_mismatch" | "provider_process_exited" | "provider_stdout_closed" | "provider_process_output_closed" | "provider_process_status_failed" | "provider_initialize_timeout" | "provider_initialize_protocol_error" | "provider_request_timeout" | "provider_request_protocol_error" | "provider_frame_too_large" | "provider_transport_failed" | "native_runner_process_exited" | "planning_mode_unsupported" | "native_event_replay_conflict" | "native_provider_model_rejected" | "native_current_wake_comments_unread" | "native_current_wake_comments_changed_after_read" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; if (error instanceof NativeProviderTerminalFailure) { if (error.providerCode === "approval_required") return "native_provider_approval_required"; // Failed terminals retain their security meaning across the provider facade. // A stopped process is insufficient evidence to recover an integrity breach. if ( /(?:binding_mismatch|start_mismatch|replay_conflict|digest_mismatch|invalid_semantic_result|conflicting_semantic_result|provider_event_type_invalid)/.test( error.providerCode, ) ) return "native_event_replay_conflict"; return "native_provider_terminal_failed"; } if (error instanceof NativeSessionProtocolIntegrityError) return "native_event_replay_conflict"; if (error instanceof NativeSessionCleanupQuarantinedError) return "native_session_cleanup_quarantined"; const message = error instanceof Error ? error.message : String(error); if (/native_provider_model_rejected/i.test(message)) return "native_provider_model_rejected"; if (/native_adopted_runner_authentication_timeout/i.test(message)) { return "native_adopted_runner_authentication_timeout"; } if (/^runner_remote_artifact_platform_mismatch(?::|$)/.test(message)) { return "runner_remote_artifact_platform_mismatch"; } if (/runner_remote_provider_artifact_incompatible/i.test(message)) { return "runner_remote_provider_artifact_incompatible"; } if (/provider_process_exited/i.test(message)) { return "provider_process_exited"; } if (/provider_stdout_closed/i.test(message)) { return "provider_stdout_closed"; } if (/provider_process_output_closed/i.test(message)) { return "provider_process_output_closed"; } if (/provider_process_status_failed/i.test(message)) { return "provider_process_status_failed"; } if (/provider_initialize_timeout/i.test(message)) { return "provider_initialize_timeout"; } if (/provider_initialize_protocol_error/i.test(message)) { return "provider_initialize_protocol_error"; } if (/provider_request_timeout/i.test(message)) { return "provider_request_timeout"; } if (/provider_request_protocol_error/i.test(message)) { return "provider_request_protocol_error"; } if (/provider_frame_too_large|stdout frame exceeded/i.test(message)) { return "provider_frame_too_large"; } if ( /provider_transport_failed|invalid JSON-RPC|provider failed/i.test(message) ) { return "provider_transport_failed"; } if ( /native_runner_process_exited|runnerd exited|runner process failed/i.test( message, ) ) { return "native_runner_process_exited"; } if (/planning_mode_unsupported/i.test(message)) { return "planning_mode_unsupported"; } if (/native_event_replay_conflict/i.test(message)) { return "native_event_replay_conflict"; } if (/native_current_wake_comments_changed_after_read/i.test(message)) { return "native_current_wake_comments_changed_after_read"; } if (/native_current_wake_comments_unread/i.test(message)) { return "native_current_wake_comments_unread"; } return "native_session_interrupted"; } const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; const PROVIDER_DURABLE_EVENT_TYPES = new Set([ "harness.ready", "session.started", "session.resumed", "session.updated", "turn.started", "provider.event", "provider.rpc_result", ]); type NativeRecoveryMode = "bootstrap_retry" | "exact_checkpoint_resume" | "ambiguous_state"; export async function nativeProviderRecoveryEvidence(input: { db: Db; runId: string; sourceFailureCode: ReturnType; }): Promise<{ recoveryMode: NativeRecoveryMode; providerSessionEstablished: boolean; providerEventsExist: boolean; checkpointExists: boolean; }> { const run = await input.db .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.runId)) .limit(1) .then((rows) => rows[0] ?? null); const checkpoint = record(run?.runnerProfileJson).sessionCheckpoint; const checkpointRecord = record(checkpoint); const checkpointExists = Object.keys(checkpointRecord).length > 0; const providerSessionEstablished = (typeof checkpointRecord.providerSessionId === "string" && checkpointRecord.providerSessionId.length > 0) || Object.keys(record(checkpointRecord.providerIdentity)).length > 0; const durableEvents = await input.db .select({ eventType: heartbeatRunEvents.eventType }) .from(heartbeatRunEvents) .where( and( eq(heartbeatRunEvents.runId, input.runId), inArray(heartbeatRunEvents.eventType, [ ...PROVIDER_DURABLE_EVENT_TYPES, ]), ), ) .limit(1); const providerEventsExist = durableEvents.some((event) => PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType), ); if ( checkpointExists && providerSessionEstablished && record(checkpointRecord.terminal).runTerminalState !== "failed" ) { return { recoveryMode: "exact_checkpoint_resume", providerSessionEstablished: true, providerEventsExist, checkpointExists, }; } const definitelyPreSession = new Set< ReturnType >([ "runner_remote_provider_artifact_incompatible", "runner_remote_artifact_platform_mismatch", "provider_process_exited", "provider_stdout_closed", "provider_process_output_closed", "provider_process_status_failed", "provider_initialize_timeout", "provider_initialize_protocol_error", "provider_request_timeout", "provider_request_protocol_error", "native_runner_process_exited", ]).has(input.sourceFailureCode); if (!checkpointExists && !providerEventsExist && definitelyPreSession) { return { recoveryMode: "bootstrap_retry", providerSessionEstablished: false, providerEventsExist: false, checkpointExists: false, }; } return { recoveryMode: "ambiguous_state", providerSessionEstablished: providerSessionEstablished || providerEventsExist, providerEventsExist, checkpointExists, }; } function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } type FirstAgentEventKind = | "reasoning" | "agentMessage" | "toolCall" | "dynamicToolCall"; /** Returns the first provider activity worth measuring after turn.started. */ export function firstMeaningfulAgentEventKind( event: Pick, ): FirstAgentEventKind | null { const payload = record(event.payload); if (event.eventType === "tool.execution.started") { return typeof payload.executionId === "string" && payload.executionId.trim() ? "toolCall" : null; } if ( event.eventType !== "item.started" && event.eventType !== "item.delta" && event.eventType !== "item.completed" ) return null; const kind = payload.kind; if ( kind !== "reasoning" && kind !== "agentMessage" && kind !== "toolCall" && kind !== "dynamicToolCall" ) return null; if (event.eventType !== "item.delta") return kind; const text = [payload.text, payload.delta, payload.content].find( (value): value is string => typeof value === "string" && value.trim().length > 0, ); return text ? kind : null; } export type NativeSessionSteeringState = { disposition: "available" | "unsupported" | "temporarily_unavailable"; activeTurnId: string | null; }; export class NativeSessionSteeringError extends Error { constructor( readonly code: | "steering_unsupported" | "steering_temporarily_unavailable" | "steering_stale_turn" | "steering_timeout" | "steering_rejected", message: string, ) { super(message); this.name = "NativeSessionSteeringError"; } } export class NativeRuntimeRequestResolutionError extends Error { constructor( readonly code: | "native_session_not_active" | "runtime_request_resolution_unsupported" | "runtime_request_stale_turn" | "runtime_request_resolution_conflict", message: string, ) { super(message); this.name = "NativeRuntimeRequestResolutionError"; } } /** Resolve a provider runtime request on an in-process native backend. */ export async function resolveNativeRuntimeRequest(input: { runId: string; requestId: string; turnId: string; resolution: HarnessRuntimeRequestResolution; /** * Revalidate the caller's durable lifecycle and authorization immediately * before the provider mutation. Capability and snapshot reads above this * edge are asynchronous, so route-level checks performed before entering * this helper are not sufficient to authorize the eventual dispatch. */ authorizeBeforeDispatch: () => Promise; }): Promise<{ commandId: string }> { const active = activeNativeSessions.get(input.runId); if (!active) { throw new NativeRuntimeRequestResolutionError( "native_session_not_active", "The active native session is not attached.", ); } const capabilities = await active.session.capabilities(); if ( !capabilities.runtimeRequestResolution || active.session.resolveRuntimeRequest === undefined ) { throw new NativeRuntimeRequestResolutionError( "runtime_request_resolution_unsupported", "This native session does not resolve runtime requests in-process.", ); } const snapshot = await active.session.snapshot(); if (snapshot.activeTurnId !== input.turnId) { throw new NativeRuntimeRequestResolutionError( "runtime_request_stale_turn", "The runtime request belongs to a turn that is no longer active.", ); } const key = `${input.runId}:${input.requestId}`; const fingerprint = JSON.stringify({ turnId: input.turnId, resolution: input.resolution, }); const prior = nativeRuntimeRequestResolutions.get(key); if (prior) { if (prior.fingerprint !== fingerprint) { throw new NativeRuntimeRequestResolutionError( "runtime_request_resolution_conflict", "A different response was already submitted for this runtime request.", ); } await prior.pending; return { commandId: prior.commandId }; } const commandId = `native-runtime-response:${randomUUID()}`; // Reserve the request key before yielding to authorization or provider I/O. // This makes duplicate retries join one dispatch and makes a conflicting // response fail closed even while the first authorization check is pending. const pending = Promise.resolve().then(async () => { await input.authorizeBeforeDispatch(); if (activeNativeSessions.get(input.runId) !== active) { throw new NativeRuntimeRequestResolutionError( "native_session_not_active", "The active native session changed before the response was dispatched.", ); } await active.session.resolveRuntimeRequest!({ requestId: input.requestId, turnId: input.turnId, resolution: input.resolution, }); }); const resolution: NativeRuntimeRequestResolution = { runId: input.runId, fingerprint, commandId, pending, completedAt: null, }; nativeRuntimeRequestResolutions.set(key, resolution); try { await pending; resolution.completedAt = Date.now(); pruneNativeRuntimeRequestResolutionCache(); return { commandId }; } catch (error) { if (nativeRuntimeRequestResolutions.get(key) === resolution) { nativeRuntimeRequestResolutions.delete(key); } throw error; } } export async function getNativeSessionSteeringState( runId: string, ): Promise { const active = activeNativeSessions.get(runId); if (!active) return { disposition: "temporarily_unavailable", activeTurnId: null }; const capabilities = await active.session.capabilities(); if (!capabilities.steering || !active.session.steer) { return { disposition: "unsupported", activeTurnId: null }; } const snapshot = await active.session.snapshot(); return { disposition: snapshot.activeTurnId ? "available" : "temporarily_unavailable", activeTurnId: snapshot.activeTurnId ?? null, }; } // Receipts live as long as this controller process. Durable identity reservations // hold credential acquisition after a restart until a provider receipt is known. const steeringDeliveries = new Map>(); function clearSteeringDeliveries(runId: string) { for (const key of steeringDeliveries.keys()) if (key.startsWith(`${runId}:`)) steeringDeliveries.delete(key); } /** Dispatches a true same-turn steering message and resolves only after ack. */ export async function steerNativeSession(input: { runId: string; message: string; correlationId: string; timeoutMs?: number; onAcknowledged?: () => Promise; }): Promise<{ turnId: string }> { const active = activeNativeSessions.get(input.runId); if (!active) { throw new NativeSessionSteeringError( "steering_temporarily_unavailable", "The active native session is not attached.", ); } const capabilities = await active.session.capabilities(); if (!capabilities.steering || !active.session.steer) { throw new NativeSessionSteeringError( "steering_unsupported", "This provider does not support same-turn steering.", ); } const snapshot = await active.session.snapshot(); const turnId = snapshot.activeTurnId ?? null; if (!turnId) { throw new NativeSessionSteeringError( "steering_stale_turn", "The target turn is no longer active.", ); } const deliveryKey = `${input.runId}:${input.correlationId}`; let delivery = steeringDeliveries.get(deliveryKey); if (!delivery) { delivery = active.session .steer({ turnId, message: { role: "user", text: input.message }, correlationId: input.correlationId, }) .then(() => ({ turnId })); steeringDeliveries.set(deliveryKey, delivery); void delivery.catch(() => { steeringDeliveries.delete(deliveryKey); }); } // The route still serializes queue mutations on the task until acknowledgement. // Reconciliation can acquire that task lock after success or a timeout; never // join it from the provider's acknowledgement path. if (input.onAcknowledged) void delivery.then(input.onAcknowledged).catch(() => undefined); let timeout: ReturnType | null = null; try { const acknowledged = await Promise.race([ delivery, new Promise((_resolve, reject) => { timeout = setTimeout( () => reject( new NativeSessionSteeringError( "steering_timeout", "The provider did not acknowledge steering in time.", ), ), input.timeoutMs ?? 10_000, ); }), ]); return acknowledged; } catch (error) { if (error instanceof NativeSessionSteeringError) throw error; const message = error instanceof Error ? error.message : String(error); if (/stale|terminal|active turn/i.test(message)) { throw new NativeSessionSteeringError( "steering_stale_turn", "The target turn is no longer active.", ); } if (/unsupported|unavailable|capability/i.test(message)) { throw new NativeSessionSteeringError( "steering_unsupported", "This provider does not support same-turn steering.", ); } throw new NativeSessionSteeringError( "steering_rejected", "The provider rejected the steering message.", ); } finally { if (timeout) clearTimeout(timeout); } } export function cancelNativeSession( runId: string, reason: string, ): Promise; export function cancelNativeSession( runId: string, reason: string, options: { db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean; cancellationRequestId?: string; }, ): Promise<{ dispatched: boolean; decision: NativeStatusDecision | null; decisionId: string | null; auditId: string | null; }>; export async function cancelNativeSession( runId: string, reason: string, options?: { db: Db; scope?: "turn" | "run" | "issue"; replacementAccepted?: boolean; cancellationRequestId?: string; }, ): Promise< | boolean | { dispatched: boolean; decision: NativeStatusDecision | null; decisionId: string | null; auditId: string | null; } > { const runStop = (options?.scope ?? "run") === "run"; let decision: NativeStatusDecision | null = null; let decisionContext: { companyId: string; issueId: string; assessmentId: string | null; priorStatus: string; priorStatusVersion: number; priorDecisionId: string | null; coordinatorDecisionId: string | null; agentId: string; } | null = null; if (options) { const run = await options.db .select({ agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, nativePhase: heartbeatRuns.nativePhase, errorCode: heartbeatRuns.errorCode, }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .limit(1) .then((rows) => rows[0] ?? null); if (run?.runtimeMode === "native") { if (isNativeRunnerOwnershipHeld(run)) throw new NativeRunnerOwnershipUnverifiedError(); const issueId = run.nativeIssueId; if (!issueId) throw new Error("native_cancellation_binding_missing"); const issue = await options.db .select({ status: issues.status, statusVersion: issues.statusVersion, lastStatusDecisionId: issues.lastStatusDecisionId, }) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) .limit(1) .then((rows) => rows[0] ?? null); if (!issue) throw new Error("native_cancellation_binding_missing"); const coordinator = await options.db .select({ assessmentId: nativeRunFinalizations.assessmentId, decisionId: nativeRunFinalizations.decisionId, }) .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.issueId, issueId), ), ) .limit(1) .then((rows) => rows[0] ?? null); if (!coordinator) throw new Error("native_cancellation_coordinator_missing"); decision = resolveNativeCancellationStatus({ scope: options.scope ?? "run", priorIssueStatus: issue.status as NativeAuthoritativeIssueStatus, agentId: run.agentId, replacementAccepted: options.replacementAccepted, }); decisionContext = { companyId: run.companyId, issueId, assessmentId: coordinator.assessmentId ?? null, priorStatus: issue.status, priorStatusVersion: Number(issue.statusVersion), priorDecisionId: issue.lastStatusDecisionId, coordinatorDecisionId: coordinator.decisionId ?? null, agentId: run.agentId, }; } } let decisionId: string | null = null; let auditId: string | null = null; let cancellationIntentId: string | null = null; let recoveringCancellationIntent = false; let priorCoordinatorDecisionIdAtIntent: string | null = null; if (options && decision && decisionContext) { const cancellationDecision = decision; const cancellationContext = decisionContext; const effects = cancellationDecision.effects.map((effect) => effect.kind); let intentPublication: Parameters[0] | null = null; const intent = await options.db.transaction(async (tx) => { const lockedRun = await tx .select({ agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, nativeIssueId: heartbeatRuns.nativeIssueId, resultJson: heartbeatRuns.resultJson, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, nativePhase: heartbeatRuns.nativePhase, errorCode: heartbeatRuns.errorCode, }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .for("update") .limit(1) .then((rows) => rows[0] ?? null); if ( !lockedRun || lockedRun.runtimeMode !== "native" || lockedRun.companyId !== cancellationContext.companyId || lockedRun.agentId !== cancellationContext.agentId || lockedRun.nativeIssueId !== cancellationContext.issueId ) { throw new Error("native_cancellation_binding_changed"); } if (isNativeRunnerOwnershipHeld(lockedRun)) throw new NativeRunnerOwnershipUnverifiedError(); const retryCancellation = lockedRun.status === "failed" || record(record(lockedRun.resultJson).startupCancellation).retryCancellation === true; const coordinatorQuery = tx .select({ runId: nativeRunFinalizations.runId, phase: nativeRunFinalizations.phase, failureCode: nativeRunFinalizations.failureCode }) .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.companyId, cancellationContext.companyId), eq(nativeRunFinalizations.issueId, cancellationContext.issueId), ), ); // Recheck after reservation, at the same transaction that records intent // and disables the retry. NOWAIT avoids coordinator -> run lock inversion. const coordinator = await (retryCancellation ? coordinatorQuery.for("update", { noWait: true }) : coordinatorQuery) .limit(1).then((rows) => rows[0] ?? null); if (!coordinator) throw new Error("native_cancellation_coordinator_missing"); const resultJson = record(lockedRun.resultJson); // A default Stop joins an already reserved caller intent; it cannot // replace that intent while the originating request is still dispatching. const requestedId = options.cancellationRequestId ?? cancellationRequestId(record(resultJson.startupCancellation).cancellationRequestId); if (requestedId) assertCancellationRequest(resultJson, requestedId); if (retryCancellation && !nativeRetryCancellationEligible({ runId, companyId: cancellationContext.companyId, issueId: cancellationContext.issueId, resultJson, requestId: requestedId, scope: options.scope ?? "run", }, coordinator)) throw conflict("Native retry is no longer cancellable"); const existing = record(resultJson.nativeCancellation); const existingIntentId = typeof existing.intentId === "string" && existing.intentId.length > 0 ? existing.intentId : null; if (existingIntentId) { const matchingIntent = existing.schema === "paperclip.native-cancellation.v1" && existing.companyId === cancellationContext.companyId && existing.runId === runId && existing.issueId === cancellationContext.issueId && existing.scope === (options.scope ?? "run") && existing.reasonCode === cancellationDecision.reasonCode && JSON.stringify(existing.effects) === JSON.stringify(effects); if (!matchingIntent) throw new Error("native_cancellation_intent_conflict"); const existingAuditId = typeof existing.intentAuditId === "string" && existing.intentAuditId.length > 0 ? existing.intentAuditId : null; if (!existingAuditId) throw new Error("native_cancellation_intent_audit_missing"); return { intentId: existingIntentId, auditId: existingAuditId, acknowledged: existing.dispatchState === "acknowledged", dispatched: existing.dispatched === true, decisionId: typeof existing.decisionId === "string" ? existing.decisionId : null, priorCoordinatorDecisionId: typeof existing.priorCoordinatorDecisionId === "string" ? existing.priorCoordinatorDecisionId : null, existing: true, }; } const intentId = requestedId ? callerCancellationIntentId(requestedId) : `native-cancellation:${randomUUID()}`; const activity = await persistActivity(tx as unknown as Db, { companyId: cancellationContext.companyId, actorType: "system", actorId: "native-session-cancellation", action: "native.cancellation_intent_recorded", entityType: "heartbeat_run", entityId: runId, agentId: cancellationContext.agentId, runId, issueId: cancellationContext.issueId, details: { intentId, scope: options.scope ?? "run", reasonCode: cancellationDecision.reasonCode, effects, }, }); const intentAuditId = activity.activity?.id ?? null; if (!intentAuditId) throw new Error("native_cancellation_intent_audit_missing"); const written = await tx .update(heartbeatRuns) .set({ resultJson: { ...resultJson, nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId, intentAuditId, companyId: cancellationContext.companyId, runId, issueId: cancellationContext.issueId, scope: options.scope ?? "run", reasonCode: cancellationDecision.reasonCode, effects, dispatchState: "pending", dispatched: false, decisionId: null, priorCoordinatorDecisionId: cancellationContext.coordinatorDecisionId, recordedAt: new Date().toISOString(), }, }, updatedAt: new Date(), }) .where( and( eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, cancellationContext.companyId), eq(heartbeatRuns.agentId, cancellationContext.agentId), eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), ), ) .returning({ id: heartbeatRuns.id }) .then((rows) => rows[0] ?? null); if (!written) throw new Error("native_cancellation_binding_changed"); // Cancellation is also an authority fence for a durable retry whose // preceding provider has already failed. No in-memory session is needed. await tx .update(nativeRunFinalizations) .set({ phase: "terminal_failure", nextAttemptAt: null, leaseOwner: null, leaseExpiresAt: null, recoveryState: "blocked", failureCode: "native_retry_cancelled", updatedAt: new Date(), }) .where( and( eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.companyId, cancellationContext.companyId), eq(nativeRunFinalizations.phase, "retryable_failure"), ), ); intentPublication = activity.publication; return { intentId, auditId: intentAuditId, acknowledged: false, dispatched: false, decisionId: null, priorCoordinatorDecisionId: cancellationContext.coordinatorDecisionId, existing: false, }; }).catch(rethrowNativeCancellationLockConflict); if (intentPublication) publishActivity(intentPublication); cancellationIntentId = intent.intentId; auditId = intent.auditId; decisionId = intent.decisionId; recoveringCancellationIntent = intent.existing; priorCoordinatorDecisionIdAtIntent = intent.priorCoordinatorDecisionId; if (intent.acknowledged && (!runStop || intent.dispatched || (!nativeSessionStartups.has(runId) && !activeNativeSessions.has(runId)))) { return { dispatched: intent.dispatched, decision, decisionId, auditId, }; } } // Startup already owns a provider lifetime even before publishing its handle. // Do not acknowledge a no-op Stop and let that owner submit a turn afterwards. const startup = runStop && !activeNativeSessions.has(runId) ? nativeSessionStartups.get(runId) : undefined; let settleStartupCancellation: (() => void) | undefined; if (startup) { startup.stopRequested = true; startup.cancellationSettled = new Promise(resolve => { settleStartupCancellation = resolve; }); } try { const active = activeNativeSessions.get(runId) ?? (startup ? await waitForNativeSessionStartup(runId, () => new NativeCancellationPendingRecoveryError()) : null); let dispatched = false; if (active) { dispatched = true; if (!active.cancelRequested) { active.cancelRequested = true; try { if (active.session.cancel) { const cancellationAbort = new AbortController(); const cleanup = active.session.cancel({ reason, signal: cancellationAbort.signal, }).cleanup; let cleanupTimer: ReturnType | undefined; const settled = await Promise.race([ cleanup.then( () => true, () => true, ), new Promise((resolve) => { cleanupTimer = setTimeout( () => resolve(false), NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS, ); }), ]); if (cleanupTimer) clearTimeout(cleanupTimer); if (!settled) { cancellationAbort.abort( new Error("native session cancellation cleanup timed out"), ); void cleanup.catch(() => undefined); } } else if (active.session.interrupt) await active.session.interrupt({ reason }); } catch (error) { active.cancelRequested = false; throw error; } } } if (!options) return dispatched; if (decision && decisionContext) { const cancellationDecision = decision; const cancellationContext = decisionContext; if (!cancellationIntentId || !auditId) throw new Error("native_cancellation_intent_audit_missing"); if ( recoveringCancellationIntent && cancellationContext.coordinatorDecisionId !== priorCoordinatorDecisionIdAtIntent ) { decisionId ??= cancellationContext.coordinatorDecisionId; } if ( cancellationContext.assessmentId && cancellationDecision.reasonCode !== null && !decisionId ) { const committed = await commitNativeStatusDecision({ db: options.db, companyId: cancellationContext.companyId, issueId: cancellationContext.issueId, runId, assessmentId: cancellationContext.assessmentId, priorStatus: cancellationContext.priorStatus, priorStatusVersion: cancellationContext.priorStatusVersion, priorDecisionId: cancellationContext.priorDecisionId, decision: cancellationDecision, }); decisionId = committed.decision.id; } const acknowledgement = await options.db.transaction(async (tx) => { const lockedRun = await tx .select({ agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, nativeIssueId: heartbeatRuns.nativeIssueId, resultJson: heartbeatRuns.resultJson, runtimeMode: heartbeatRuns.runtimeMode, }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .for("update") .limit(1) .then((rows) => rows[0] ?? null); if ( !lockedRun || lockedRun.runtimeMode !== "native" || lockedRun.companyId !== cancellationContext.companyId || lockedRun.agentId !== cancellationContext.agentId || lockedRun.nativeIssueId !== cancellationContext.issueId ) { throw new Error("native_cancellation_binding_changed"); } const coordinator = await tx .select({ runId: nativeRunFinalizations.runId }) .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, runId), eq(nativeRunFinalizations.companyId, cancellationContext.companyId), eq(nativeRunFinalizations.issueId, cancellationContext.issueId), ), ) .limit(1) .then((rows) => rows[0] ?? null); if (!coordinator) throw new Error("native_cancellation_coordinator_missing"); const resultJson = record(lockedRun.resultJson); const intent = record(resultJson.nativeCancellation); const matchingIntent = intent.schema === "paperclip.native-cancellation.v1" && intent.intentId === cancellationIntentId && intent.intentAuditId === auditId && intent.companyId === cancellationContext.companyId && intent.runId === runId && intent.issueId === cancellationContext.issueId; if (!matchingIntent) throw new Error("native_cancellation_intent_conflict"); if (intent.dispatchState === "acknowledged" && (intent.dispatched === true || !dispatched)) { return { publication: null, decisionId: typeof intent.decisionId === "string" ? intent.decisionId : decisionId, }; } if ( options.replacementAccepted && cancellationDecision.effects.some( (effect) => effect.kind === "accept_replacement_turn", ) ) { await tx .update(heartbeatRuns) .set({ status: "running", continuationAttempt: sql`${heartbeatRuns.continuationAttempt} + 1`, nextAction: "Accept a replacement native turn on the existing run.", updatedAt: new Date(), }) .where( and( eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, cancellationContext.companyId), eq(heartbeatRuns.agentId, cancellationContext.agentId), eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), ), ); } const activity = await persistActivity(tx as unknown as Db, { companyId: cancellationContext.companyId, actorType: "system", actorId: "native-session-cancellation", action: "native.cancellation_dispatch_acknowledged", entityType: "heartbeat_run", entityId: runId, agentId: cancellationContext.agentId, runId, issueId: cancellationContext.issueId, details: { intentId: cancellationIntentId, intentAuditId: auditId, scope: options.scope ?? "run", reasonCode: cancellationDecision.reasonCode, effects: cancellationDecision.effects.map((effect) => effect.kind), dispatched, decisionId, }, }); const acknowledgementAuditId = activity.activity?.id ?? null; if (!acknowledgementAuditId) throw new Error("native_cancellation_ack_audit_missing"); const cancellationWrite = await tx .update(heartbeatRuns) .set({ resultJson: { ...resultJson, nativeCancellation: { ...intent, dispatchState: "acknowledged", dispatched, decisionId, acknowledgementAuditId, acknowledgedAt: new Date().toISOString(), }, }, updatedAt: new Date(), }) .where( and( eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, cancellationContext.companyId), eq(heartbeatRuns.agentId, cancellationContext.agentId), eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), ), ) .returning({ id: heartbeatRuns.id }) .then((rows) => rows[0] ?? null); if (!cancellationWrite) throw new Error("native_cancellation_binding_changed"); return { publication: activity.publication, decisionId }; }); decisionId = acknowledgement.decisionId; if (acknowledgement.publication) publishActivity(acknowledgement.publication); } return { dispatched, decision, decisionId, auditId }; } finally { settleStartupCancellation?.(); } } /** Authenticated cancellation scope projected through the shared arbiter. */ export function resolveNativeCancellationStatus(input: { scope: "turn" | "run" | "issue"; priorIssueStatus: NativeAuthoritativeIssueStatus; agentId: string; replacementAccepted?: boolean; }): NativeStatusDecision { if (input.scope === "turn") { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "preserve", toStatus: input.priorIssueStatus, reasonCode: input.replacementAccepted ? null : "cancellation_turn_only", unblockDescriptor: null, effects: [{ kind: "accept_replacement_turn" }], }; } if (input.scope === "run") { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "preserve", toStatus: input.priorIssueStatus, reasonCode: "cancellation_run_only", unblockDescriptor: null, effects: [{ kind: "release_run_resources" }], }; } return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "cancelled", toStatus: "cancelled", reasonCode: "cancellation_issue_authorized", unblockDescriptor: null, effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }], }; } export async function renewNativeSessionExecutionLease(input: { db: Db; runId: string; companyId: string; issueId: string; leaseOwner: string; attempt: number; controller?: NativeControllerIdentity; leaseTtlMs?: number; }): Promise { const controller = input.controller ?? (await currentNativeControllerIdentity()); const leaseTtlMs = input.leaseTtlMs ?? NATIVE_SESSION_EXECUTION_LEASE_TTL_MS; if ( !Number.isInteger(leaseTtlMs) || leaseTtlMs < 1_000 || leaseTtlMs > NATIVE_SESSION_EXECUTION_LEASE_TTL_MS ) { throw new Error("native_session_lease_ttl_invalid"); } const [updated] = await input.db .update(nativeRunFinalizations) .set({ leaseExpiresAt: sql`now() + (${leaseTtlMs} * interval '1 millisecond')`, updatedAt: sql`now()`, }) .where( and( eq(nativeRunFinalizations.runId, input.runId), eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId), eq(nativeRunFinalizations.leaseOwner, input.leaseOwner), eq(nativeRunFinalizations.attempt, input.attempt), eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid), eq( nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt, ), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .returning({ runId: nativeRunFinalizations.runId }); if (!updated) throw new Error("native_session_lease_lost"); } /** Computer ACPX cold starts stage the complete qualified Linux provider pack. * Measured 1.1 GiB transfers plus compression/verification need about 24 minutes; * this finite admission budget does not extend warm leases or turn execution. */ export function nativeSessionBootstrapTimeoutMs( providerKind: NativeExecutionInput["provider"]["kind"], target?: Pick, "kind" | "transport"> | { kind: "local" } | null, ): number | undefined { return providerKind === "acpx" && target?.kind === "remote" && target.transport === "computer" ? 30 * 60 * 1_000 : undefined; } function startNativeSessionExecutionLeaseRenewal(input: { db: Db; runId: string; companyId: string; issueId: string; leaseOwner: string; attempt: number; controller: NativeControllerIdentity; }): { stop: () => Promise } { let leaseLost: Error | null = null; let renewal = Promise.resolve(); const renew = () => { if (leaseLost) return; renewal = renewal .then(() => renewNativeSessionExecutionLease(input)) .then(() => undefined) .catch(async (error: unknown) => { leaseLost = error instanceof Error ? error : new Error("native_session_lease_lost"); await cancelNativeSession( input.runId, "native session execution lease lost", ).catch(() => undefined); }); }; const timer = setInterval( renew, NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS, ); timer.unref?.(); return { stop: async () => { clearInterval(timer); await renewal; if (leaseLost) throw leaseLost; }, }; } export async function executePaperclipNativeSession(input: { getFreshSessionHandoff?: () => Promise; /** Retire a retained transport so recovery can replace provider tool declarations. */ refreshTools?: boolean; db: Db; execution: NativeExecutionInput; runnerInstanceId: string; /** Trusted task identity from the heartbeat orchestration. */ conversationMode?: boolean; /** Configured total turn bound; zero/unset is unlimited. */ turnTimeoutMs?: number; leaseOwner?: string; restartRecovery?: NativeRestartRecoveryClaim; onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string; }) => Promise; /** Test seam at the provider boundary; production uses a qualified package backend. */ backend?: NativeSessionBackend; useRunnerd?: boolean; /** Paperclip adapter identity used to scope the durable goal projection. */ adapterType?: string; /** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */ chatAttachmentReadScope?: NativeChatAttachmentReadScope; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; onEvent?: (event: AdapterRuntimeEvent) => Promise; /** Only this run's registered private instruction entry. * Probe at terminal; persist only after owned shutdown. */ instructionWorkingCopy?: NativeInstructionWorkingCopy; /** Persist task-level continuity before a durable goal can outlive this run. */ onUsage?: (receipt: AdapterUsageCheckpoint) => Promise; onGoalCheckpoint?: (snapshot: PersistedNativeSession) => Promise; sessionGoalControl?: NativeSessionGoalControl | null; resumeSessionGoalHeartbeat?: boolean; preparationSpans?: NativeRunHistoricalSpan[]; /** Use a session-owned GitHub broker, rebound only after run ownership is acquired. */ managedGitHub?: boolean; /** Resolved adapter env; the runner transport applies a provider allowlist before spawn. */ dotWorkspaceRoot?: string; runnerEnvironment?: NodeJS.ProcessEnv; /** Private grant materialization; never a user-configured host path. */ managedAiCredentialHome?: string; managedAiCredentialIdentity?: string; billingIdentity?: Pick; runnerExecutionTarget?: AdapterExecutionTarget | null; /** Resolved per-run authorization; not an independent instance setting. */ runnerIngressAuthorized?: boolean; runnerPublicUrl?: string | null; runnerCaBundlePath?: string | null; runnerRemoteBinaryPath?: string | null; runnerRemoteCodexPath?: string | null; runnerRemoteCodexNpmSpec?: string | null; runnerRemoteProviderPackPath?: string | null; stopTaskForReassignment?: (target: { companyId: string; issueId: string; agentId: string; runId: string | null }) => Promise; syncIssueExternalObjects?: (issueId: string) => Promise; enqueueWakeup?: ( agentId: string, options: { source: "assignment" | "automation"; triggerDetail: "system"; reason: "issue_assigned" | "issue_commented"; payload: Record; idempotencyKey: string; requestedByActorType: "agent"; requestedByActorId: string; contextSnapshot: Record; }, ) => Promise; }): Promise { const agentKeyId = input.runnerEnvironment?.PAPERCLIP_AGENT_KEY_ID; if (agentKeyId) { // Recovered pre-upgrade inputs also need the current public identity marker // before deciding whether an already-running provider can be reused. input = { ...input, execution: { ...input.execution, binding: { ...input.execution.binding, agentKeyId }, } }; } const runId = input.execution.binding.runId; if (nativeSessionStartups.has(runId)) { throw new Error("native_session_supervisor_busy"); } // Register before the first asynchronous operation on either backend path. // A duplicate execution must not replace the original startup handoff. let resolveStartup!: (session: ActiveNativeSession | null) => void; let settle!: () => void; const startup: NativeSessionStartup = { settled: new Promise(resolve => { settle = resolve; }), settle: () => settle(), promise: new Promise(resolve => { resolveStartup = resolve; }), resolve: session => resolveStartup(session), }; nativeSessionStartups.set(runId, startup); let preparedInput: typeof input = input; let cleanupStagedAttachments: () => Promise = async () => undefined; let sessionScopeId: string | null = null; let ownsSessionScope = false; let executionFailure: unknown; const ownerScope = nativeSessionOwnerScope( input.execution, input.runnerExecutionTarget?.environmentId ?? null, ); const ownerToken = Symbol("native execution owner"); try { // Reserve across session identities before retiring or staging anything. if (executingNativeOwnerScopes.has(ownerScope)) { throw new Error("native_session_supervisor_busy"); } executingNativeOwnerScopes.set(ownerScope, ownerToken); await retireSupersededWarmNativeSessions( ownerScope, nativeSessionScopeKey(input.execution), ); if (input.execution.provider.kind === "openai_dot" && isCloudManagedInstance() && (!input.useRunnerd || input.runnerExecutionTarget?.kind !== "remote" || input.runnerExecutionTarget.transport !== "sandbox")) { throw new Error("dot_cloud_requires_managed_runner"); } if (!input.useRunnerd) { return await executePaperclipNativeSessionWithinScope(input); } // The session scope is unaffected by appending server-staged attachment // descriptors. Claim it before any workspace scrub/write so a duplicate // execution cannot truncate or replace the active turn's staging inode. sessionScopeId = nativeSessionScopeKey(input.execution); if (executingRunnerdSessionScopes.has(sessionScopeId)) { throw new Error("native_session_supervisor_busy"); } executingRunnerdSessionScopes.set( sessionScopeId, input.execution.binding.runId, ); ownsSessionScope = true; // The shutdown sweep can have removed an idle owner while its remote // checkpoint is still being saved. The scope reservation also prevents // a later sweep from closing an owner this turn is about to acquire. await closingWarmNativeSessions.get(sessionScopeId); if (input.execution.provider.kind === "openai_dot") { return await executePaperclipNativeSessionWithinScope(input); } const workspaceRoot = input.execution.workspace.cwd; if (workspaceRoot === null) throw new Error("native_workspace_required"); const targetKind = input.runnerExecutionTarget?.kind ?? "local"; const chatAttachmentReadScope = new NativeChatAttachmentReadScope({ db: input.db, binding: input.execution.binding, workspaceRoot, executionTargetKind: targetKind, }); preparedInput = { ...input, chatAttachmentReadScope }; cleanupStagedAttachments = () => chatAttachmentReadScope.close(); const attachmentStage = await stageNativeRunnerWakeAttachments({ db: input.db, binding: { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, runId: input.execution.binding.runId, agentId: input.execution.binding.agentId, workspaceRoot, executionTargetKind: targetKind, }, }); cleanupStagedAttachments = async () => { const cleanupResults = await Promise.allSettled([ chatAttachmentReadScope.close(), attachmentStage.cleanup(), ]); if (cleanupResults.some((result) => result.status === "rejected")) { throw new Error("paperclip_runner_attachment_staging_cleanup_failed"); } }; const stagedPrompt = renderNativeRunnerStagedAttachmentPrompt( attachmentStage.attachments, ); if (stagedPrompt) { preparedInput = { ...preparedInput, execution: parseNativeExecutionInput({ ...input.execution, task: { ...input.execution.task, prompt: `${input.execution.task.prompt}\n\n${stagedPrompt}`, }, }), }; } return await executePaperclipNativeSessionWithinScope(preparedInput); } catch (error) { executionFailure = error; throw error; } finally { if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) { executingNativeOwnerScopes.delete(ownerScope); } startup.resolve(null); startup.settle(); if (nativeSessionStartups.get(runId) === startup) { nativeSessionStartups.delete(runId); } if ( ownsSessionScope && sessionScopeId !== null && executingRunnerdSessionScopes.get(sessionScopeId) === input.execution.binding.runId ) { executingRunnerdSessionScopes.delete(sessionScopeId); } try { await cleanupStagedAttachments(); } catch (cleanupError) { // Cleanup is a confidentiality incident, but it occurs after the native // provider may already have completed the turn. Reclassifying that turn // as failed could replay provider side effects. Emit a private runtime // health event while preserving the provider result/error disposition. await input .onEvent?.({ eventType: "native.attachment_staging_cleanup_failed", level: "error", message: "Native inbound attachment staging cleanup failed.", payload: { runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, executionAlreadyFailed: executionFailure !== undefined, }, }) .catch(() => undefined); } } } async function executePaperclipNativeSessionWithinScope( input: Parameters[0], ): Promise { const identityRedactor = createAgentIdentityRedactor(input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY); const redactIdentityText = (text: string) => redactSensitiveText( identityRedactor.chunk("diagnostic", text) + identityRedactor.finish("diagnostic"), ); if ( input.execution.provider.kind !== "codex" && input.execution.provider.kind !== "opencode" && input.execution.provider.kind !== "claude_managed" && input.execution.provider.kind !== "aws_agentcore" && input.execution.provider.kind !== "acpx" && input.execution.provider.kind !== "openai_dot" ) { throw new Error("paperclip_runner_provider_unsupported"); } if ( input.execution.provider.kind === "acpx" && ["copilot"].includes(input.execution.provider.agent) && !resolveAcpxQualification(input.execution.provider, process.env) ) { throw new Error( "paperclip_runner_provider_unsupported: ACPX candidate requires exact host qualification authorization", ); } const preparationSpans = input.preparationSpans ?? []; const preparationStarts = nativeRunPreparationStarts( preparationSpans, Date.now(), ); const trace = createNativeRunTrace({ runId: input.execution.binding.runId, startedAtMs: preparationStarts.runStartedAtMs, onEvent: input.onEvent, }); const toolTrace = createNativeToolTrace(trace); const taskPrepareScope = trace.start("task.prepare", { parentName: "task.run", startedAtMs: preparationStarts.preparationStartedAtMs, }); const environmentSpans = preparationSpans.filter( (span) => span.name === "environment.acquire" || span.name === "environment.workspace.realize", ); const environmentStartedAtMs = environmentSpans.reduce( (earliest, span) => Math.min(earliest, span.startedAtMs), Date.now(), ); const environmentEndedAtMs = environmentSpans.reduce( (latest, span) => Math.max(latest, span.endedAtMs), environmentStartedAtMs, ); const environmentScope = environmentSpans.length > 0 ? trace.start("environment.startup", { parentName: "task.prepare", startedAtMs: environmentStartedAtMs, }) : null; for (const span of preparationSpans) { const rootMilestone = isNativeRunRootHistoricalSpan(span.name); await trace.record({ ...span, parentName: rootMilestone ? "task.run" : environmentSpans.includes(span) ? "environment.startup" : "task.prepare", }); } if (environmentScope) { await trace.end(environmentScope, { endedAtMs: environmentEndedAtMs }); } let retainedTransition: VerifiedWarmTransitionBinding | undefined; if (input.useRunnerd) { retainedTransition = await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox", // A retained warm runner is deliberately still ready rather than // suspended. Only the exact idle in-process owner may rotate that // prior-run authority; after a hard restart the map is empty and the // durable-state verifier continues to require a suspended runner. allowRetainedWarmRunner: hasIdleWarmNativeSessionOwner(input), allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote", onLog: input.onLog, restartRecovery: input.restartRecovery, runnerExecutionTarget: input.runnerExecutionTarget, }); } const durableRunnerBinding = input.useRunnerd ? loadRunnerdDurableBinding(input.execution, retainedTransition) : null; const effectiveRunnerInstanceId = durableRunnerBinding?.runnerInstanceId ?? input.runnerInstanceId; if (effectiveRunnerInstanceId !== input.runnerInstanceId) { await input.db .update(heartbeatRuns) .set({ runnerInstanceId: effectiveRunnerInstanceId, updatedAt: new Date(), }) .where( and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), ), ); } const leaseOwner = input.leaseOwner ?? `${effectiveRunnerInstanceId}:${randomUUID()}`; const controller = await currentNativeControllerIdentity(); const leaseNow = new Date(); const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000); let attempt: number; try { attempt = await trace.measure( "native.coordinator.claim", () => input.db.transaction(async (tx) => { const coordinator = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq( nativeRunFinalizations.companyId, input.execution.binding.companyId, ), eq( nativeRunFinalizations.issueId, input.execution.binding.issueId, ), ), ) .for("update") .limit(1) .then((rows) => rows[0] ?? null); if (!coordinator) throw new Error("native_finalization_coordinator_missing"); const leaseNow = new Date(); const leaseExpiresAt = new Date( leaseNow.getTime() + NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, ); // A durable result means provider execution already completed. The // recovery/finalization path must reconcile it; never reacquire a // provider session and execute the turn a second time. if (coordinator.resultId) throw new NativeResultPendingFinalizationError(); const boundRun = await tx .select({ retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, nativeIssueId: heartbeatRuns.nativeIssueId, resultJson: heartbeatRuns.resultJson, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.execution.binding.runId)) .for("update") .limit(1) .then((rows) => rows[0] ?? null); if ( !boundRun || boundRun.runtimeMode !== "native" || boundRun.companyId !== input.execution.binding.companyId || boundRun.agentId !== input.execution.binding.agentId || boundRun.nativeIssueId !== input.execution.binding.issueId ) { throw new Error("native_execution_binding_changed"); } // A cancellation can win after heartbeat dispatch admission but // before this claim. Never revive a terminal run or a settled startup. if (boundRun.status !== "running" || boundRun.resultJson?.startupCancellation || coordinator.phase === "terminal_failure") { throw new NativeCancellationPendingRecoveryError(); } const cancellationIntent = record( record(boundRun.resultJson).nativeCancellation, ); if ( cancellationIntent.scope === "run" && (cancellationIntent.dispatchState === "pending" || cancellationIntent.dispatchState === "acknowledged") ) { const intentMatchesBinding = cancellationIntent.schema === "paperclip.native-cancellation.v1" && cancellationIntent.companyId === input.execution.binding.companyId && cancellationIntent.runId === input.execution.binding.runId && cancellationIntent.issueId === input.execution.binding.issueId; if (!intentMatchesBinding) throw new Error("native_cancellation_intent_conflict"); // Run cancellation remains a claim fence after dispatch ack until // the heartbeat cancellation path terminalizes the run. throw new NativeCancellationPendingRecoveryError(); } if (["committed", "applied"].includes(coordinator.phase)) throw new Error("native_run_already_committed"); if ( coordinator.leaseOwner && coordinator.leaseOwner !== leaseOwner && coordinator.leaseExpiresAt && coordinator.leaseExpiresAt > leaseNow ) throw new Error("native_finalization_lease_busy"); const recovering = input.restartRecovery; if ( recovering && (recovering.runId !== coordinator.runId || recovering.leaseOwner !== leaseOwner || coordinator.leaseOwner !== leaseOwner || coordinator.controllerBootId !== controller.bootId || coordinator.controllerPid !== controller.pid || coordinator.controllerGeneration !== recovering.controllerGeneration) ) { throw new Error("native_restart_recovery_claim_changed"); } let incidentAttempts = coordinator.attempt; if ( boundRun.retryOfRunId && boundRun.scheduledRetryReason === "native_safe_replacement" ) { const [predecessor] = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.runId, boundRun.retryOfRunId), eq( nativeRunFinalizations.companyId, input.execution.binding.companyId, ), eq( nativeRunFinalizations.issueId, input.execution.binding.issueId, ), ), ); if ( !predecessor || predecessor.failureDetail?.successorRunId !== input.execution.binding.runId || predecessor.phase !== "terminal_failure" ) throw new Error("native_replacement_lineage_invalid"); incidentAttempts = Math.max(incidentAttempts, predecessor.attempt); } const nextAttempt = nextNativeProviderAttempt( incidentAttempts, recovering?.kind, ); if (nextAttempt > 3) throw new Error("native_session_retry_exhausted"); const nextControllerGeneration = recovering ? recovering.controllerGeneration : coordinator.controllerBootId === controller.bootId ? Math.max(1, coordinator.controllerGeneration) : coordinator.controllerGeneration + 1; const claimed = await tx .update(nativeRunFinalizations) .set({ phase: "observed", attempt: nextAttempt, leaseOwner, leaseExpiresAt, controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: nextControllerGeneration, failureCode: null, failureDetail: null, nextAttemptAt: null, updatedAt: leaseNow, }) .where( and( eq(nativeRunFinalizations.runId, coordinator.runId), eq( nativeRunFinalizations.companyId, input.execution.binding.companyId, ), eq( nativeRunFinalizations.issueId, input.execution.binding.issueId, ), eq(nativeRunFinalizations.attempt, coordinator.attempt), eq(nativeRunFinalizations.phase, coordinator.phase), ), ) .returning({ runId: nativeRunFinalizations.runId }) .then((rows) => rows[0] ?? null); if (!claimed) throw new Error("native_session_lease_lost"); await tx .update(heartbeatRuns) .set({ nativePhase: "observed", nativePhaseUpdatedAt: leaseNow, updatedAt: leaseNow, }) .where(eq(heartbeatRuns.id, coordinator.runId)); return nextAttempt; }), { parentName: "task.prepare" }, ); await trace.end(taskPrepareScope); } catch (error) { await trace.end(taskPrepareScope, { outcome: "failed" }); await trace.finish("failed"); throw error; } const controlPlaneInstanceId = `${effectiveRunnerInstanceId}:control`; const planSynchronizations: PlanSynchronization[] = []; const upsertPlanSynchronization = ( synchronization: PlanSynchronization, ): void => { const existingIndex = planSynchronizations.findIndex( (candidate) => candidate.eventId === synchronization.eventId, ); if (existingIndex >= 0) { planSynchronizations[existingIndex] = synchronization; return; } planSynchronizations.push(synchronization); }; const recordPlanSynchronization = async (event: { sourceEventId: string; turnId?: string; eventType: string; payload: Record; }) => { const synchronization = await synchronizeCompletedProviderPlan({ db: input.db, execution: input.execution, event, }); if (!synchronization) return; upsertPlanSynchronization(synchronization); const activity = await persistActivity(input.db, { companyId: input.execution.binding.companyId, actorType: "agent", actorId: input.execution.binding.agentId, agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, action: "issue.document_updated", entityType: "issue", entityId: input.execution.binding.issueId, details: { key: "plan", source: "native_plan_synchronization", synchronization, }, }); publishActivity(activity.publication); if (input.onLog) await input.onLog( "stdout", `${JSON.stringify({ type: "paperclip.plan.synchronization", synchronization })}\n`, ); }; let nativeSessionExecuteStartedAtMs = Date.now(); let sessionStartedAtMs: number | null = null; let sessionStartupMode: "bootstrap" | "resume" | null = null; let turnSubmittedAtMs: number | null = null; let turnStartedAtMs: number | null = null; let firstAgentEventRecorded = false; let providerUsageLimitObserved = false; const providerFailureObservation = createNativeProviderFailureObservation(input.execution.provider); let turnCompletedAtMs: number | null = null; let runnerSessionStartupScope: NativeRunSpanScope | null = null; let agentTurnScope: NativeRunSpanScope | null = null; let taskSettleScope: NativeRunSpanScope | null = null; const governedWaitObservation = createGovernedWaitEventObservation( resolvePendingGovernedWait, ); const projectSessionGoalEvent = (event: PrpEvent) => applyRunnerGoalPrpEvent( input.db, { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, adapterType: input.adapterType ?? (input.useRunnerd ? "paperclip_runner" : "codex_local"), }, { eventType: event.eventType, sourceInstanceId: event.sourceInstanceId, sourceRunId: event.runId, sourceSeq: event.sourceSeq, payload: event.payload, }, ); const liveQuestions = createLocalNativeQuestionBridge({ db: input.db, binding: { ...input.execution.binding, normalizedSessionId: nativeSessionKey(input.execution), runnerSourceInstanceId: effectiveRunnerInstanceId }, resolve: resolveNativeRuntimeRequest, }); let observedAccountingUsage: Record | null = null; let observedAccountingTurn: string | undefined; let nativeAccountingComplete = false; // OpenCode and ACPX report their current turn; Codex reports a cumulative run delta. // Rebuild per-turn state from the durable event log when a controller resumes. const turnAccounting = ["opencode", "acpx"].includes(input.execution.provider.kind) ? createNativeTurnAccounting(input.execution.provider) : null; if (turnAccounting) { const history = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, input.execution.binding.companyId), eq(heartbeatRunEvents.runId, input.execution.binding.runId), or(sql`${heartbeatRunEvents.payload}->'prpEvent'->'payload'->>'kind' = 'usage'`, inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"])), )).orderBy(heartbeatRunEvents.seq); for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent); } const persistAccountingUsage = async (usage: Record | null, complete: boolean) => { const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }); const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined; complete = complete && accountingUsage !== undefined; const costStatus = complete && usage?.accountingCostIncomplete !== true ? input.execution.provider.kind === "aws_agentcore" ? "estimated" as const : undefined : "unpriced" as const; const billing = resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity); const costUsdExact = typeof usage?.accountingCostUsdExact === "string" ? usage.accountingCostUsdExact : undefined; await input.onUsage?.({ usage: accountingUsage, ...billing, complete, usageBasis: "per_run", costUsdExact, model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costStatus }); if (!input.onUsage) await input.db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ ...accountingUsage, ...billing, accountingReceiptReady: complete, model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costUsdExact: costUsdExact ?? null, costStatus: costStatus ?? null, usageSource: "per_run", })}::jsonb`, }).where(and(eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), isNull(heartbeatRuns.costAccountedAt))); }; // appendEvent awaits this receipt before the runtime may commit completeRun. // Replayed events must rebuild it too after a stop between event and receipt. const observeAccountingEvent = async (event: PrpEvent) => { if (turnAccounting) { if (!turnAccounting.observe(event)) return; const snapshot = turnAccounting.snapshot(); observedAccountingUsage = snapshot.usage; observedAccountingTurn = snapshot.turnId; await persistAccountingUsage(snapshot.usage, snapshot.complete); return; } const payload = record(event.payload); const usage = record(payload.usage); const delta = record(usage.runDelta); if (event.eventType === "turn.started") { observedAccountingTurn = undefined; if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false); } // A warm attachment seeds a zero snapshot before any provider report. // Only a fresh run delta for this turn can close its accounting. A later // malformed report invalidates completeness instead of retaining a // smaller earlier total or falling back to cumulative session usage. if (payload.kind === "usage") { observedAccountingTurn = undefined; if (Object.hasOwn(usage, "runDelta")) { observedAccountingUsage = { runDelta: delta, runDeltaComplete: usage.runDeltaComplete }; if (normalizeNativeUsage(observedAccountingUsage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })) { observedAccountingTurn = event.turnId; } } if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false); } if (["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType) && observedAccountingUsage && observedAccountingTurn !== undefined && event.turnId === observedAccountingTurn) { await persistAccountingUsage(observedAccountingUsage, true); } }; let completedConversationReply: PrpEvent | null = null; const assertControllerActive = () => { if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { throw new NativeControllerDetachedForRestartError(); } }; const controlPlane = new PaperclipControlPlanePort( input.db, { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, runId: input.execution.binding.runId, agentId: input.execution.binding.agentId, sessionId: nativeSessionKey(input.execution), completionContractId: input.execution.completionContract.id, completionContractSha256: input.execution.completionContract.sha256, sourceInstanceId: effectiveRunnerInstanceId, controlPlaneSourceInstanceId: controlPlaneInstanceId, }, { privateKeyPem: input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY, assertControllerActive, onCommittedEvent: async (event) => { await observeAccountingEvent(event); await toolTrace.observe(event); if (event.eventType === "item.completed" && record(event.payload).kind === "agentMessage" && record(event.payload).channel === "final") { completedConversationReply = event; } await liveQuestions.observe(event); await projectSessionGoalEvent(event); providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event); providerFailureObservation.observe(event); const eventAtMs = Date.parse(event.emittedAt); const milestoneAtMs = Number.isFinite(eventAtMs) ? eventAtMs : Date.now(); if ( event.eventType === "session.started" && sessionStartedAtMs === null ) { sessionStartedAtMs = milestoneAtMs; sessionStartupMode = "bootstrap"; await trace.record({ name: "runner.session.bootstrap", parentName: "runner.session.startup", startedAtMs: nativeSessionExecuteStartedAtMs, endedAtMs: milestoneAtMs, }); } if ( event.eventType === "turn.submitted" && turnSubmittedAtMs === null ) { turnSubmittedAtMs = milestoneAtMs; // A recovered provider session does not emit session.started again. In // that case the first durable turn.submitted event is the earliest // transport-neutral proof that runnerd reattached to the exact // provider session and is ready for work. Keep that startup time out of // runner.turn.submit so cold-resume latency is visible as its own span. if (sessionStartedAtMs === null) { await trace.record({ name: "runner.session.resume", parentName: "runner.session.startup", startedAtMs: nativeSessionExecuteStartedAtMs, endedAtMs: milestoneAtMs, attributes: { provider: input.execution.provider.kind, strategy: "exact_provider_session", }, }); sessionStartedAtMs = milestoneAtMs; sessionStartupMode = "resume"; } await trace.record({ name: "runner.turn.submit", parentName: "runner.session.startup", startedAtMs: sessionStartedAtMs, endedAtMs: milestoneAtMs, }); if (runnerSessionStartupScope) { trace.annotate(runnerSessionStartupScope, { mode: sessionStartupMode ?? "bootstrap", }); await trace.end(runnerSessionStartupScope, { endedAtMs: milestoneAtMs, }); } agentTurnScope = trace.start("agent.turn", { parentName: "native.session.execute", startedAtMs: milestoneAtMs, attributes: { provider: input.execution.provider.kind }, }); trace.activate(agentTurnScope); } if (event.eventType === "turn.started" && turnStartedAtMs === null) { turnStartedAtMs = milestoneAtMs; await trace.record({ name: "provider.turn.queue", parentName: "agent.turn", startedAtMs: turnSubmittedAtMs ?? milestoneAtMs, endedAtMs: milestoneAtMs, }); } if (!firstAgentEventRecorded && turnStartedAtMs !== null) { const kind = firstMeaningfulAgentEventKind(event); if (kind) { firstAgentEventRecorded = true; await trace.record({ name: "provider.time_to_first_agent_event", parentName: "agent.turn", startedAtMs: turnStartedAtMs, endedAtMs: milestoneAtMs, attributes: { eventKind: kind }, }); } } if ( [ "turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled", ].includes(event.eventType) && turnCompletedAtMs === null ) { turnCompletedAtMs = milestoneAtMs; if (!agentTurnScope) { agentTurnScope = trace.start("agent.turn", { parentName: "native.session.execute", startedAtMs: turnSubmittedAtMs ?? turnStartedAtMs ?? nativeSessionExecuteStartedAtMs, attributes: { provider: input.execution.provider.kind }, }); trace.activate(agentTurnScope); } const outcome = event.eventType === "turn.completed" ? "ok" : "failed"; await trace.end(agentTurnScope, { endedAtMs: milestoneAtMs, outcome, }); taskSettleScope = trace.start("task.settle", { parentName: "task.run", startedAtMs: milestoneAtMs, }); trace.activate(taskSettleScope); } if (input.onLog) await input.onLog( "stdout", `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`, ); if ( input.onEvent && [ "session.capabilities.updated", "session.goal.snapshot", "session.goal.updated", "session.goal.cleared", "turn.started", "turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled", ].includes(event.eventType) ) { await input.onEvent({ eventType: event.eventType, stream: "system", level: event.eventType.includes("failed") ? "error" : "info", payload: { ...(event.payload as Record), prpSourceSeq: event.sourceSeq, prpSourceInstanceId: event.sourceInstanceId, }, }); } const inputMetric = runtimeInputLifecycleMetric(event); if (inputMetric && input.onLog) { await input.onLog( "stdout", `${JSON.stringify({ type: "paperclip.runtime_input.metric", ...inputMetric, })}\n`, ); } const questionFallback = await materializeRuntimeQuestionFallback({ db: input.db, binding: input.execution.binding, event, }); if (questionFallback) { if (input.onLog) { const origin = record(record(record(event.payload).request).origin); await input.onLog( "stdout", `${JSON.stringify({ type: "paperclip.runtime_input.metric", outcome: record(event.payload).reason === "durable_handoff" ? "durable_handoff_materialized" : "provider_loss_materialized", requestId: record(event.payload).requestId, interactionId: questionFallback.interaction.id, adapter: typeof origin.adapter === "string" ? origin.adapter : "unknown", })}\n`, ); } } await governedWaitObservation.observe( event, event.eventType === "item.completed" || questionFallback !== null, ); await recordPlanSynchronization( event as { sourceEventId: string; turnId?: string; eventType: string; payload: Record; }, ); }, onDuplicateEvent: async (event) => { await observeAccountingEvent(event); // A crash can happen after the event commit but before its callback // finishes. Recover only idempotent durable projections here; activity, // publication, logging, trace, and metric effects remain committed-only. await liveQuestions.observe(event); await projectSessionGoalEvent(event); providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event); providerFailureObservation.observe(event); const questionFallback = await materializeRuntimeQuestionFallback({ db: input.db, binding: input.execution.binding, event, }); await governedWaitObservation.observe( event, event.eventType === "item.completed" || questionFallback !== null, ); const planSynchronization = await synchronizeCompletedProviderPlan({ db: input.db, execution: input.execution, event: event as { sourceEventId: string; turnId?: string; eventType: string; payload: Record; }, }); if (planSynchronization) { upsertPlanSynchronization(planSynchronization); } }, }, ); let native: Awaited>; let releaseGoalController: (() => void) | null = null; const releaseRegisteredGoalController = () => { // The controller is assigned from the asynchronous onSession callback. // Keep release idempotent without relying on TypeScript's synchronous // control-flow model for that callback assignment. const release: unknown = releaseGoalController; if (typeof release === "function") release(); releaseGoalController = null; }; const lifecyclePolicy = input.execution.session?.lifecyclePolicy ?? { mode: "per_turn" as const, idleTimeoutMs: null, }; const warmSessionId = lifecyclePolicy.mode === "warm" ? nativeSessionScopeKey(input.execution) : null; const warmConfigDigest = lifecyclePolicy.mode === "warm" ? nativeSessionConfigDigest( input.execution, input.runnerExecutionTarget?.kind ?? "local", ) : null; const configuredEnvironmentDigest = createHash("sha256") .update(JSON.stringify(configuredEnvironment(input.runnerEnvironment))) .digest("hex"); const warmSessionOwnerToken = Symbol( `native-warm-session:${input.execution.binding.runId}`, ); let existingWarmSession: NativeSession | undefined; let managedCredentialSession: NativeSession | undefined; let githubAccess: NativeGitHubAccess | undefined; let releaseGitHubRun: (() => void) | undefined; let persistedWarmSession: PersistedNativeSession | null | undefined; if (warmSessionId !== null && warmConfigDigest !== null) { const entry = warmNativeSessions.get(warmSessionId); if (entry) { if (entry.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy"); entry.preparingRunId = undefined; // Old run-scoped environments still require process replacement. A // session-owned broker can change run authority without replacing it. const hasBrokerCapability = Boolean( !input.managedGitHub && input.runnerEnvironment?.PAPERCLIP_GITHUB_BROKER_TOKEN, ); const credentialRunChanged = Boolean(entry.credentialRunId) !== hasBrokerCapability || (hasBrokerCapability && entry.credentialRunId !== input.execution.binding.runId); if ( input.refreshTools === true || (input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" && (entry.computerTarget?.resourceAuthority.computerId !== input.runnerExecutionTarget.resourceAuthority.computerId || entry.computerTarget?.resourceAuthority.ownerId !== input.runnerExecutionTarget.resourceAuthority.ownerId)) || entry.closeOnReleaseReason !== undefined || entry.configDigest !== warmConfigDigest || entry.configuredEnvironmentDigest !== configuredEnvironmentDigest || entry.instructionRoot !== input.instructionWorkingCopy?.root || entry.managedAiCredentialIdentity !== input.managedAiCredentialIdentity || credentialRunChanged || Boolean(entry.githubAccess) !== Boolean(input.managedGitHub) || entry.githubAccess?.ready === false || entry.githubAuthenticationMode !== input.runnerEnvironment?.PAPERCLIP_GITHUB_AUTH_MODE || entry.networkAccess !== (input.runnerEnvironment?.PAPERCLIP_RUNNER_NETWORK_ACCESS === "enabled") ) { const preparedCopy = entry.instructionPreparationRunId === input.execution.binding.runId; if (entry.busy && !preparedCopy) throw new Error("native_session_supervisor_busy"); if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.busy = true; entry.closeOnReleaseReason = "warm native session configuration changed"; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason, input.execution.binding.runId); warmNativeSessions.delete(warmSessionId); } finally { entry.busy = false; } if (preparedCopy) throw new Error("native_instruction_preparation_configuration_changed"); persistedWarmSession = loadWarmNativeCheckpoint( input.execution, warmConfigDigest, input.runnerExecutionTarget?.kind ?? "local", ); } else { if (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy"); if (entry.instructionWorkingCopy?.runId && entry.instructionPreparationRunId !== input.execution.binding.runId) { throw new Error("native_instruction_materialization_not_claimed"); } if (entry.instructionPreparationRunId !== undefined && await entry.instructionWorkingCopy!.hasChanges()) { entry.closeOnReleaseReason = "warm instruction copy changed during preparation"; try { await closeWarmNativeSession(entry, entry.closeOnReleaseReason); warmNativeSessions.delete(warmSessionId); } finally { entry.busy = false; } throw new Error("native_instruction_preparation_copy_changed"); } entry.instructionPreparationRunId = undefined; entry.instructionWorkingCopy = input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy; entry.busy = true; entry.runId = input.execution.binding.runId; entry.computerTarget = input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined; entry.ownerToken = warmSessionOwnerToken; entry.environmentId = input.runnerExecutionTarget?.environmentId ?? null; if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; existingWarmSession = entry.session; githubAccess = entry.githubAccess; } } else { persistedWarmSession = loadWarmNativeCheckpoint( input.execution, warmConfigDigest, input.runnerExecutionTarget?.kind ?? "local", ); } } async function resolvePendingGovernedWait() { const continuingInteractionIds = continuingPendingInteractionIds( input.execution, ); const interaction = await input.db .select({ id: issueThreadInteractions.id, title: issueThreadInteractions.title, summary: issueThreadInteractions.summary, }) .from(issueThreadInteractions) .where( and( eq( issueThreadInteractions.companyId, input.execution.binding.companyId, ), eq(issueThreadInteractions.issueId, input.execution.binding.issueId), or( eq( issueThreadInteractions.sourceRunId, input.execution.binding.runId, ), and( eq(issueThreadInteractions.kind, "connection_intent"), eq( issueThreadInteractions.createdByAgentId, input.execution.binding.agentId, ), ), ...(continuingInteractionIds.length > 0 ? [inArray(issueThreadInteractions.id, continuingInteractionIds)] : []), ), eq(issueThreadInteractions.status, "pending"), activeIssueInteractionCondition({ runId: input.execution.binding.runId }), // Live provider questions resume their current turn; only durable // wake-based cards park it. A timeout creates a separate fallback. sql`not (${issueThreadInteractions.kind} = 'ask_user_questions' and ${issueThreadInteractions.continuationPolicy} = 'none' and coalesce(${issueThreadInteractions.idempotencyKey}, '') like 'paperclip-runner-question:%')`, ), ) .orderBy( desc(issueThreadInteractions.createdAt), desc(issueThreadInteractions.id), ) .limit(1) .then((rows) => rows[0] ?? null); if (interaction) return nativeGovernedWaitResult({ interaction, completionContract: input.execution.completionContract.contract, }); // A ready connection can become installed after the provider snapshot was // pinned. Its already-durable wake is also a valid reason to end this turn. const [refresh] = await input.db .select({ id: agentWakeupRequests.id, key: agentWakeupRequests.idempotencyKey, }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, input.execution.binding.companyId), eq(agentWakeupRequests.agentId, input.execution.binding.agentId), like( agentWakeupRequests.idempotencyKey, `connection-intent:tools:${input.execution.binding.runId}:%`, ), notInArray(agentWakeupRequests.status, [ "skipped", "failed", "cancelled", ]), ), ) .limit(1); return refresh?.key ? nativeToolsRefreshWaitResult({ wakeId: refresh.id, key: refresh.key, completionContract: input.execution.completionContract.contract, }) : null; } const runnerExecution: NativeExecutionInput = input.execution.schema !== "paperclip.native-execution-input.v6" && input.useRunnerd && input.runnerExecutionTarget?.kind === "remote" ? { ...input.execution, workspace: { ...input.execution.workspace, cwd: input.runnerExecutionTarget.remoteCwd, }, } : input.execution; let remoteLaunchAttempted = false; let remoteBackendPreparationStarted = false; const leaseRenewal = startNativeSessionExecutionLeaseRenewal({ db: input.db, runId: input.execution.binding.runId, companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, leaseOwner, attempt, controller, }); try { if (input.managedGitHub && input.execution.provider.kind !== "openai_dot") { if (!input.execution.workspace.cwd) throw new Error("native_workspace_required"); githubAccess ??= await createNativeGitHubAccess({ scope: input.execution.binding, target: input.runnerExecutionTarget, cwd: input.execution.workspace.cwd, env: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment), resolveCredentials: (binding) => resolveGitHubOperationCredentials(input.db, binding), onLog: input.onLog, }); releaseGitHubRun = githubAccess.activate(input.execution.binding); input = { ...input, runnerEnvironment: { ...input.runnerEnvironment, ...githubAccess.env } }; } const expectedCurrentWakeComments = await resolveCurrentWakeCommentsBinding( input.db, input.execution.binding, ); remoteBackendPreparationStarted = input.useRunnerd === true && input.backend === undefined; const runnerdBackend = input.useRunnerd && input.backend === undefined ? await createRunnerdBackend({ ...input, // Durable scope and prior-run verification use the controller's // canonical workspace identity. createRunnerdBackend separately // projects remoteCwd into the provider execution boundary. execution: input.execution, runnerInstanceId: effectiveRunnerInstanceId, durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, onRemoteLaunchAttempt: () => { remoteLaunchAttempted = true; }, trace, toolTrace, }) : null; const remoteCleanupLease = input.runnerExecutionTarget?.kind === "remote" && (input.runnerExecutionTarget.transport === "sandbox" || input.runnerExecutionTarget.transport === "computer") && input.runnerExecutionTarget.leaseId ? await input.db.select({ provider: environmentLeases.provider, providerLeaseId: environmentLeases.providerLeaseId }) .from(environmentLeases).where(and( eq(environmentLeases.companyId, input.execution.binding.companyId), eq(environmentLeases.id, input.runnerExecutionTarget.leaseId), )).then(rows => rows[0]) : null; nativeSessionExecuteStartedAtMs = Date.now(); native = await trace.measure( "native.session.execute", async () => { runnerSessionStartupScope = trace.start("runner.session.startup", { parentName: "native.session.execute", startedAtMs: nativeSessionExecuteStartedAtMs, }); trace.activate(runnerSessionStartupScope); const result = await trace.run(runnerSessionStartupScope, () => executeNativeSession({ resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner", getFreshSessionHandoff: input.getFreshSessionHandoff, onSessionAdmission: async () => { // Invalidate prior stop evidence before a backend can spawn. await appendHeartbeatRunEvent(input.db, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, agentId: input.execution.binding.agentId, eventType: PROCESS_START_REQUESTED, stream: "system", level: "info", message: "Native execution requested; prior local stop evidence no longer applies.", }); }, input: runnerExecution, remoteCleanupScope: remoteCleanupLease ? remoteLeaseCleanupScope(remoteCleanupLease) : undefined, turnTimeoutMs: input.turnTimeoutMs, bootstrapTimeoutMs: nativeSessionBootstrapTimeoutMs(input.execution.provider.kind, input.runnerExecutionTarget), backend: input.backend ?? runnerdBackend ?? createNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId, onSpawn: input.onSpawn, opencodeEnvironment: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment), acpxEnvironment: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment), opencodeRuntimeDirectory: resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "opencode", ), acpxRuntimeDirectory: resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "acpx", ), }), controlPlane, runnerInstanceId: effectiveRunnerInstanceId, controlPlaneInstanceId, resolveGovernedWait: ({ event }) => { assertControllerActive(); const result = governedWaitObservation.consume(event); if (result) { const startup = nativeSessionStartups.get(input.execution.binding.runId); if (startup) startup.governedWait = true; } return result; }, resolveMissingResult: async ({ terminalEvent }) => { // Governed waits take precedence over an ordinary chat reply. // Execution tasks still require their normal semantic finish. if (terminalEvent.eventType !== "turn.completed") return null; const governedWait = await resolvePendingGovernedWait(); if (governedWait) return governedWait; if (nativeProviderLifecycle(input.execution.provider)?.planWait) { const planWait = await readNativePlanWait(input.db, input.execution.binding); if (planWait?.source.terminalEventId === terminalEvent.sourceEventId) return planWait.result; } if (await readCompletedNativePermissionDecline(input.db, input.execution.binding, terminalEvent, input.execution.provider)) { throw new NativePermissionDeclinedError(); } const [conversation] = await input.db .select({ agentId: issues.conversationAgentId }) .from(issues) .where(and( eq(issues.id, input.execution.binding.issueId), eq(issues.companyId, input.execution.binding.companyId), )) .limit(1); return nativeConversationReplyResult({ conversation: conversation?.agentId === input.execution.binding.agentId, terminalEvent, replyEvent: completedConversationReply, completionContract: input.execution.completionContract.contract, }); }, existingSession: existingWarmSession, persistedSession: persistedWarmSession, keepSessionOpen: warmSessionId !== null, sessionGoalControl: input.sessionGoalControl, resumeSessionGoalHeartbeat: input.resumeSessionGoalHeartbeat, // Every durable runner must finish its bounded suspension before // the next run verifies and rotates the saved authority. requireSessionCloseBeforeReturn: runnerdBackend !== null || input.instructionWorkingCopy !== undefined, onSessionClosed: input.instructionWorkingCopy?.collectStopped, onCheckpoint: async (snapshot) => { if (snapshot.governedWait) { const startup = nativeSessionStartups.get(input.execution.binding.runId); if (startup) startup.governedWait = true; } snapshot = identityRedactor.redact(snapshot); if (warmSessionId !== null && warmConfigDigest !== null) { await persistWarmNativeCheckpoint( input.execution, warmConfigDigest, snapshot, ); } if ( snapshot.goal || input.sessionGoalControl || input.resumeSessionGoalHeartbeat ) { await input.onGoalCheckpoint?.(snapshot); } }, onPostCompletionEnrichmentFailure: async ({ stage, error }) => { const detail = redactIdentityText( error instanceof Error ? error.message : String(error), ).slice(-4_096); await input.onLog?.( "stderr", `[paperclip-runner] post-completion ${stage} enrichment failed: ${detail}\n`, ); }, onSessionQuarantined: async (reason) => { await input.onLog?.( "stderr", `[paperclip-runner] warm native session quarantined: ${redactIdentityText(reason).slice(-1_000)}\n`, ); }, onContinuityBreak: async (continuity) => { const atMs = Date.now(); await trace.record({ name: "provider.session.continuity_break", parentName: "native.session.execute", startedAtMs: atMs, endedAtMs: atMs, outcome: "failed", attributes: { reason: continuity.reason, previousDriverSessionId: continuity.previousDriverSessionId, previousProviderSessionId: continuity.previousProviderSessionId ?? "unavailable", replacementDriverSessionId: continuity.replacementDriverSessionId, replacementProviderSessionId: continuity.replacementProviderSessionId ?? "unavailable", }, }); await input.onLog?.( "stderr", `[paperclip-runner] provider session continuity break: exact resume failed (${continuity.reason}); old driver session=${continuity.previousDriverSessionId}, old provider session=${continuity.previousProviderSessionId ?? "unavailable"}, replacement driver session=${continuity.replacementDriverSessionId}, replacement provider session=${continuity.replacementProviderSessionId ?? "unavailable"}\n`, ); }, onSession: async (session) => { if (session && input.managedAiCredentialHome) { managedCredentialSession = runnerdBackend?.bindManagedSession(session) ?? session; } releaseRegisteredGoalController(); liveQuestions.close(); if (session?.goal) { releaseGoalController = registerLiveRunnerGoalController( { companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, }, { control: async (control) => { await applyNativeSessionGoalControl(session, control); }, }, ); } if ( session && warmSessionId !== null && warmConfigDigest !== null ) { const existing = warmNativeSessions.get(warmSessionId); if (existing) { if (existing.ownerToken !== warmSessionOwnerToken) { throw new Error("native_session_supervisor_busy"); } existing.session = session; } else warmNativeSessions.set(warmSessionId, { instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy, instructionRoot: input.instructionWorkingCopy?.root, agentId: input.execution.binding.agentId, managedAiCredentialIdentity: input.managedAiCredentialIdentity, githubAuthenticationMode: input.runnerEnvironment?.PAPERCLIP_GITHUB_AUTH_MODE, networkAccess: input.runnerEnvironment ?.PAPERCLIP_RUNNER_NETWORK_ACCESS === "enabled", githubAccess, credentialRunId: !input.managedGitHub && input.runnerEnvironment ?.PAPERCLIP_GITHUB_BROKER_TOKEN ? input.execution.binding.runId : undefined, session, runId: input.execution.binding.runId, computerTarget: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined, ownerToken: warmSessionOwnerToken, configDigest: warmConfigDigest, configuredEnvironmentDigest, ownerScope: nativeSessionOwnerScope( input.execution, input.runnerExecutionTarget?.environmentId ?? null, ), companyId: input.execution.binding.companyId, environmentId: input.runnerExecutionTarget?.environmentId ?? null, busy: true, idleTimer: null, lastActivityAt: new Date().toISOString(), }); const owner = warmNativeSessions.get(warmSessionId); if (owner) { owner.runId = input.execution.binding.runId; owner.computerTarget = input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined; } if (owner && input.instructionWorkingCopy?.runId && input.instructionWorkingCopy.checkpointWarm) owner.instructionCopy = { runId: input.instructionWorkingCopy.runId, root: input.instructionWorkingCopy.root, targetIdentity: instructionTargetIdentity(input.runnerExecutionTarget), collectStopped: input.instructionWorkingCopy.collectStopped, }; } else if (!session && warmSessionId !== null) { const existing = warmNativeSessions.get(warmSessionId); // onSession(null) quarantines a transport that can no longer // be reused. Remove only this execution's generation so a // late failure cannot evict a successor session. if (existing?.ownerToken === warmSessionOwnerToken) { if (existing.idleTimer !== null) { clearTimeout(existing.idleTimer); } warmNativeSessions.delete(warmSessionId); } } if (session) { const active: ActiveNativeSession = { session, cancelRequested: false }; activeNativeSessions.set(input.execution.binding.runId, active); const startup = nativeSessionStartups.get(input.execution.binding.runId); startup?.resolve(active); if (startup?.stopRequested) { // Publication wakes the Stop caller; wait for its durable ACK // before unwinding. No provider turn may be submitted between // session startup and the execution-owned cleanup below. await startup.cancellationSettled; // If startup exceeded the caller's deadline, its late handle // must still be cancelled instead of escaping the Stop fence. await cancelNativeSession(input.execution.binding.runId, "Stop requested during native startup"); throw new Error("native_finalization_missing: session returned no semantic result"); } // Stop can win after the coordinator claim while the provider // session is still opening. Publishing the handle before this // read closes both sides of the race: earlier Stop is durable; // later Stop can cancel this exact active session. const [currentRun] = await input.db.select({ status: heartbeatRuns.status, resultJson: heartbeatRuns.resultJson, }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), )).limit(1); const cancellation = record(currentRun?.resultJson?.nativeCancellation); if (!currentRun || currentRun.status !== "running" || currentRun.resultJson?.startupCancellation || (cancellation.scope === "run" && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) { await cancelNativeSession(input.execution.binding.runId, "Run stopped during native session startup"); // The execution-owned finally closes the session when this // callback fails; no provider turn may follow publication. throw new NativeCancellationPendingRecoveryError(); } if (session.resolveRuntimeRequest) await liveQuestions.attach(); if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { if (session.detachControllerForRestart) await detachActiveNativeSessionForRestart(active); } if (active.cancelRequested) throw new NativeCancellationPendingRecoveryError(); } else { liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); clearSteeringDeliveries(input.execution.binding.runId); clearNativeRuntimeRequestResolutions( input.execution.binding.runId, ); } }, }), ); await trace.end(runnerSessionStartupScope, { outcome: result.terminal.runTerminalState === "succeeded" ? "ok" : "failed", }); return result; }, { parentName: "task.run" }, ); // A detached consumer can resolve successfully after its stream closes. // Only the replacement controller may settle the run or certify accounting. assertControllerActive(); // Persist provider accounting before any workspace/issue finalization. A // detached controller or failed finalizer must not lose a completed turn. // session.usage() may be an attachment baseline, a partial report, or a // cumulative session total. Use the observed run delta, and require its // turn to match the terminal result before certifying it as complete. if (turnAccounting) { const snapshot = turnAccounting.finish(native.turnId); observedAccountingUsage = snapshot.usage; nativeAccountingComplete = snapshot.complete && snapshot.turnId === native.turnId; } else nativeAccountingComplete = observedAccountingUsage !== null && observedAccountingTurn !== undefined && observedAccountingTurn === native.turnId; native = { ...native, usage: observedAccountingUsage }; await persistAccountingUsage(native.usage, nativeAccountingComplete); try { await completeManagedNativeCredentialTurn(managedCredentialSession); } catch { // A durable result remains successful if optional credential refresh // fails. Retire this owner so it cannot keep stale auth on a warm turn. if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true); } await input.onLog?.("stderr", "[paperclip-runner] managed credential refresh failed; provider session retired.\n"); } if (native.terminal.runTerminalState === "succeeded") { // A truncated, verified external-chat wake cannot settle from the // provider's partial inline prompt. The run-scoped reader records a // durable complete-page receipt; this fence revalidates that exact // current snapshot before any native finalization can become authoritative. await assertCurrentWakeCommentsRead( input.db, input.execution.binding, expectedCurrentWakeComments, ); } await leaseRenewal.stop(); await trace.record({ name: "native.result.finalize", parentName: "task.settle", startedAtMs: turnCompletedAtMs ?? nativeSessionExecuteStartedAtMs, endedAtMs: Date.now(), }); liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); clearSteeringDeliveries(input.execution.binding.runId); clearNativeRuntimeRequestResolutions(input.execution.binding.runId); } catch (error) { // Preserve typed failure semantics while removing credentials before any // persistence, reporting, logging, or rethrow at this boundary. if (error instanceof Error) { const originalMessage = error.message; error.message = redactIdentityText(originalMessage); if (error.stack) error.stack = redactIdentityText(originalMessage ? error.stack.split(originalMessage).join(error.message) : error.stack); } else { error = identityRedactor.redact(error); } if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { await leaseRenewal.stop().catch(() => undefined); liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); // Disconnecting deliberately ends the old event consumer. It is not a // provider failure and must not overwrite the shutdown adoption record // with a retry or release the still-live runner's lease. throw new NativeControllerDetachedForRestartError(); } if (remoteBackendPreparationStarted) { try { await retireUnlaunchedComputerAttempt(input.runnerExecutionTarget, remoteLaunchAttempted); } catch { // Failure to establish the exact tombstone leaves queued commands or // a lost launch acknowledgement ambiguous. Never retry that owner. error = new NativeSessionCleanupQuarantinedError(); } } const protocolIntegrityFailure = error instanceof NativeSessionProtocolIntegrityError ? error : null; const ownershipUnverified = nativeSessionFailureSourceCode(error) === NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT; const attemptFailureStep = async (operation: () => unknown) => { try { await operation(); } catch (secondaryError) { if (protocolIntegrityFailure === null) throw secondaryError; } }; const stoppedLeaseRenewal = leaseRenewal.stop().catch(() => undefined); try { await input.db .update(nativeRunFinalizations) .set({ controlDeadlineAt: new Date( Date.now() + EXECUTION_CONTROL_DEADLINE_MS, ), }) .where( and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), ), ); const failedAtMs = Date.now(); const executionFailureMessage = redactIdentityText( error instanceof Error ? error.message : String(error), ).slice(-4_096); if (!taskSettleScope) { taskSettleScope = trace.start("task.settle", { parentName: "task.run", startedAtMs: failedAtMs, }); } trace.activate(taskSettleScope); liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); clearSteeringDeliveries(input.execution.binding.runId); clearNativeRuntimeRequestResolutions(input.execution.binding.runId); // Stop before paperclip_finish is normal. Bounded provider teardown has // finished; a missing result must not overwrite cancellation or trigger // another turn to perform completion bookkeeping. const stoppedBeforeFirstTurn = error instanceof Error && error.message === "native_session_cancelled"; if (protocolIntegrityFailure === null && error instanceof Error && (stoppedBeforeFirstTurn || error instanceof NativePermissionDeclinedError || error.message === "native_finalization_missing: session returned no semantic result")) { const [stoppedRun] = await input.db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), eq(heartbeatRuns.nativeIssueId, input.execution.binding.issueId), )).limit(1); const stopIntent = record(stoppedRun?.resultJson?.nativeCancellation); // The backend can reject the first turn while the Stop API is still // recording its acknowledgement. Preserve that audited, exactly bound // cancellation instead of racing it with a generic provider failure. const pendingStartupStop = stoppedBeforeFirstTurn && stopIntent.schema === "paperclip.native-cancellation.v1" && stopIntent.companyId === input.execution.binding.companyId && stopIntent.runId === input.execution.binding.runId && stopIntent.issueId === input.execution.binding.issueId && stopIntent.scope === "run" && typeof stopIntent.intentAuditId === "string" && stopIntent.intentAuditId.length > 0 && ["pending", "acknowledged"].includes(String(stopIntent.dispatchState)); if (stoppedRun && (pendingStartupStop || hasAcknowledgedNativeStopIntent(stoppedRun) || hasAcknowledgedNativeReassignmentStopIntent(stoppedRun))) { const [settled] = await input.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null, leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null, updatedAt: new Date(), }).where(and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.companyId, input.execution.binding.companyId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), isNull(nativeRunFinalizations.resultId), )).returning({ runId: nativeRunFinalizations.runId }); if (settled) { await stoppedLeaseRenewal; if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true); } error = new NativeCancellationPendingRecoveryError(); } } } if ( error instanceof NativeResultPendingFinalizationError || error instanceof NativeCancellationPendingRecoveryError ) { // This is not a provider failure and must not overwrite the durable // result/coordinator state. The heartbeat boundary will either hand an // already-materialized result to the finalizer or retain the durable // cancellation intent for cancellation recovery. if (taskSettleScope) { await trace.end(taskSettleScope, { outcome: "ok" }); } await trace.finish("ok"); throw error; } const now = new Date(); const classifiedFailureCode = nativeSessionFailureSourceCode(error); const sourceFailureCode = classifiedFailureCode === "native_event_replay_conflict" ? classifiedFailureCode : providerUsageLimitObserved ? "native_provider_usage_limit" : classifiedFailureCode; const recoveryEvidence = await nativeProviderRecoveryEvidence({ db: input.db, runId: input.execution.binding.runId, sourceFailureCode, }); const disposition = nativeSessionFailureDisposition( attempt, now, sourceFailureCode, ); const phase = recoveryEvidence.recoveryMode === "ambiguous_state" ? ("terminal_failure" as const) : disposition.phase; const failureCode = recoveryEvidence.recoveryMode === "ambiguous_state" ? sourceFailureCode : disposition.failureCode; const nextAttemptAt = recoveryEvidence.recoveryMode === "ambiguous_state" ? null : disposition.nextAttemptAt; const recoveryProjection = nativeSessionRecoveryProjection({ phase, failureCode, agentId: input.execution.binding.agentId, }); const { exhausted } = recoveryProjection; const integrityFailure = sourceFailureCode === "native_event_replay_conflict"; const message = redactIdentityText( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); const sanitizedStderrTail = redactIdentityText(message).slice(-4_096); // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction // commits, so a rolled-back write never reports a false failure. let terminalRunToReport: typeof heartbeatRuns.$inferSelect | null = null; await input.db.transaction(async (tx) => { await tx.execute( sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`, ); // Use the same issue-before-run lock order as admission. A late failure // can terminalize its own run, but cannot change a reassigned task. const [failureTask] = await tx .select() .from(issues) .where( and( eq(issues.id, input.execution.binding.issueId), eq(issues.companyId, input.execution.binding.companyId), ), ) .for("update"); const updated = await tx .update(nativeRunFinalizations) .set({ phase, controlDeadlineAt: null, leaseOwner: null, leaseExpiresAt: null, recoveryState: phase === "retryable_failure" ? "resuming_session" : "blocked", failureCode, failureDetail: identityRedactor.redact({ message, originalFailureCode: error instanceof NativeProviderTerminalFailure ? error.providerCode : sourceFailureCode, recoverable: error instanceof NativeProviderTerminalFailure ? error.recoverable : phase === "retryable_failure", recoveryMode: recoveryEvidence.recoveryMode, providerSessionEstablished: recoveryEvidence.providerSessionEstablished, providerEventsExist: recoveryEvidence.providerEventsExist, checkpointExists: recoveryEvidence.checkpointExists, recoveryOwner: recoveryProjection.recoveryOwner, nextAction: sourceFailureCode === "native_permission_declined" ? "Provider permission was declined. Inspect the task and give explicit direction before starting further provider work. Automatic recovery is stopped." : sourceFailureCode === "native_provider_approval_required" ? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler." : sourceFailureCode === "native_session_cleanup_quarantined" ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." : integrityFailure ? "Inspect the persisted runner events and checkpoint for a source-sequence integrity conflict; automatic recovery is stopped." : sourceFailureCode === "native_provider_usage_limit" ? "Restore model provider usage capacity, then explicitly retry the task. Automatic retries cannot resolve an exhausted provider allowance." : ownershipUnverified ? "Inspect the retained runner's executable and authenticated connection. Do not replace its provider until ownership is safely resolved." : exhausted ? "Inspect the persisted native session after its bounded resume budget was exhausted." : recoveryEvidence.recoveryMode === "bootstrap_retry" ? "Retry provider bootstrap on this same run; durable evidence proves no provider session or provider event was created." : "Resume this same run from its exact persisted native provider checkpoint after the retry delay.", }), nextAttemptAt, recoveryHistory: sql`( select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb) from jsonb_array_elements( coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) || jsonb_build_array(${JSON.stringify({ at: now.toISOString(), disposition: phase, reason: sourceFailureCode, controllerBootId: controller.bootId, controllerGeneration: input.restartRecovery?.controllerGeneration ?? null, providerAttempt: attempt, stderrTail: sanitizedStderrTail, providerSessionEstablished: recoveryEvidence.providerSessionEstablished, checkpointExists: recoveryEvidence.checkpointExists, })}::jsonb) ) with ordinality as history(item, ordinal) where ordinal > greatest( jsonb_array_length( coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) || jsonb_build_array(${JSON.stringify({ at: now.toISOString(), disposition: phase, reason: sourceFailureCode, controllerBootId: controller.bootId, controllerGeneration: input.restartRecovery?.controllerGeneration ?? null, providerAttempt: attempt, stderrTail: sanitizedStderrTail, providerSessionEstablished: recoveryEvidence.providerSessionEstablished, checkpointExists: recoveryEvidence.checkpointExists, })}::jsonb) ) - 20, 0 ) )`, updatedAt: now, }) .where( and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq( nativeRunFinalizations.companyId, input.execution.binding.companyId, ), eq( nativeRunFinalizations.issueId, input.execution.binding.issueId, ), eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid), eq( nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt, ), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .returning({ runId: nativeRunFinalizations.runId }) .then((rows) => rows[0] ?? null); if (!updated) throw new Error("native_session_lease_lost"); const [runBeforeWrite] = await tx .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.execution.binding.runId)) .for("update"); const [updatedRun] = await tx .update(heartbeatRuns) .set({ // An authentication timeout does not prove the retained runner or // its provider stopped. Preserve physical ownership until verified. ...(!ownershipUnverified ? { status: "failed", executionStatusDeliveryId: randomUUID(), finishedAt: now, } : {}), nativePhase: phase, nativePhaseUpdatedAt: now, error: message, errorCode: ownershipUnverified ? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE : sourceFailureCode, updatedAt: now, }) .where(eq(heartbeatRuns.id, input.execution.binding.runId)) .returning(); if ( updatedRun && runBeforeWrite && updatedRun.status !== runBeforeWrite.status ) { terminalRunToReport = updatedRun; } const stillOwnsTask = failureTask?.assigneeAgentId === input.execution.binding.agentId && ["in_progress", "in_review"].includes(failureTask.status) && (!failureTask.executionRunId || failureTask.executionRunId === input.execution.binding.runId) && (!failureTask.checkoutRunId || failureTask.checkoutRunId === input.execution.binding.runId); let failureBlockStatusVersion: number | undefined; if (stillOwnsTask && recoveryProjection.issueStatus) { const projected = await issueService(tx as unknown as Db).update( input.execution.binding.issueId, { status: recoveryProjection.issueStatus }, tx, ); if (projected?.status === "blocked") failureBlockStatusVersion = projected.statusVersion; } if ( !ownershipUnverified && stillOwnsTask && phase === "terminal_failure" ) { await tx .update(issues) .set({ executionRunId: null, checkoutRunId: null, updatedAt: now }) .where(eq(issues.id, input.execution.binding.issueId)); } if (!stillOwnsTask) return; await issueRecoveryActionService( tx as unknown as Db, ).upsertSourceScoped({ companyId: input.execution.binding.companyId, sourceIssueId: input.execution.binding.issueId, kind: "active_run_watchdog", ownerType: recoveryProjection.recoveryActionOwnerType, ownerAgentId: recoveryProjection.recoveryActionOwnerAgentId, returnOwnerAgentId: input.execution.binding.agentId, cause: recoveryProjection.recoveryActionCause, fingerprint: createHash("sha256") .update(`${input.execution.binding.runId}:${failureCode}`) .digest("hex"), evidence: { runId: input.execution.binding.runId, ...(failureBlockStatusVersion !== undefined ? { nativeFailureBlock: { runId: input.execution.binding.runId, statusVersion: failureBlockStatusVersion, } } : {}), coordinatorAttempt: attempt, sourceFailureCode, recoveryDisposition: failureCode, recoveryMode: recoveryEvidence.recoveryMode, providerSessionEstablished: recoveryEvidence.providerSessionEstablished, }, nextAction: sourceFailureCode === "native_permission_declined" ? "Provider permission was declined. Inspect the task and give explicit direction before starting further provider work. Automatic recovery is stopped." : sourceFailureCode === "native_provider_approval_required" ? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler." : sourceFailureCode === "native_session_cleanup_quarantined" ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." : integrityFailure ? "Inspect the persisted runner event collision and explicitly repair or replace the run; automatic retries are disabled." : sourceFailureCode === "native_provider_usage_limit" ? "Restore model provider usage capacity, then explicitly retry the task; automatic retries are stopped." : ownershipUnverified ? "Resolve the retained runner's authentication or executable compatibility before an explicit recovery; do not blindly restart, cancel, or replace its provider session." : exhausted ? "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped." : recoveryEvidence.recoveryMode === "bootstrap_retry" ? "Retry bootstrap on the same run without manufacturing a provider checkpoint." : "Resume the exact persisted native session on the same heartbeat run.", wakePolicy: nextAttemptAt ? { kind: "resume_native_run", runId: input.execution.binding.runId, notBefore: nextAttemptAt.toISOString(), } : null, maxAttempts: 3, supersedeOnIdentityChange: recoveryProjection.supersedeOnIdentityChange, }); }); if (terminalRunToReport) void reportRunFailure(input.db, terminalRunToReport); await boundedExecutionCleanup(async () => { await stoppedLeaseRenewal; await attemptFailureStep(() => input.onLog?.( "stderr", `[paperclip-runner] native session execution failed: ${executionFailureMessage}\n`, ), ); if (runnerSessionStartupScope) { await attemptFailureStep(() => trace.end(runnerSessionStartupScope!, { endedAtMs: failedAtMs, outcome: "failed", }), ); } if (agentTurnScope) { await attemptFailureStep(() => trace.end(agentTurnScope!, { endedAtMs: failedAtMs, outcome: "failed", }), ); } if ( !ownershipUnverified && warmSessionId !== null && lifecyclePolicy.mode === "warm" ) { await attemptFailureStep(() => releaseWarmNativeSession( warmSessionId!, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true, ), ); } if (taskSettleScope) await trace.end(taskSettleScope, { outcome: "failed" }); await trace.finish("failed"); }); throw error; } finally { // Even a secondary logging/recovery-write failure cannot authorize // heartbeat to release the retained process or its task ownership. if (ownershipUnverified) throw new NativeRunnerOwnershipUnverifiedError(); if (protocolIntegrityFailure !== null) throw protocolIntegrityFailure; } } finally { releaseGitHubRun?.(); // A startup failure or onSession(null) must not leak a transport. A warm // owner retains only the inactive broker until its normal retirement. if (githubAccess && (!warmSessionId || warmNativeSessions.get(warmSessionId)?.githubAccess !== githubAccess)) { await githubAccess.stop(); } } if ( planSynchronizations.length === 0 && "executionMode" in input.execution && input.execution.executionMode === "plan" ) { const markdown = semanticProviderPlanMarkdown( native.result as unknown as Record, ); if (markdown) { const digest = createHash("sha256").update(markdown).digest("hex"); await recordPlanSynchronization({ sourceEventId: `semantic-plan:${input.execution.binding.runId}:${digest}`, ...(native.turnId ? { turnId: native.turnId } : {}), eventType: "plan.updated", payload: { schema: "paperclip.plan.updated.v1", planId: `semantic:${native.turnId ?? input.execution.binding.runId}`, revision: 1, complete: true, markdown, source: "semantic_result_artifact", }, }); } } const releaseNow = new Date(); const released = await input.db .update(nativeRunFinalizations) .set({ leaseOwner: null, leaseExpiresAt: null, recoveryState: null, recoveryRequestId: null, updatedAt: releaseNow, }) .where( and( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.companyId, input.execution.binding.companyId), eq(nativeRunFinalizations.issueId, input.execution.binding.issueId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid), eq( nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt, ), gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), ), ) .returning({ runId: nativeRunFinalizations.runId }) .then((rows) => rows[0] ?? null); if (!released) throw new Error("native_session_lease_lost"); const finalization: NativeFinalizationResult = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, companyId: input.execution.binding.companyId, result: native.result as unknown as Record, terminal: native.terminal, turnId: native.turnId, sourceInstanceId: effectiveRunnerInstanceId, normalizedSessionId: native.normalizedSessionId, providerSessionId: native.providerSessionId, driverKind: native.driverKind, driverVersion: native.driverVersion, nativeEventCount: native.nativeEventCount, highestContiguousSourceSeq: native.highestContiguousSourceSeq, workspaceFinalizeStatus: "pending", }; // A following run cannot attach until the prior run's durable finalization // is committed. Provider completion alone is not an authority boundary. if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { // A structured failed/cancelled result completes the protocol without // throwing. Heartbeat still stops its sandbox, so retire the provider now // while its transport can collect files and checkpoint the suspended runner. // Leaving it in the warm map makes the next turn retire a stopped transport. const retainSession = native.terminal.runTerminalState === "succeeded"; const instructionCopy = input.instructionWorkingCopy; const ownedSession = warmNativeSessions.get(warmSessionId); let collectInstructions: boolean; try { collectInstructions = Boolean(instructionCopy && ownedSession?.ownerToken === warmSessionOwnerToken && (instructionCopy.checkpointWarm ? !await instructionCopy.checkpointWarm() : await instructionCopy.hasChanges())); } catch (error) { // A checkpoint or its receipt callback can reject. Retire this owner // before heartbeat stops the sandbox, preserving the initiating error. await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true) .catch(() => undefined); throw error; } const collectedByOwner = ownedSession?.instructionCopy?.collectStopped === instructionCopy?.collectStopped || ownedSession?.instructionWorkingCopy?.collectStopped === instructionCopy?.collectStopped; if (collectInstructions && ownedSession) { // Keep the unchanged warm path intact. A changed private instruction copy // requires the existing checkpoint-and-close boundary before collection. ownedSession.closeOnReleaseReason = "registered instruction edits require stopped-provider collection"; } await releaseWarmNativeSession( warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, !retainSession, ); if (collectInstructions && !collectedByOwner) await instructionCopy!.collectStopped(); } const providerFailure = providerFailureObservation.forTerminal(native.turnId, native.terminal); const adapterResult: AdapterExecutionResult = { exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1, signal: null, timedOut: false, errorMessage: native.terminal.runTerminalState === "succeeded" ? null : providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`, ...(providerFailure ? { errorCode: providerFailure.errorCode } : {}), resultJson: { nativeResult: native.result as unknown as Record, nativeTerminal: native.terminal as unknown as Record, ...(input.execution.provider.kind === "openai_dot" ? { providerAccounting: { usage: null, cost: null, externallyBilled: true }, externalStopConfirmed: false } : {}), ...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}), ...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}), planSynchronizations, }, summary: native.result.summary, sessionId: native.normalizedSessionId, sessionDisplayId: native.providerSessionId ?? native.normalizedSessionId, ...resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity), model: input.execution.provider.model, usage: normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }), costUsd: input.execution.provider.kind !== "openai_dot" && normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }) ? nativeUsageCostUsd(native.usage, input.execution.provider) ?? null : null, costUsdExact: typeof native.usage?.accountingCostUsdExact === "string" ? native.usage.accountingCostUsdExact : undefined, costStatus: nativeAccountingComplete && native.usage?.accountingCostIncomplete !== true ? input.execution.provider.kind === "aws_agentcore" ? "estimated" : undefined : "unpriced", usageComplete: nativeAccountingComplete, usageBasis: "per_run", nativeFinalization: finalization, }; if (taskSettleScope) { await trace.end(taskSettleScope, { outcome: native.terminal.runTerminalState === "succeeded" ? "ok" : "failed", }); } await trace.finish( native.terminal.runTerminalState === "succeeded" ? "ok" : "failed", ); return adapterResult; } function numericUsageField( usage: Record | null, keys: string[], ): number | undefined { if (!usage) return undefined; for (const key of keys) { const value = usage[key]; if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value; } return undefined; } function nativeUsageMeasurement(usage: Record) { const nestedUsage = record(usage.usage); // An explicit partial delta must not fall through to a full session total. if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta); if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta); const candidates = [ record(usage.runDelta), record(nestedUsage.runDelta), record(usage.total), record(nestedUsage.total), record(usage.cumulative), record(nestedUsage.cumulative), nestedUsage, usage, ]; return ( candidates.find( (candidate) => numericUsageField(candidate, [ "inputTokens", "input", "promptTokens", "outputTokens", "output", "completionTokens", ]) !== undefined, ) ?? usage ); } export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string { if (provider.kind === "acpx") { if (provider.agent === "cursor") return "cursor"; if (provider.agent === "copilot") return "github"; if (provider.agent === "pi") return "openrouter"; } return "openai"; } function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) { const turns = new Map | null; terminal: boolean }>(); const sequences = new Map(); let currentTurn: string | undefined; const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]; const snapshot = () => { if (turns.size === 0) return { usage: null, turnId: undefined, complete: false }; let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0; let cost = 0n, costKnown = false, costIncomplete = false, complete = true; for (const [turnId, turn] of turns) { const normalized = normalizeNativeUsage(turn.usage); complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined; inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0); outputTokens += normalized?.outputTokens ?? 0; cacheReadTokens += normalized?.cachedInputTokens ?? 0; cacheWriteTokens += normalized?.cacheWriteTokens ?? 0; const price = nativeUsageCostUsd(turn.usage, provider); if (price === undefined) costIncomplete = true; else { cost += usdToUnits(price); costKnown = true; } } // A free empty attempt contributes no price evidence for later paid work. // Keep an incomplete zero subtotal absent so complete aggregate tokens can // still receive an estimate. Preserve positive known spend and true zero totals. const retainCost = costKnown && (!costIncomplete || cost > 0n); const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, ...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) }; const current = currentTurn ? turns.get(currentTurn) : undefined; return { usage: { runDelta, accountingCostIncomplete: costIncomplete, ...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) }, turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete }; }; return { observe(event: PrpEvent) { if (!event) return false; const payload = record(event.payload); const terminal = terminalTypes.includes(event.eventType); if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false; // Historical replay can revisit an earlier report after restoration. // Re-persist its aggregate, but never rewind or count that report twice. if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) { if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true; sequences.set(event.sourceInstanceId, event.sourceSeq); } const key = event.turnId ?? "unattributed"; const turn = turns.get(key) ?? { usage: null, terminal: false }; if (event.eventType === "turn.started") { currentTurn = event.turnId; turn.usage = null; turn.terminal = false; } else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId; if (payload.kind === "usage") { const usage = record(payload.usage); turn.usage = Object.hasOwn(usage, "runDelta") ? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null; } if (terminal) turn.terminal = true; turns.set(key, turn); return true; }, snapshot, finish(turnId: string | null) { const turn = turnId ? turns.get(turnId) : undefined; if (turn) turn.terminal = true; return snapshot(); }, }; } export function nativeUsageCostUsd( usage: Record | null, provider?: NativeExecutionInput["provider"], ) { // The ACP normalization contract fills absent per-turn cost with zero and // reports actual cost cumulatively. Until it carries an authoritative run // delta with provenance, neither value is a candidate's billed USD receipt. if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined; if (!usage) return undefined; const measurement = nativeUsageMeasurement(usage); const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta"); const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]) ?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])); // PRP currently defaults an absent providerCostUsd to zero. That zero is // not proof of a free request when the runtime observed token consumption. if (direct !== undefined && direct > 0) return direct; const normalized = normalizeNativeUsage(usage); if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0; const cost = record(hasRunDelta ? measurement.cost : usage.cost); const currency = typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; if (currency !== "USD") return undefined; return numericUsageField(cost, ["amount", "total"]); } /** Billing identity follows the selected runtime and resolved credentials. * Unknown authentication remains unknown; a provider name does not prove billing mode. */ export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick): Pick { if (identity) return identity.provider === "openai" && identity.billingType === "metered_api" ? { ...identity, biller: inferOpenAiCompatibleBiller(env, identity.biller ?? "unknown") } : identity; if (provider.kind === "claude_managed") return { provider: "anthropic", biller: "anthropic", billingType: "metered_api" }; if (provider.kind === "aws_agentcore") return { provider: "aws_agentcore", biller: "aws_agentcore", billingType: "metered_api" }; if (provider.kind === "acpx" && ["cursor", "copilot"].includes(provider.agent)) { const biller = nativeUsageBiller(provider); return { provider: biller, biller, billingType: "unknown" }; } if (provider.kind === "acpx" && provider.agent === "claude") { if (env.CLAUDE_CODE_USE_BEDROCK === "1") return { provider: "anthropic", biller: "aws_bedrock", billingType: "metered_api" }; return { provider: "anthropic", biller: "anthropic", billingType: env.ANTHROPIC_API_KEY ? "metered_api" : env.CLAUDE_CODE_OAUTH_TOKEN ? "subscription_included" : "unknown" }; } if (provider.kind === "opencode" || (provider.kind === "acpx" && provider.agent === "pi")) { const parts = (provider.model ?? "").split("/"); const biller = parts.length > 1 ? parts[0] : provider.kind === "acpx" ? nativeUsageBiller(provider) : "unknown"; const modelProvider = biller === "openrouter" && parts.length > 2 ? parts[1] : biller; return { provider: modelProvider, biller, billingType: "unknown" }; } if (provider.kind !== "codex" && !(provider.kind === "acpx" && provider.agent === "codex")) return { provider: "unknown", biller: "unknown", billingType: "unknown" }; return { provider: "openai", biller: env.OPENAI_API_KEY ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType: env.OPENAI_API_KEY ? "metered_api" : "unknown" }; } export function normalizeNativeUsage(usage: Record | null, options: { inputIncludesCacheReads?: boolean } = {}) { if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined; const measurement = nativeUsageMeasurement(usage); const cache = record(measurement.cache); const count = (source: Record, keys: string[]) => { for (const key of keys) { if (source[key] === undefined) continue; const value = source[key]; return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN; } return undefined; }; const input = count(measurement, ["inputTokens", "input", "promptTokens"]); const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]); const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]); const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]); if (input === undefined || outputTokens === undefined || ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined; const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0) + (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0); if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined; return { inputTokens, outputTokens, ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }), ...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }), }; } function processEnvironment( environment: NodeJS.ProcessEnv, ): Record { return Object.fromEntries( Object.entries(environment).filter( (entry): entry is [string, string] => typeof entry[1] === "string", ), ); } /** Preserve package-manager shims that resolve dependencies relative to argv[0]. */ export function buildRemoteCodexLauncherCommand( sourcePath: string, targetPath: string, ): string { if (sourcePath === targetPath) throw new Error("runner_remote_preinstalled_source_conflict"); const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; const launcher = `#!/bin/sh\nexec ${quote(sourcePath)} "$@"\n`; // Replace atomically: writing through an existing symlink would corrupt the // image's shared CLI, and another run may be executing this launcher already. return ( `umask 077; mkdir -p ${quote(posix.dirname(targetPath))} && ` + `[ ! -d ${quote(targetPath)} ] && ` + `paperclip_codex_launcher_tmp=$(mktemp ${quote(targetPath + ".tmp.XXXXXX")}) && ` + `trap 'rm -f "$paperclip_codex_launcher_tmp"' 0 && ` + `printf '%s' ${quote(launcher)} > "$paperclip_codex_launcher_tmp" && ` + `chmod 700 "$paperclip_codex_launcher_tmp" && ` + `mv -f "$paperclip_codex_launcher_tmp" ${quote(targetPath)}` ); } export function mayUsePreinstalledRunnerArtifact( configuredRemoteBinaryPath: string | null | undefined, ): boolean { return !configuredRemoteBinaryPath?.trim(); } const RUNNERD_BUILD_METADATA_SCHEMA = "paperclip-runner/runnerd-build-metadata/v1"; const RUNNERD_BINARY_CONTRACT_VERSION = 2; const REMOTE_PROVIDER_PACK_SCHEMA = "paperclip-runner/remote-provider-pack/v1"; const REMOTE_PROVIDER_PACK_PINS = { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion, opencode: "1.18.34", acpx: QUALIFIED_ACPX_VERSION, claudeAcp: QUALIFIED_ACPX_PROFILES.claude.agentServerVersion, codexAcp: QUALIFIED_ACPX_PROFILES.codex.agentServerVersion, grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion, } as const; const REMOTE_PROVIDER_PACK_PROFILE_DIGESTS = { grok: QUALIFIED_ACPX_PROFILES.grok.commandDigest, claude: QUALIFIED_ACPX_PROFILES.claude.commandDigest, codex: QUALIFIED_ACPX_PROFILES.codex.commandDigest, } as const; const REMOTE_PROVIDER_PACK_ARTIFACT_PATHS = { grokLauncher: "dist/providers/grok/launcher.cjs", nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", } as const; type RemoteProviderPackManifest = { schema: typeof REMOTE_PROVIDER_PACK_SCHEMA; digest: string; payload: { pins: typeof REMOTE_PROVIDER_PACK_PINS; target: { platform: string; architecture: string }; runnerSourceRevision: string; distDigest: string; bridgeDigest: string; acpxProfileDigests: typeof REMOTE_PROVIDER_PACK_PROFILE_DIGESTS; providers?: Partial>; candidateProviders?: Partial>; artifacts: { grokLauncher: { path: string; sha256: string }; nodeCommand: { path: string; sha256: string }; productionLock: { path: string; sha256: string }; opencodeCommand: { path: string; sha256: string }; opencodeExecutable: { path: string; sha256: string }; opencodeProxy: { path: string; sha256: string }; acpxSidecar: { path: string; sha256: string }; }; }; }; function sha256File(path: string): string { return `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`; } export function sha256DirectoryTree(root: string): string { const hash = createHash("sha256"); const visit = (directory: string, prefix = "") => { const entries = readdirSync(directory, { withFileTypes: true }).sort( (left, right) => left.name.localeCompare(right.name), ); for (const entry of entries) { const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; const absolutePath = join(directory, entry.name); if (entry.isDirectory()) { hash.update(`directory\0${relativePath}\n`); visit(absolutePath, relativePath); } else if (entry.isFile()) { hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`); } else if (entry.isSymbolicLink()) { hash.update( `symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`, ); } else { throw new Error( `runner_remote_provider_artifact_incompatible: unsupported dist entry ${relativePath}`, ); } } }; visit(root); return `sha256:${hash.digest("hex")}`; } function providerPackRelativePath(value: unknown, field: string): string { if ( typeof value !== "string" || value.length === 0 || value.startsWith("/") || value.includes("\\") || posix.normalize(value) !== value || value.split("/").includes("..") ) { throw new Error( `runner_remote_provider_artifact_incompatible: invalid ${field}`, ); } return value; } export function readRemoteProviderPackManifest(packRoot: string): RemoteProviderPackManifest { return readRemoteProviderPackIdentity(packRoot, true); } export function readBundledRemoteProviderPackManifest(manifestPath = bundledRemoteProviderPackManifestPath()): RemoteProviderPackManifest { const manifest = readRemoteProviderPackIdentity(dirname(manifestPath), false); if (manifest.payload.target.platform !== "linux" || manifest.payload.target.architecture !== "x64") { throw new Error("runner_remote_provider_artifact_incompatible: bundled image pack must target Linux x64"); } return manifest; } function readRemoteProviderPackIdentity(packRoot: string, verifyControllerFiles: boolean): RemoteProviderPackManifest { let manifest: RemoteProviderPackManifest; try { manifest = JSON.parse( readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"), ) as RemoteProviderPackManifest; } catch (error) { // The terminal run report keeps the outer message, not the cause chain. // Keep a bounded reason there; raw filesystem errors include private paths. const code = (error as NodeJS.ErrnoException | null)?.code; const reason = error instanceof SyntaxError ? "invalid_json" : code === "ENOENT" ? "missing" : code === "EACCES" || code === "EPERM" ? "permission_denied" : code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type" : "io_error"; throw new Error( `runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`, { cause: error }, ); } return validateRemoteProviderPackIdentity(manifest, packRoot, verifyControllerFiles); } export function validateRemoteProviderPackIdentity( manifest: RemoteProviderPackManifest, packRoot = "", verifyControllerFiles = false, retainedAuthority?: { digest: string; agent: string }, ): RemoteProviderPackManifest { const payload = manifest?.payload; // A retained session may outlive unrelated bundled-provider upgrades. Its // active provider must still satisfy this release's qualified ACPX contract. const retainedAgent = retainedAuthority?.agent; const retainedProfile = retainedAgent === "claude" || retainedAgent === "codex" || retainedAgent === "grok" ? retainedAgent : null; const activePins = retainedProfile === "claude" ? ["acpx", "claudeAcp"] as const : retainedProfile === "codex" ? ["acpx", "codex", "codexAcp"] as const : ["acpx", "grok"] as const; const pinsMatch = retainedProfile ? canonicalJson(Object.keys(payload?.pins ?? {}).sort()) === canonicalJson(Object.keys(REMOTE_PROVIDER_PACK_PINS).sort()) && Object.values(payload?.pins ?? {}).every(value => typeof value === "string" && value.length > 0 && value.length <= 120) && activePins.every(key => payload?.pins?.[key] === REMOTE_PROVIDER_PACK_PINS[key]) && canonicalJson(Object.keys(payload?.acpxProfileDigests ?? {}).sort()) === canonicalJson(Object.keys(REMOTE_PROVIDER_PACK_PROFILE_DIGESTS).sort()) && Object.values(payload?.acpxProfileDigests ?? {}).every(value => /^sha256:[0-9a-f]{64}$/.test(value)) && payload?.acpxProfileDigests?.[retainedProfile] === REMOTE_PROVIDER_PACK_PROFILE_DIGESTS[retainedProfile] : canonicalJson(payload?.pins) === canonicalJson(REMOTE_PROVIDER_PACK_PINS) && canonicalJson(payload?.acpxProfileDigests) === canonicalJson(REMOTE_PROVIDER_PACK_PROFILE_DIGESTS); if ( manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA || !payload || !pinsMatch || typeof payload.target?.platform !== "string" || typeof payload.target?.architecture !== "string" || !/^[0-9a-f]{40}(?:-dirty)?$/.test(payload.runnerSourceRevision) || !/^sha256:[0-9a-f]{64}$/.test(payload.distDigest) ) { throw new Error( "runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match", ); } const digest = `sha256:${createHash("sha256") .update(canonicalJson(payload)) .digest("hex")}`; if (manifest.digest !== digest || (retainedAuthority && digest !== retainedAuthority.digest)) { throw new Error( "runner_remote_provider_artifact_incompatible: provider pack manifest digest mismatch", ); } const artifactEntries = [ ["Grok builtin launcher", payload.artifacts?.grokLauncher, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.grokLauncher], [ "provider Node", payload.artifacts?.nodeCommand, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.nodeCommand, ], [ "production lockfile", payload.artifacts?.productionLock, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.productionLock, ], [ "OpenCode command", payload.artifacts?.opencodeCommand, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeCommand, ], [ "OpenCode executable", payload.artifacts?.opencodeExecutable, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeExecutable, ], [ "OpenCode proxy", payload.artifacts?.opencodeProxy, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeProxy, ], [ "ACPX sidecar", payload.artifacts?.acpxSidecar, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.acpxSidecar, ], ] as const; for (const [label, artifact, expectedPath] of artifactEntries) { const artifactPath = providerPackRelativePath( artifact?.path, `${label} path`, ); if (artifactPath !== expectedPath) { throw new Error( `runner_remote_provider_artifact_incompatible: ${label} path must be ${expectedPath}`, ); } if ( !/^sha256:[a-f0-9]{64}$/.test(artifact?.sha256 ?? "") || (verifyControllerFiles && sha256File(resolve(packRoot, artifactPath)) !== artifact.sha256) ) { throw new Error( `runner_remote_provider_artifact_incompatible: ${label} digest mismatch`, ); } } if (verifyControllerFiles && sha256DirectoryTree(resolve(packRoot, "dist")) !== payload.distDigest) { throw new Error( "runner_remote_provider_artifact_incompatible: provider dist tree digest mismatch", ); } for (const [inventory, candidates] of [["providers", payload.providers], ["candidateProviders", payload.candidateProviders]] as const) { if (candidates === undefined) continue; if (!candidates || typeof candidates !== "object" || Array.isArray(candidates) || Object.keys(candidates).length > 3) { throw new Error("runner_remote_provider_artifact_incompatible: invalid candidate inventory"); } for (const [provider, candidate] of Object.entries(candidates)) { const expectedPath = `provider-assets/${provider}/${payload.target.platform}-${payload.target.architecture}`; if (!(inventory === "providers" ? ["pi", "cursor"] : ["cursor", "copilot", "pi"]).includes(provider) || !candidate || Object.keys(candidate).some(key => !["version", "profileDigest", "closureDigest", "qualification", "path", "sha256"].includes(key)) || candidate.qualification !== (["pi", "cursor"].includes(provider) ? "qualified" : "pending") || candidate.path !== expectedPath || typeof candidate.version !== "string" || !candidate.version || candidate.version.length > 120 || !/^sha256:[a-f0-9]{64}$/.test(candidate.profileDigest) || !/^sha256:[a-f0-9]{64}$/.test(candidate.closureDigest) || !/^sha256:[a-f0-9]{64}$/.test(candidate.sha256)) { throw new Error("runner_remote_provider_artifact_incompatible: invalid candidate identity"); } const candidatePath = providerPackRelativePath(candidate.path, "candidate assets"); if (provider === "cursor" && !retainedProfile) { const target = `${payload.target.platform}-${payload.target.architecture}`; const distribution = Object.hasOwn(CURSOR_DISTRIBUTION_PINS, target) ? CURSOR_DISTRIBUTION_PINS[target as keyof typeof CURSOR_DISTRIBUTION_PINS] : undefined; if (!distribution || candidate.version !== QUALIFIED_ACPX_PROFILES.cursor.agentServerVersion || candidate.profileDigest !== QUALIFIED_ACPX_PROFILES.cursor.commandDigest || candidate.closureDigest !== `sha256:${distribution.closureSha256}`) { throw new Error("runner_remote_provider_artifact_incompatible: Cursor profile or closure does not match this release"); } } if (verifyControllerFiles && sha256DirectoryTree(resolve(packRoot, candidatePath)) !== candidate.sha256) { throw new Error("runner_remote_provider_artifact_incompatible: candidate asset tree digest mismatch"); } } } const bridgeDigest = `sha256:${createHash("sha256") .update(payload.artifacts.opencodeProxy.sha256) .update("\n") .update(payload.artifacts.acpxSidecar.sha256) .update("\n") .update(payload.distDigest) .digest("hex")}`; if (payload.bridgeDigest !== bridgeDigest) { throw new Error( "runner_remote_provider_artifact_incompatible: provider bridge digest mismatch", ); } return structuredClone(manifest); } export function assertRemoteRunnerBuildMetadata( value: unknown, requiredMode: "dial_wss" | "listen_ws", requiredExternalProvider?: "openai_dot_mcp", ): void { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error("runner_remote_artifact_metadata_invalid"); } const metadata = value as Record; if ( metadata.schema !== RUNNERD_BUILD_METADATA_SCHEMA || metadata.binaryName !== "paperclip-runnerd" || metadata.packageName !== "@paperclipai/paperclip-runner" || metadata.binaryContractVersion !== RUNNERD_BINARY_CONTRACT_VERSION ) { throw new Error("runner_remote_artifact_contract_incompatible"); } // Older sandbox images share binary contract v2, but reject the zero // lifetime now used by the controller and cannot renew connection leases. // Reject them before launch so preparation stages the bundled runner instead. if (requiredExternalProvider && (!Array.isArray(metadata.externalProviderCapabilities) || !metadata.externalProviderCapabilities.includes(requiredExternalProvider))) { throw new Error("runner_remote_dot_capability_missing: install the current Paperclip Runner artifact"); } const sessionCapabilities = Array.isArray(metadata.durableSessionCapabilities) ? metadata.durableSessionCapabilities : []; for (const capability of ["unlimited_runtime", "connection_lease_renewal"]) { if (!sessionCapabilities.includes(capability)) { throw new Error(`runner_remote_session_capability_missing:${capability}`); } } const modes = Array.isArray(metadata.prpTransportModes) ? metadata.prpTransportModes : []; if (!modes.includes(requiredMode)) { throw new Error( `runner_remote_transport_capability_missing:${requiredMode}`, ); } } export function nativeComputerToolForExecution( execution: NativeExecutionInput, target: AdapterExecutionTarget | null | undefined, ): AdapterComputerExecutionTarget["computerTool"] | undefined { if (target?.kind !== "remote" || target.transport !== "computer" || execution.task.workMode !== "standard" || ("executionMode" in execution && execution.executionMode === "plan")) return undefined; return target.computerTool; } export async function stageRemoteRunnerFile(input: { target: Extract; runner: CommandManagedRuntimeRunner; sourcePath: string; targetPath: string; mode: number; }): Promise { const runner = input.runner; if (runner.syncIn) { await runner.syncIn([ { operationId: `runner-stage-${randomUUID()}`, files: [ { sourcePath: input.sourcePath, targetPath: input.targetPath, kind: "file", mode: input.mode, }, ], }, ]); return; } const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; const run = async (script: string, stdin?: string) => { const result = await runner.execute({ command: "sh", args: ["-c", script], stdin, bypassSession: true, timeoutMs: 180_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error(`runner_remote_staging_failed: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`); } }; if (lstatSync(input.sourcePath).size <= 4 * 1024 * 1024) { await run(`umask 077; mkdir -p ${quote(posix.dirname(input.targetPath))} && ` + `base64 -d > ${quote(input.targetPath)} && chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`, readFileSync(input.sourcePath).toString("base64")); return; } const remoteUpload = `${input.targetPath}.upload-${randomUUID()}`; try { await uploadRemoteRunnerFileChunks({ sourcePath: input.sourcePath, remotePath: remoteUpload, run }); await run(`chmod ${input.mode.toString(8)} ${quote(remoteUpload)} && mv -f ${quote(remoteUpload)} ${quote(input.targetPath)}`); } finally { await run(`rm -f ${quote(remoteUpload)}`).catch(() => undefined); } } const REMOTE_ARTIFACT_UPLOAD_CONCURRENCY = 8; let activeRemoteArtifactUploads = 0; const remoteArtifactUploadWaiters: Array<() => void> = []; async function withRemoteArtifactUploadSlot(operation: () => Promise): Promise { await new Promise((resolve) => { if (activeRemoteArtifactUploads < REMOTE_ARTIFACT_UPLOAD_CONCURRENCY) { activeRemoteArtifactUploads++; resolve(); } else remoteArtifactUploadWaiters.push(resolve); }); try { await operation(); } finally { const next = remoteArtifactUploadWaiters.shift(); if (next) next(); else activeRemoteArtifactUploads--; } } async function uploadRemoteRunnerFileChunks(input: { sourcePath: string; remotePath: string; run: (script: string, stdin?: string) => Promise; onProgress?: (completedBytes: number, totalBytes: number) => Promise; }): Promise { const { run, remotePath } = input; const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; const pending: Promise[] = []; let uploadError: unknown; try { await run(`umask 077; mkdir -p ${quote(posix.dirname(remotePath))} && : > ${quote(remotePath)}`); const digest = createHash("sha256"); const totalBytes = lstatSync(input.sourcePath).size; let completedBytes = 0; let progressAt = Date.now(); await input.onProgress?.(0, totalBytes); let chunkIndex = 0; const stream = createReadStream(input.sourcePath, { highWaterMark: 4 * 1024 * 1024 }); for await (const bytes of stream) { const chunk = bytes as Buffer; digest.update(chunk); completedBytes += chunk.length; // Disjoint fixed offsets permit bounded parallel upload and idempotent // writes. The final digest rejects missing, duplicated, or partial bytes. const offset = chunkIndex++; pending.push(withRemoteArtifactUploadSlot(() => run( `base64 -d | dd of=${quote(remotePath)} bs=4194304 seek=${offset} conv=notrunc 2>/dev/null`, chunk.toString("base64"))).catch((error) => { uploadError ??= error; })); if (pending.length === REMOTE_ARTIFACT_UPLOAD_CONCURRENCY) { await Promise.all(pending); pending.length = 0; if (uploadError) throw uploadError; if (Date.now() - progressAt >= 15_000) { await input.onProgress?.(completedBytes, totalBytes); progressAt = Date.now(); } } } await Promise.all(pending); if (uploadError) throw uploadError; await input.onProgress?.(totalBytes, totalBytes); const expectedDigest = digest.digest("hex"); await run(`test "$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${quote(remotePath)}; else shasum -a 256 ${quote(remotePath)}; fi | cut -d ' ' -f 1)" = ${quote(expectedDigest)}`); } finally { // Do not remove or rename the upload while another offset write is active. await Promise.all(pending); } } function archiveExcludeArgs(entries: readonly string[]): string[] { for (const entry of entries) { const segments = entry.split("/"); if ( entry.length === 0 || entry.startsWith("/") || segments.some( (segment) => segment === "" || segment === "." || segment === ".." || !/^[A-Za-z0-9._-]+$/.test(segment), ) ) { throw new Error("runner_remote_checkpoint_exclusion_invalid"); } } return entries.map((entry) => `--exclude=./${entry}`); } export async function stageRemoteRunnerDirectory(input: { target: Extract; runner: CommandManagedRuntimeRunner; sourcePath: string; targetPath: string; mode: number; excludeEntries?: readonly string[]; onProgress?: (completedBytes: number, totalBytes: number) => Promise; }): Promise { const excludeArgs = archiveExcludeArgs(input.excludeEntries ?? []); if (input.runner.syncIn) { let stagingRoot: string | null = null; let sourcePath = input.sourcePath; try { if (excludeArgs.length > 0) { stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-restore-")); const archive = execFileSync( "tar", [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, ); execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], { input: archive, maxBuffer: 64 * 1024 * 1024, }); sourcePath = stagingRoot; } await input.runner.syncIn([ { operationId: `runner-stage-dir-${randomUUID()}`, files: [ { sourcePath, targetPath: input.targetPath, kind: "directory", mode: input.mode, }, ], }, ]); } finally { if (stagingRoot) rmSync(stagingRoot, { recursive: true, force: true }); } return; } const stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-upload-")); const archivePath = join(stagingRoot, "payload.tar.gz"); const remoteArchive = `${input.targetPath}.upload-${randomUUID()}.tar.gz`; const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; const run = async (script: string, stdin?: string) => { const result = await input.runner.execute({ command: "sh", args: ["-c", script], stdin, bypassSession: true, timeoutMs: 180_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error(`runner_remote_directory_staging_failed: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`); } }; try { // Provider packs can exceed a gigabyte. Keep compression off the event loop // and transfer from disk in bounded chunks instead of buffering the archive. await new Promise((resolve, reject) => { execFile("tar", [...excludeArgs, "-czf", archivePath, "-C", input.sourcePath, "."], { maxBuffer: 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } }, (error) => error ? reject(error) : resolve()); }); await uploadRemoteRunnerFileChunks({ sourcePath: archivePath, remotePath: remoteArchive, run, onProgress: input.onProgress, }); await run(`umask 077 && mkdir -p ${quote(input.targetPath)} && ` + `tar -xzf ${quote(remoteArchive)} -C ${quote(input.targetPath)} && chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`); } finally { rmSync(stagingRoot, { recursive: true, force: true }); await run(`rm -f ${quote(remoteArchive)}`).catch(() => undefined); } } async function remoteRunnerPathExists(input: { runner: CommandManagedRuntimeRunner; path: string; kind: "file" | "directory"; }): Promise { const escapedPath = input.path.replaceAll("'", "'\\''"); const result = await input.runner.execute({ command: "sh", args: ["-c", `test -${input.kind === "file" ? "f" : "d"} '${escapedPath}'`], bypassSession: true, timeoutMs: 10_000, }); return result.exitCode === 0 && !result.timedOut; } function assertSafeRemoteCheckpointArchive(archive: Buffer): void { if (archive.length > MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES) { throw new Error("runner_remote_checkpoint_archive_too_large"); } let names: string[]; let verboseEntries: string[]; try { names = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8", maxBuffer: 8 * 1024 * 1024, timeout: 30_000, }) .split("\n") .filter((line) => line.length > 0); verboseEntries = execFileSync("tar", ["-tvzf", "-"], { input: archive, encoding: "utf8", maxBuffer: 8 * 1024 * 1024, timeout: 30_000, }) .split("\n") .filter((line) => line.length > 0); } catch { throw new Error("runner_remote_checkpoint_archive_invalid"); } if ( names.length === 0 || names.length > MAX_REMOTE_CHECKPOINT_ENTRIES || verboseEntries.length !== names.length ) { throw new Error("runner_remote_checkpoint_archive_invalid"); } for (const name of names) { const normalized = name.replace(/^\.\//, ""); if ( name.includes("\0") || name.startsWith("/") || normalized.split("/").some((part) => part === "..") ) { throw new Error("runner_remote_checkpoint_archive_unsafe_path"); } } for (const entry of verboseEntries) { const type = entry.trimStart()[0]; if (type !== "-" && type !== "d") { throw new Error("runner_remote_checkpoint_archive_unsafe_entry"); } } try { execFileSync("tar", ["-xOzf", "-"], { input: archive, stdio: ["pipe", "pipe", "pipe"], maxBuffer: MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES, timeout: 60_000, }); } catch { throw new Error("runner_remote_checkpoint_archive_expanded_too_large"); } } function assertSafeExtractedCheckpoint(root: string): void { const pending = [root]; let entries = 0; let totalBytes = 0; while (pending.length > 0) { const directory = pending.pop()!; for (const entry of readdirSync(directory, { withFileTypes: true })) { entries += 1; if (entries > MAX_REMOTE_CHECKPOINT_ENTRIES) { throw new Error("runner_remote_checkpoint_too_many_entries"); } const path = join(directory, entry.name); const metadata = lstatSync(path); if ( metadata.isSymbolicLink() || (!metadata.isDirectory() && !metadata.isFile()) ) { throw new Error("runner_remote_checkpoint_unsafe_entry"); } if (metadata.isDirectory()) { pending.push(path); } else { totalBytes += metadata.size; if (totalBytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES) { throw new Error("runner_remote_checkpoint_expanded_too_large"); } } } } } /** * Copy a remote runner directory into durable local state. * * Some provider homes contain process-local scratch trees (for example Codex's * `tmp/arg0` executable aliases). Those aliases can point outside the directory * and are neither portable nor required to resume a provider session. When a * provider supplies a native syncOut implementation, create a sibling snapshot * first so the live runtime remains untouched and the provider's archive safety * checks still apply to every persisted entry. */ export async function syncRemoteRunnerDirectoryOut(input: { runner: CommandManagedRuntimeRunner; sourcePath: string; targetPath: string; mode: number; excludeEntries?: readonly string[]; }): Promise { if ( !(await remoteRunnerPathExists({ runner: input.runner, path: input.sourcePath, kind: "directory", })) ) return; mkdirSync(resolve(input.targetPath, ".."), { recursive: true, mode: 0o700 }); const excluded = input.excludeEntries ?? []; const excludeArgs = archiveExcludeArgs(excluded) .map((argument) => `'${argument}'`) .join(" "); if (input.runner.syncOut) { let syncSourcePath = input.sourcePath; let snapshotPath: string | null = null; if (excluded.length > 0) { const checkpointId = randomUUID(); snapshotPath = posix.join( posix.dirname(input.sourcePath), `.paperclip-checkpoint-${checkpointId}`, ); const archivePath = `${snapshotPath}.tar`; const escapedSource = input.sourcePath.replaceAll("'", "'\\''"); const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''"); const escapedArchive = archivePath.replaceAll("'", "'\\''"); const snapshotResult = await input.runner.execute({ command: "sh", args: [ "-c", `set -e; umask 077; mkdir -p '${escapedSnapshot}'; ` + `trap "rm -f '${escapedArchive}'" EXIT; ` + `tar ${excludeArgs} -cf '${escapedArchive}' -C '${escapedSource}' .; ` + `tar -xf '${escapedArchive}' -C '${escapedSnapshot}'`, ], bypassSession: true, timeoutMs: 120_000, }); if (snapshotResult.exitCode !== 0 || snapshotResult.timedOut) { throw new Error("runner_remote_checkpoint_snapshot_failed"); } syncSourcePath = snapshotPath; } try { await input.runner.syncOut([ { operationId: `runner-checkpoint-dir-${randomUUID()}`, files: [ { sourcePath: syncSourcePath, targetPath: input.targetPath, kind: "directory", mode: input.mode, }, ], }, ]); } finally { if (snapshotPath) { const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''"); await input.runner .execute({ command: "sh", args: ["-c", `rm -rf -- '${escapedSnapshot}'`], bypassSession: true, timeoutMs: 30_000, }) .catch(() => undefined); } } return; } const escapedSource = input.sourcePath.replaceAll("'", "'\\''"); const result = await input.runner.execute({ command: "sh", args: ["-c", `tar ${excludeArgs} -czf - -C '${escapedSource}' . | base64`], bypassSession: true, timeoutMs: 120_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_remote_checkpoint_failed"); } const archive = Buffer.from(result.stdout.replace(/\s+/g, ""), "base64"); assertSafeRemoteCheckpointArchive(archive); const parent = resolve(input.targetPath, ".."); const stagingRoot = mkdtempSync(join(parent, ".paperclip-checkpoint-")); const stagedTarget = join(stagingRoot, "payload"); const previousTarget = join( parent, `.paperclip-checkpoint-previous-${randomUUID()}`, ); let previousMoved = false; let replacementInstalled = false; try { mkdirSync(stagedTarget, { recursive: true, mode: input.mode }); execFileSync( "tar", [ "--no-same-owner", "--no-same-permissions", "-xzf", "-", "-C", stagedTarget, ], { input: archive, maxBuffer: 8 * 1024 * 1024, timeout: 60_000 }, ); assertSafeExtractedCheckpoint(stagedTarget); chmodSync(stagedTarget, input.mode); if (existsSync(input.targetPath)) { renameSync(input.targetPath, previousTarget); previousMoved = true; } renameSync(stagedTarget, input.targetPath); replacementInstalled = true; if (previousMoved) { rmSync(previousTarget, { recursive: true, force: true }); previousMoved = false; } } catch (error) { if ( previousMoved && !replacementInstalled && !existsSync(input.targetPath) ) { try { renameSync(previousTarget, input.targetPath); previousMoved = false; } catch { // Leave the last durable checkpoint at previousTarget. Deleting it in // finally would turn a failed replacement into irreversible data loss. } } throw error; } finally { if (previousMoved && replacementInstalled) { rmSync(previousTarget, { recursive: true, force: true }); } rmSync(stagingRoot, { recursive: true, force: true }); } } type RetiredComputerCheckpointInput = { execution: NativeExecutionInput; priorExecution: NativeExecutionInput; identity: RunnerdDurableIdentity; runner: Record; control: Record; provider: Record; checkpoint: Record; currentCheckpoint: Record; }; type RetiredComputerCheckpointUnsettledReason = | "provider_kind" | "runner_schema" | "runner_lifecycle" | "runner_identity" | "controller_identity" | "runner_outbox" | "runner_terminal_delivery" | "controller_commands" | "controller_terminal_event" | "controller_terminal_identity" | "provider_terminal_authority" | "prior_run_identity" | "issue_scope" | "session_scope" | "session_config" | "provider_session_identity" | `${"prior" | "current"}_checkpoint_${"provider_session" | "driver" | "active_turn" | "runtime_requests" | "run_identity" | "session_identity" | "company_identity" | "agent_identity" | "issue_identity"}`; /** Fixed reason codes are safe to surface in run logs; never include checkpoint * values, provider configuration, prompts, paths, or credentials in diagnostics. */ export function retiredComputerCheckpointUnsettledReason( input: RetiredComputerCheckpointInput, ): RetiredComputerCheckpointUnsettledReason | null { const { execution, priorExecution, identity, runner, control, provider, checkpoint, currentCheckpoint } = input; if (execution.provider.kind !== "codex" || priorExecution.provider.kind !== "codex") return "provider_kind"; const events = control.committedEvents; const commands = control.commands; const terminal = record(record(Array.isArray(events) ? events.at(-1) : null).envelope); const fields = ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"] as const; if (runner.schema !== RUNNERD_STATE_SCHEMA) return "runner_schema"; if (runner.lifecycle !== "ready") return "runner_lifecycle"; if (fields.some(key => runner[key] !== identity[key])) return "runner_identity"; if (fields.some(key => record(control.identity)[key] !== identity[key])) return "controller_identity"; if (!Array.isArray(runner.outbox) || runner.outbox.length !== 0) return "runner_outbox"; if (runner.pendingTerminalDelivery !== null) return "runner_terminal_delivery"; if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed")) return "controller_commands"; if (!Array.isArray(events) || record(events.at(-1)).eventType !== "run.terminal") return "controller_terminal_event"; if (terminal.runId !== identity.runId || terminal.normalizedSessionId !== identity.normalizedSessionId) return "controller_terminal_identity"; if (provider.completedTurnAuthoritative !== true) return "provider_terminal_authority"; if (priorExecution.binding.runId !== identity.runId) return "prior_run_identity"; if (priorExecution.binding.issueId !== execution.binding.issueId) return "issue_scope"; if (nativeSessionScopeKey(priorExecution) !== nativeSessionScopeKey(execution)) return "session_scope"; if (nativeSessionConfigDigest(priorExecution) !== nativeSessionConfigDigest(execution)) return "session_config"; const providerIdentity = providerSessionIdentityFromDurableProviderState({ execution: priorExecution, providerState: provider }); if (!providerIdentity.providerSessionId) return "provider_session_identity"; for (const [source, value, expectedRunId] of [ ["prior", checkpoint, identity.runId], ["current", currentCheckpoint, execution.binding.runId], ] as const) { const binding = record(value.identity); if (value.providerSessionId !== providerIdentity.providerSessionId) return `${source}_checkpoint_provider_session`; if (value.driverKind !== priorExecution.session.driverKind) return `${source}_checkpoint_driver`; if (value.activeTurnId !== null) return `${source}_checkpoint_active_turn`; if (!Array.isArray(value.pendingRuntimeRequests) || value.pendingRuntimeRequests.length !== 0) return `${source}_checkpoint_runtime_requests`; if (binding.runId !== expectedRunId) return `${source}_checkpoint_run_identity`; if (binding.sessionId !== identity.normalizedSessionId) return `${source}_checkpoint_session_identity`; if (binding.companyId !== execution.binding.companyId) return `${source}_checkpoint_company_identity`; if (binding.agentId !== execution.binding.agentId) return `${source}_checkpoint_agent_identity`; if (binding.issueId !== execution.binding.issueId) return `${source}_checkpoint_issue_identity`; } return null; } /** A dead computer runner may retain a ready checkpoint after controller loss. * Match the local quiescent-recovery proof before sealing it for normal rotation. */ export function retiredComputerCheckpointIsSettled(input: RetiredComputerCheckpointInput): boolean { return retiredComputerCheckpointUnsettledReason(input) === null; } async function sealRetiredComputerRunnerState(input: { db: Db; execution: NativeExecutionInput; root: string; target: Extract; runner: CommandManagedRuntimeRunner; stateDirectory: string; }): Promise { const identity = readRunnerdDurableIdentity(input.root); if (!identity || identity.runId === input.execution.binding.runId || input.execution.provider.kind !== "codex") return; if (!durableIdentityMatchesSession(identity, input.execution)) throw new Error("runner_remote_retirement_scope_mismatch"); const controlPath = resolve(input.root, "control-plane", "control-plane-state.json"); const controlBytes = readBoundedNativeFile(controlPath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "recovery_state_too_large"); const runnerPath = posix.join(input.stateDirectory, "runner-state.json"); const providerPath = posix.join(input.stateDirectory, runnerProviderStateFilename(input.execution)); const read = await input.runner.execute({ command: "python3", args: ["-c", String.raw` import sys,os,stat,json,base64 rows=[] for p in sys.argv[1:]: s=os.lstat(p) if not stat.S_ISREG(s.st_mode) or s.st_size>16777216:raise RuntimeError('unsafe recovery state') with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:data=f.read(16777217) if len(data)>16777216:raise RuntimeError('unsafe recovery state') rows.append(base64.b64encode(data).decode()) print(json.dumps(rows))`, runnerPath, providerPath], bypassSession: true, timeoutMs: 10_000 }); if (read.exitCode !== 0 || read.timedOut) throw new Error("runner_remote_retirement_state_unavailable"); const [runnerBytes, providerBytes] = (JSON.parse(read.stdout) as string[]).map(value => Buffer.from(value, "base64")); const runner = record(JSON.parse(runnerBytes!.toString("utf8"))); if (runner.lifecycle === "suspended") return; const runs = await input.db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, runnerProfileJson: heartbeatRuns.runnerProfileJson }) .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), inArray(heartbeatRuns.id, [identity.runId, input.execution.binding.runId]))); const prior = runs.find(run => run.id === identity.runId); const current = runs.find(run => run.id === input.execution.binding.runId); if (!prior || !current || !TERMINAL_HEARTBEAT_RUN_STATUSES.has(prior.status)) throw new Error("runner_remote_retirement_authority_unverified"); const profile = record(prior.runnerProfileJson); const previous = parseNativeExecutionInput(profile.nativeExecutionInput); const unsettledReason = retiredComputerCheckpointUnsettledReason({ execution: input.execution, priorExecution: previous, identity, runner, control: record(JSON.parse(controlBytes.toString("utf8"))), provider: record(JSON.parse(providerBytes!.toString("utf8"))), checkpoint: record(profile.sessionCheckpoint), currentCheckpoint: record(record(current.runnerProfileJson).sessionCheckpoint) }); if (unsettledReason !== null) throw new Error(`runner_remote_retirement_checkpoint_unsettled: ${unsettledReason}`); const priorAuthority = await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: input.root, identity, execution: input.execution, allowVerifiedBackup: false, allowRetainedWarmRunner: false, remoteRunnerState: true, runnerExecutionTarget: input.target }); if (priorAuthority !== "verified") throw new Error(`runner_remote_retirement_checkpoint_unsettled: prior_authority_${priorAuthority}`); // Recheck exact durable retirement after all awaited reads; never infer death // from a missing PID or turn status. The compare-and-swap preserves ambiguity. const workspace = readNativeComputerWorkspaceReference(profile.nativeComputerWorkspace); if (!workspace || !input.target.environmentId || !(await computerService(input.db).isRetired({ companyId: input.execution.binding.companyId, environmentId: input.target.environmentId, owner: workspace.computerOwner, agentId: input.execution.binding.agentId, runId: identity.runId, })) || !controlBytes.equals(readBoundedNativeFile(controlPath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "recovery_state_too_large"))) throw new Error("runner_remote_retirement_evidence_changed"); await compareAndSealRetiredComputerCheckpoint({ runner: input.runner, runnerPath, providerPath, runnerBytes: runnerBytes!, providerBytes: providerBytes! }); } /** Exact retirement is checked by the caller; publication must still compare * the complete bytes so a changed or interrupted checkpoint is never rotated. */ export async function compareAndSealRetiredComputerCheckpoint(input: { runner: CommandManagedRuntimeRunner; runnerPath: string; providerPath: string; runnerBytes: Buffer; providerBytes: Buffer; }): Promise { const result = await input.runner.execute({ command: "python3", args: ["-c", String.raw` import sys,os,stat,json,hashlib,tempfile runner,provider,expected_runner,expected_provider=sys.argv[1:] for p,digest in [(runner,expected_runner),(provider,expected_provider)]: s=os.lstat(p) if not stat.S_ISREG(s.st_mode) or s.st_size>16777216:raise RuntimeError('unsafe recovery state') with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:data=f.read(16777217) if len(data)>16777216:raise RuntimeError('unsafe recovery state') if hashlib.sha256(data).hexdigest()!=digest:raise RuntimeError('recovery evidence changed') if p==runner:state=json.loads(data) state['lifecycle']='suspended' fd,tmp=tempfile.mkstemp(prefix='.retired-checkpoint-',dir=os.path.dirname(runner)) try: with os.fdopen(fd,'w') as f:json.dump(state,f,separators=(',',':'));f.flush();os.fsync(f.fileno()) # Recheck immediately before publication, including the provider checkpoint. for p,digest in [(runner,expected_runner),(provider,expected_provider)]: with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:current=f.read(16777217) if hashlib.sha256(current).hexdigest()!=digest:raise RuntimeError('recovery evidence changed') os.replace(tmp,runner) finally: if os.path.exists(tmp):os.unlink(tmp) `, input.runnerPath, input.providerPath, createHash("sha256").update(input.runnerBytes).digest("hex"), createHash("sha256").update(input.providerBytes).digest("hex")], bypassSession: true, timeoutMs: 10_000 }); if (result.exitCode !== 0 || result.timedOut) throw new Error("runner_remote_retirement_checkpoint_cas_failed"); } async function readRemoteRunnerState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; }): Promise> { const statePath = posix.join(input.stateDirectory, "runner-state.json"); const escapedPath = statePath.replaceAll("'", "'\\''"); const result = await input.runner.execute({ command: "sh", args: ["-c", `test -f '${escapedPath}' && base64 < '${escapedPath}'`], bypassSession: true, timeoutMs: 10_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error( `runner_remote_state_unavailable: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`, ); } return record( JSON.parse( Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"), ), ); } async function readRemoteRunnerProviderState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; execution: NativeExecutionInput; }): Promise> { const statePath = posix.join( input.stateDirectory, runnerProviderStateFilename(input.execution), ); const escapedPath = statePath.replaceAll("'", "'\\''"); const result = await input.runner.execute({ command: "sh", args: ["-c", `test -f '${escapedPath}' && base64 < '${escapedPath}'`], bypassSession: true, timeoutMs: 10_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_remote_provider_state_unavailable"); } return record( JSON.parse( Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"), ), ); } const REMOTE_RUNNER_PROCESS_IDENTITY_WAIT_MS = 20_000; const REMOTE_RUNNER_PROCESS_POLL_MS = 1_000; // Linux boot identity plus start ticks identifies a process generation across // PID reuse. exec preserves both the launch shell's PID and its start ticks. const REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT = "process_fingerprint=$(node -e 'const fs=require(\"node:fs\");try{const stat=fs.readFileSync(\"/proc/\"+process.argv[1]+\"/stat\",\"utf8\");const end=stat.lastIndexOf(\") \");const ticks=stat.slice(end+2).trim().split(/\\s+/)[19];const boot=fs.readFileSync(\"/proc/sys/kernel/random/boot_id\",\"utf8\").trim();if(end<0||!/^\\d+$/.test(ticks||\"\")||!/^[-a-fA-F0-9]+$/.test(boot))process.exit(4);process.stdout.write(\"linux:\"+boot+\":\"+ticks)}catch{process.exit(4)}' \"$pid\")"; const REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT = 'set -eu; identity_path=$1; expected_nonce=$2; expected_runner_id=$3; expected_pid=$4; test -f "$identity_path" && test ! -L "$identity_path" || exit 3; { IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path"; test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test "$pid" = "$expected_pid" && test -n "$started_at" || exit 4; kill -0 "$pid" 2>/dev/null || exit 3; if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 4; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 4; fi'; export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT = 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT + '; then :; else process_fingerprint=""; fi; identity_tmp="${identity_path}.tmp.$$"; printf "%s\\n%s\\n%s\\n%s\\n%s\\n" "$identity_nonce" "$$" "$started_at" "$runner_instance_id" "$process_fingerprint" > "$identity_tmp"; chmod 0600 "$identity_tmp"; mv -f -- "$identity_tmp" "$identity_path"; exec "$@"'; const REMOTE_RUNNER_FAILED_IDENTITY_CLEANUP_SCRIPT = 'set -eu; identity_path=$1; expected_nonce=$2; expected_runner_id=$3; marker_wait=0; while { test ! -f "$identity_path" || test -L "$identity_path"; } && test "$marker_wait" -lt 50; do marker_wait=$((marker_wait + 1)); sleep 0.1; done; test -f "$identity_path" && test ! -L "$identity_path" || exit 3; { IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path"; test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test -n "$started_at" || exit 4; case "$pid" in ""|*[!0-9]*) exit 4 ;; esac; test "$pid" -gt 0 || exit 4; if kill -0 "$pid" 2>/dev/null; then if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 4; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 4; fi; signal_target=$pid; if command -v ps >/dev/null 2>&1; then session_id=$(ps -o sid= -p "$pid" 2>/dev/null | tr -d " ") || true; if test "$session_id" = "$pid"; then signal_target="-$pid"; fi; fi; kill -TERM -- "$signal_target" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true; term_wait=0; while kill -0 "$pid" 2>/dev/null && test "$term_wait" -lt 50; do term_wait=$((term_wait + 1)); sleep 0.1; done; if kill -0 "$pid" 2>/dev/null; then kill -KILL -- "$signal_target" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true; kill_wait=0; while kill -0 "$pid" 2>/dev/null && test "$kill_wait" -lt 50; do kill_wait=$((kill_wait + 1)); sleep 0.1; done; fi; kill -0 "$pid" 2>/dev/null && exit 5; fi; test -f "$identity_path" && test ! -L "$identity_path" || exit 4; { IFS= read -r final_nonce; IFS= read -r final_pid; IFS= read -r final_started_at; IFS= read -r final_runner_id; } < "$identity_path"; test "$final_nonce" = "$nonce" && test "$final_pid" = "$pid" && test "$final_started_at" = "$started_at" && test "$final_runner_id" = "$runner_id" || exit 4; rm -f -- "$identity_path"'; export function parseRemoteRunnerProcessIdentity( value: string, expected: { nonce: string; runnerInstanceId: string }, ): { pid: number; startedAt: string; processStartFingerprint?: string } | null { const [nonce, rawPid, startedAt, runnerInstanceId, processStartFingerprint, ...remainder] = value .trim() .split("\n"); const pid = Number(rawPid); if ( remainder.length > 0 || nonce !== expected.nonce || runnerInstanceId !== expected.runnerInstanceId || !Number.isSafeInteger(pid) || pid <= 0 || !startedAt || Number.isNaN(new Date(startedAt).getTime()) || (processStartFingerprint !== undefined && !/^linux:[0-9a-f-]+:[0-9]+$/.test(processStartFingerprint)) ) { return null; } return { pid, startedAt, ...(processStartFingerprint ? { processStartFingerprint } : {}) }; } /** Verify remote ownership before exposing the transport's authenticated adoption path. */ export async function verifyRemoteRunnerReattachment(input: { claim: Extract< NativeRestartRecoveryClaim, { kind: "reattach_remote_runner" } >; target: AdapterExecutionTarget | null | undefined; identity: Record; runId: string; normalizedSessionId: string; }) { const { claim, target, identity } = input; if ( !adapterExecutionTargetIsCommandBacked(target) || !target.runner || claim.runId !== input.runId || identity.runId !== input.runId || identity.normalizedSessionId !== input.normalizedSessionId || claim.remote.providerLeaseId !== (target.transport === "computer" ? target.resourceAuthority.ownerId : target.sandboxLeaseAcquisition?.providerLeaseId) || (target.transport === "sandbox" && target.sandboxLeaseAcquisition?.outcome === "replacement") || claim.remote.remoteCwd !== target.remoteCwd ) { throw new Error("native_remote_recovery_lease_mismatch"); } if (target.transport === "computer") { const owner = claim.remote.computerOwner; if (!owner || owner.computerId !== target.resourceAuthority.computerId || owner.ownerId !== target.resourceAuthority.ownerId || owner.generation !== target.resourceAuthority.generation || owner.listenerPort !== target.listenerPort || !(await target.inspectProcess()).running) throw new Error("computer_process_recovery_mismatch"); } const runner = target.runner; const stateDirectory = posix.join( remoteRunnerStorageRoots(target, input.normalizedSessionId).sessionRoot, "runner", ); const runnerState = await readRemoteRunnerState({ runner, stateDirectory }); for (const field of [ "runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId", ]) { if ( typeof identity[field] !== "string" || !identity[field] || runnerState[field] !== identity[field] ) { throw new Error("runner_remote_recovery_identity_mismatch"); } } const identityPath = posix.join(stateDirectory, "runner-process.identity"); const marker = await runner.execute({ command: "sh", args: [ "-c", 'test -f "$1" && test ! -L "$1" && cat -- "$1"', "paperclip-runner-recovery-identity", identityPath, ], bypassSession: true, timeoutMs: 10_000, }); const nonce = marker.stdout.split("\n")[0] ?? ""; const runnerInstanceId = identity.runnerInstanceId as string; const processIdentity = marker.exitCode === 0 && !marker.timedOut && /^[a-zA-Z0-9_-]+$/.test(nonce) ? parseRemoteRunnerProcessIdentity(marker.stdout, { nonce, runnerInstanceId, }) : null; if (!processIdentity?.processStartFingerprint) throw new Error("runner_remote_process_identity_unavailable"); const expectedStartFingerprint = processIdentity.processStartFingerprint; const check = async (signal?: NodeJS.Signals) => { const result = await runner.execute({ command: "sh", args: [ "-c", `${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; test "$process_fingerprint" = "$5" || exit 4` + (signal ? '; kill -"$6" "$pid"' : ""), "paperclip-runner-recovery-check", identityPath, nonce, runnerInstanceId, String(processIdentity.pid), expectedStartFingerprint, ...(signal ? [signal.slice(3)] : []), ], bypassSession: true, timeoutMs: 10_000, }); return result.exitCode === 0 && !result.timedOut; }; if (!(await check())) throw new Error("runner_remote_process_identity_unavailable"); // These checks authorize an attempt to reconnect, not the task itself. The // transport still requires authentication against the exact durable PRP key. return { ...processIdentity, processGroupId: null, isAlive: () => check(), signal: check, }; } async function waitForRemoteRunnerProcessIdentity(input: { runner: CommandManagedRuntimeRunner; identityPath: string; nonce: string; runnerInstanceId: string; requireProcessStartFingerprint?: boolean; }): Promise<{ pid: number; startedAt: string; processStartFingerprint?: string }> { const deadline = Date.now() + REMOTE_RUNNER_PROCESS_IDENTITY_WAIT_MS; while (Date.now() < deadline) { const result = await input.runner .execute({ command: "sh", args: [ "-c", 'test -f "$1" && test ! -L "$1" && cat -- "$1"', "paperclip-runner-process-identity", input.identityPath, ], bypassSession: true, // An owned remote command includes SSH and supervisor admission before // reading this tiny file. Its normal roundtrip can exceed two seconds. // Give the RPC room while preserving the total identity-wait deadline. timeoutMs: Math.max(1, Math.min(10_000, deadline - Date.now())), }) .catch(() => null); const identity = result && result.exitCode === 0 && !result.timedOut ? parseRemoteRunnerProcessIdentity(result.stdout, input) : null; if (identity && (!input.requireProcessStartFingerprint || identity.processStartFingerprint)) return identity; await new Promise((resolve) => setTimeout(resolve, 100)); } throw new Error("runner_remote_process_identity_unavailable"); } async function cleanupRemoteRunnerAfterIdentityFailure(input: { runner: CommandManagedRuntimeRunner; identityPath: string; nonce: string; runnerInstanceId: string; }): Promise { const result = await input.runner .execute({ command: "sh", args: [ "-c", REMOTE_RUNNER_FAILED_IDENTITY_CLEANUP_SCRIPT, "paperclip-runner-identity-failure-cleanup", input.identityPath, input.nonce, input.runnerInstanceId, ], bypassSession: true, timeoutMs: 20_000, }) .catch(() => null); return result?.exitCode === 0 && result.timedOut === false; } export function createRemoteRunnerProcessLauncher(input: { target: Extract; runner: CommandManagedRuntimeRunner; remoteBinary: string; processIdentityPath: string; stateDirectory: string; diagnosticsDirectory: string; runnerInstanceId: string; ensureArtifact?: () => Promise; requireProcessStartFingerprint?: boolean; onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string; }) => Promise; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; trace?: NativeRunTrace; onRunnerProcessSpawned?: () => void; onLaunchAttempt?: () => void; }): (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle { const runner = input.runner; return (spec) => { let launchedIdentity: { nonce: string; pid: number; startedAt: string; } | null = null; const child: RunnerProcessHandle["child"] = { pid: undefined, exitCode: null, signalCode: null, kill: (requestedSignal) => { const identity = launchedIdentity; if (!identity) return false; const signal = requestedSignal === "SIGKILL" || requestedSignal === 9 ? "KILL" : requestedSignal === "SIGINT" || requestedSignal === 2 ? "INT" : "TERM"; // The durable marker, nonce, exact pid, and runner-id command line are // revalidated in the sandbox immediately before signalling. A recycled // pid or replaced marker therefore fails closed instead of killing an // unrelated process. void runner.execute({ command: "sh", args: [ "-c", `${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; kill -${signal} "$expected_pid"`, "paperclip-runner-signal", input.processIdentityPath, identity.nonce, input.runnerInstanceId, String(identity.pid), ], bypassSession: true, timeoutMs: 10_000, }).catch(async () => { // kill() follows Node's synchronous child-process contract. A deleted // sandbox or failed signal RPC must not reject outside that boundary // and crash the controller. This is not a termination receipt: the // monitor and cleanup verification still decide whether work stopped. await input.onLog?.( "stderr", "Remote runner signal failed; process termination is not confirmed.\n", ); }).catch(() => undefined); return true; }, }; const completion = (async () => { if (input.ensureArtifact) { if (input.trace) { await input.trace.measure( "runner.runtime.stage", input.ensureArtifact, { parentName: "runner.session.startup" }, ); } else { await input.ensureArtifact(); } } const launchStartedAtMs = Date.now(); // The provider's onSpawn callback is optional and some sandbox command // runners cannot report a remote pid until after the command has begun // streaming. Signal as soon as staging is complete and the launch RPC is // dispatched; this is late enough to avoid preview retries during staging // and early enough to avoid a callback-dependent deadlock. input.onRunnerProcessSpawned?.(); await input.trace?.record({ name: "runner.process.dispatch", parentName: "runner.session.startup", startedAtMs: launchStartedAtMs, endedAtMs: Date.now(), attributes: { target: "remote" }, }); const identityNonce = randomUUID(); const remoteArgs = [...spec.args]; const diagnosticsArgumentIndex = remoteArgs.indexOf( "--diagnostics-directory", ); if (diagnosticsArgumentIndex >= 0) { remoteArgs[diagnosticsArgumentIndex + 1] = input.diagnosticsDirectory; } else { remoteArgs.push("--diagnostics-directory", input.diagnosticsDirectory); } // Do not keep runnerd as the foreground command of a provider RPC. Some // sandbox command/session transports impose a provider-side lifetime on // that RPC even when Paperclip requests a longer timeout. Detach runnerd // into its own session instead; its own bounded diagnostics directory and // durable PRP state remain the authorities, and the controller monitors // the exact persisted process identity below. // From this exact boundary onward, a lost RPC acknowledgement may mean // a live process. Asset preparation failures above prove no dispatch. input.onLaunchAttempt?.(); const launchResult = input.target.transport === "computer" ? await input.target.launch({ command: "sh", args: ["-c", 'mkdir -p -- "$(dirname -- "$1")"; ' + REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "paperclip-runner-child", input.processIdentityPath, identityNonce, input.runnerInstanceId, input.diagnosticsDirectory, input.remoteBinary, ...remoteArgs], cwd: input.target.remoteCwd, env: processEnvironment(spec.environment) }) .then(() => ({ exitCode: 0, timedOut: false })) : await runner.execute({ command: "sh", args: [ "-c", 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; child_script=$4; shift 4; umask 077; identity_dir=$(dirname -- "$identity_path"); mkdir -p -- "$identity_dir"; if command -v setsid >/dev/null 2>&1; then nohup setsid sh -c "$child_script" paperclip-runner-child "$identity_path" "$identity_nonce" "$runner_instance_id" "$@" /dev/null 2>&1 & else nohup sh -c "$child_script" paperclip-runner-child "$identity_path" "$identity_nonce" "$runner_instance_id" "$@" /dev/null 2>&1 & fi', "paperclip-runner-launch", input.processIdentityPath, identityNonce, input.runnerInstanceId, REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, input.diagnosticsDirectory, input.remoteBinary, ...remoteArgs, ], cwd: input.target.remoteCwd, env: processEnvironment(spec.environment), timeoutMs: 20_000, bypassSession: true, onLog: input.onLog, }); if (launchResult.exitCode !== 0 || launchResult.timedOut) { throw new Error( launchResult.timedOut ? "runner_remote_process_launch_timed_out" : "runner_remote_process_launch_failed", ); } let identity: { pid: number; startedAt: string }; try { identity = await waitForRemoteRunnerProcessIdentity({ runner, identityPath: input.processIdentityPath, nonce: identityNonce, runnerInstanceId: input.runnerInstanceId, requireProcessStartFingerprint: input.requireProcessStartFingerprint, }); } catch { const cleanupStartedAtMs = Date.now(); const cleaned = await cleanupRemoteRunnerAfterIdentityFailure({ runner, identityPath: input.processIdentityPath, nonce: identityNonce, runnerInstanceId: input.runnerInstanceId, }); await input.trace?.record({ name: "runner.process.identity_failure_cleanup", parentName: "runner.session.startup", startedAtMs: cleanupStartedAtMs, endedAtMs: Date.now(), attributes: { cleaned }, }); if (!cleaned) { throw new Error( "runner_remote_process_identity_unavailable_cleanup_failed", ); } throw new Error("runner_remote_process_identity_unavailable"); } launchedIdentity = { nonce: identityNonce, ...identity }; child.pid = identity.pid; await input.onSpawn?.({ pid: identity.pid, processGroupId: null, startedAt: identity.startedAt, }); await input.trace?.record({ name: "runner.process.launch", parentName: "runner.session.startup", startedAtMs: launchStartedAtMs, endedAtMs: Date.now(), attributes: { identitySource: "remote_marker", detached: true }, }); let consecutiveUnknownObservations = 0; while (true) { const observed = await runner.execute({ command: "sh", args: [ "-c", REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT, "paperclip-runner-monitor", input.processIdentityPath, identityNonce, input.runnerInstanceId, String(identity.pid), ], bypassSession: true, timeoutMs: 10_000, }).catch(() => null); const identityConfirmed = observed?.exitCode === 0 && !observed.timedOut; const identityLost = observed !== null && !observed.timedOut && (observed.exitCode === 3 || observed.exitCode === 4); if (!identityLost) { // SSH/provider failures do not prove that runnerd exited. In particular, // concurrent artifact uploads can delay a monitor RPC while PRP stays // healthy. Retry unknown observations without allocating another runner. consecutiveUnknownObservations = identityConfirmed ? 0 : consecutiveUnknownObservations + 1; if (consecutiveUnknownObservations >= 3) { throw new Error(`runner_remote_process_monitor_unavailable exitCode=${observed?.exitCode ?? "unknown"} timedOut=${observed?.timedOut ?? "unknown"}`); } await new Promise((resolve) => setTimeout(resolve, REMOTE_RUNNER_PROCESS_POLL_MS), ); continue; } child.exitCode = null; const identityMismatch = observed.exitCode === 4; const diagnostic = await runner .execute({ command: "sh", args: [ "-c", 'set -eu; directory=$1; file="$directory/runnerd.stderr.log"; test -d "$directory" && test ! -L "$directory" && test -f "$file" && test ! -L "$file"; tail -c 65536 -- "$file"', "paperclip-runner-diagnostics", input.diagnosticsDirectory, ], bypassSession: true, timeoutMs: 10_000, }) .catch(() => null); const diagnosticTail = diagnostic && diagnostic.exitCode === 0 && !diagnostic.timedOut ? redactSensitiveText(diagnostic.stdout).slice(-16_384).trim() : ""; const durableState = await readRemoteRunnerState({ runner, stateDirectory: input.stateDirectory, }).catch(() => null); const lifecycle = typeof durableState?.lifecycle === "string" ? durableState.lifecycle : "unavailable"; const recoverableFailure = typeof durableState?.recoverableFailure === "string" ? durableState.recoverableFailure : typeof durableState?.recoverable_failure === "string" ? durableState.recoverable_failure : null; const stateDiagnostics = Array.isArray(durableState?.diagnostics) ? durableState.diagnostics .filter((value): value is string => typeof value === "string") .slice(-4) .map((value) => redactSensitiveText(value).slice(-1_000)) : []; const stateSummary = `runner_remote_process_exited lifecycle=${lifecycle}${recoverableFailure ? ` recoverableFailure=${redactSensitiveText(recoverableFailure).slice(-1_000)}` : ""}${stateDiagnostics.length > 0 ? ` diagnostics=${JSON.stringify(stateDiagnostics)}` : ""}`; return { code: null, signal: null, stdout: "", stderr: identityMismatch ? "runner_remote_process_identity_mismatch" : diagnosticTail || stateSummary, }; } })(); return { child, completion }; }; } /** * Select the remote runner transport before any artifact is staged or provider * endpoint is acquired. Sandbox ingress is available only to a run already * authorized by native runtime selection. */ export function resolveRemoteRunnerTransportMode(input: { target: AdapterExecutionTarget; runnerIngressAuthorized: boolean; }): "listen_ws" | "dial_wss" { if (input.target.kind !== "remote") { throw new Error("runner_transport_ineligible: remote target is required"); } const requiredMode = adapterExecutionTargetIsCommandBacked(input.target) && input.target.effectiveCapabilities?.runnerWebSocketIngress === true ? "listen_ws" : "dial_wss"; if (requiredMode === "listen_ws" && !input.runnerIngressAuthorized) { throw new Error("runner_ingress_unavailable"); } return requiredMode; } export function remoteCheckpointIncompleteFailure( settlement: "settled" | "unsettled", incompleteReason: "unavailable" | "not_suspended" | null, ): Error | null { // A transport that never completed provider bootstrap has no provider state // to preserve; its original launch failure remains authoritative. Once // runnerd has proved suspension, however, an unreadable or incomplete // checkpoint must fail the required close so outer sandbox release is // withheld. Process containment still happens in the transport finally. if (settlement === "unsettled") return null; return new Error( `runner_remote_checkpoint_incomplete: exact suspended harness state unavailable (${incompleteReason ?? "unknown"})`, ); } /** Production runnerd backend seam, exported so provider wiring can be regression tested. */ export async function createRunnerdBackend(input: { db: Db; execution: NativeExecutionInput; runnerInstanceId: string; chatAttachmentReadScope?: NativeChatAttachmentReadScope; restartRecovery?: NativeRestartRecoveryClaim; durableEnvironmentLeaseId?: string; onRemoteLaunchAttempt?: () => void; onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string; }) => Promise; dotWorkspaceRoot?: string; runnerEnvironment?: NodeJS.ProcessEnv; /** Private grant materialization; never a user-configured host path. */ managedAiCredentialHome?: string; runnerExecutionTarget?: AdapterExecutionTarget | null; /** Resolved per-run authorization; not an independent instance setting. */ runnerIngressAuthorized?: boolean; runnerPublicUrl?: string | null; runnerCaBundlePath?: string | null; runnerRemoteBinaryPath?: string | null; runnerRemoteCodexPath?: string | null; runnerRemoteCodexNpmSpec?: string | null; runnerRemoteProviderPackPath?: string | null; trace?: NativeRunTrace; toolTrace?: NativeToolTrace; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; stopTaskForReassignment?: (target: { companyId: string; issueId: string; agentId: string; runId: string | null }) => Promise; syncIssueExternalObjects?: (issueId: string) => Promise; enqueueWakeup?: ( agentId: string, options: { source: "assignment" | "automation"; triggerDetail: "system"; reason: "issue_assigned" | "issue_commented"; payload: Record; idempotencyKey: string; requestedByActorType: "agent"; requestedByActorId: string; contextSnapshot: Record; }, ) => Promise; }): Promise { const sessionScopeId = nativeSessionScopeKey(input.execution); const scopeOwner = executingRunnerdSessionScopes.get(sessionScopeId); if (scopeOwner && scopeOwner !== input.execution.binding.runId) { throw new Error("native_session_supervisor_busy"); } if (initializingSessionToolAuthorities.has(sessionScopeId)) { throw new Error("native_session_supervisor_busy"); } initializingSessionToolAuthorities.add(sessionScopeId); try { let retainedTransition: VerifiedWarmTransitionBinding | undefined; // executePaperclipNativeSession holds the full session-scope claim and // verifies/migrates the durable root before it acquires the coordinator // lease. Avoid reclassifying the same root after that path has marked its // retained warm owner busy for this run. Direct backend construction still // performs the complete fail-closed verification here. if ( executingRunnerdSessionScopes.get(sessionScopeId) !== input.execution.binding.runId || hasRetainedWarmTransitionEvidence(scopedRunnerdStateRoot(input.execution)) ) { retainedTransition = await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox", allowRetainedWarmRunner: false, allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote", onLog: input.onLog, restartRecovery: input.restartRecovery, runnerExecutionTarget: input.runnerExecutionTarget, }); } return await createRunnerdBackendWithinSessionClaim( input, sessionScopeId, retainedTransition, ); } finally { initializingSessionToolAuthorities.delete(sessionScopeId); } } /** Dot uses the same managed process launcher and authenticated PRP transports. * Its execution contract still exposes no workspace, shell, or provider secrets. */ async function prepareRemoteDotRunner({ input, target, runner, root, identity }: { input: Parameters[0]; target: Extract; runner: CommandManagedRuntimeRunner; root: string; identity: NonNullable[1]>["dotRunnerOptions"]>["identity"]; }) { const runnerBinary = input.runnerRemoteBinaryPath?.trim() || resolvePaperclipRunnerBinary(); const storage = remoteRunnerStorageRoots(target, identity.normalizedSessionId); const remoteRoot = storage.runtimeRoot; const remoteBinary = posix.join(remoteRoot, "bin", "paperclip-runnerd"); const runnerStateDirectory = posix.join(storage.sessionRoot, "runner"); const requiredMode = resolveRemoteRunnerTransportMode({ target, runnerIngressAuthorized: input.runnerIngressAuthorized === true }); const artifact = readRunnerdArtifactBinding(runnerBinary); const prepare = async () => { if (input.restartRecovery?.kind !== "reattach_remote_runner") { if (!input.runnerRemoteBinaryPath?.trim()) { const platform = await runner.execute({ command: "sh", args: ["-c", "uname -s; uname -m"], bypassSession: true, timeoutMs: 10000 }); const [os, arch] = platform.stdout.trim().split(/\r?\n/); if (platform.exitCode !== 0 || platform.timedOut || os !== (process.platform === "darwin" ? "Darwin" : "Linux") || ![process.arch === "x64" ? "x86_64" : "aarch64", process.arch].includes(arch)) { throw new Error("runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the sandbox platform"); } } await stageRemoteRunnerFile({ target, runner, sourcePath: runnerBinary, targetPath: remoteBinary, mode: 0o700 }); } const metadata = await runner.execute({ command: remoteBinary, args: ["--build-metadata"], bypassSession: true, timeoutMs: 30000 }); if (metadata.exitCode !== 0 || metadata.timedOut) throw new Error("runner_remote_artifact_verification_failed"); assertRemoteRunnerBuildMetadata(JSON.parse(metadata.stdout), requiredMode, "openai_dot_mcp"); const checksum = await runner.execute({ command: "sh", args: ["-c", 'if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi', "paperclip-dot-artifact", remoteBinary], bypassSession: true, timeoutMs: 10000 }); if (checksum.exitCode !== 0 || checksum.timedOut || `sha256:${checksum.stdout.trim().split(/\s+/)[0]}` !== artifact.digest) { throw new Error("runner_remote_artifact_verification_failed"); } }; const readProviderState = async () => { const result = await runner.execute({ command: "sh", args: ["-c", 'set -eu; if test ! -e "$1" && test ! -L "$1"; then exit 3; fi; test -f "$1" && test ! -L "$1"; test "$(wc -c < "$1")" -le 33554432; base64 < "$1"', "paperclip-dot-checkpoint", posix.join(runnerStateDirectory, "dot-provider-state.json")], bypassSession: true, timeoutMs: 10000 }); if (result.exitCode === 3 && !result.timedOut) return null; if (result.exitCode !== 0 || result.timedOut) throw new Error("dot_provider_checkpoint_unavailable"); const state = record(JSON.parse(Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"))); if (state.schema !== "paperclip.runner.dot-provider-state.v1" || state.runId !== identity.runId || state.sessionId !== identity.normalizedSessionId || state.turnId !== identity.turnId) throw new Error("dot_runner_checkpoint_authority_mismatch"); // Retain inspectable recovery evidence on the controller without changing // which execution target is authoritative for replay. mkdirSync(join(root, "runner"), { recursive: true, mode: 0o700 }); const saved = join(root, "runner", "dot-provider-state.json"); const temporary = `${saved}.${randomUUID()}.tmp`; writeFileSync(temporary, JSON.stringify(state), { mode: 0o600 }); renameSync(temporary, saved); return state; }; let spawned: () => void = () => {}; const processStarted = new Promise(resolve => { spawned = resolve; }); // A transport handshake alone does not prove a recoverable process owner. // Do not attach the Dot broker until the exact sandbox marker is validated. let identityReady: () => void = () => {}; let identityFailed: (error: unknown) => void = () => {}; const processIdentityReady = new Promise((resolve, reject) => { identityReady = resolve; identityFailed = reject; }); void processIdentityReady.catch(() => {}); const adoptExistingRunner = input.restartRecovery?.kind === "reattach_remote_runner" ? await verifyRemoteRunnerReattachment({ claim: input.restartRecovery, target, identity: { ...identity }, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution) }) : undefined; if (adoptExistingRunner) { spawned(); identityReady(); } const launch = createRemoteRunnerProcessLauncher({ target, runner, remoteBinary, processIdentityPath: posix.join(runnerStateDirectory, "runner-process.identity"), stateDirectory: runnerStateDirectory, diagnosticsDirectory: posix.join(remoteRoot, "diagnostics"), runnerInstanceId: input.runnerInstanceId, requireProcessStartFingerprint: true, onSpawn: async meta => { await input.onSpawn?.(meta); identityReady(); }, onLog: input.onLog, trace: input.trace, onRunnerProcessSpawned: () => spawned() }); const runnerProcessLauncher: typeof launch = spec => { const handle = launch(spec); void handle.completion.then(() => identityFailed(new Error("runner_remote_process_identity_unavailable")), identityFailed); return handle; }; const controlPlaneRegistration: NonNullable[1]>["dotRunnerOptions"]>["controlPlaneRegistration"]> = async authority => { let transport: PaperclipRunnerTransport; // Validate outbound URL before staging; provider ingress needs the staged // listener binary before requesting the provider's private endpoint. if (requiredMode === "dial_wss") { transport = await resolvePaperclipRunnerTransport({ target, runId: input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath, runnerIngressAuthorized: input.runnerIngressAuthorized === true }); await prepare(); } else { await prepare(); transport = await resolvePaperclipRunnerTransport({ target, runId: input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerIngressAuthorized: input.runnerIngressAuthorized === true }); } await input.onLog?.("stderr", `[paperclip-runner] Dot controller target=remote transport=${transport.mode}\n`); if (transport.mode !== "provider_ingress") { const registration = await registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, authority }); let caBundlePath = transport.mode === "direct_outbound" ? transport.caBundlePath : undefined; if (caBundlePath) { const remoteCaBundlePath = posix.join(remoteRoot, "bin", "runner-ca-bundle.pem"); await stageRemoteRunnerFile({ target, runner, sourcePath: caBundlePath, targetPath: remoteCaBundlePath, mode: 0o600 }); caBundlePath = remoteCaBundlePath; } return { ...registration, ready: async () => { await processIdentityReady; }, connection: { mode: "connect" as const, connectUrl: transport.connectUrl, ...(caBundlePath ? { caBundlePath } : {}) } }; } let outbound: ReturnType | null = null; let activation: Promise | null = null; return { connection: { mode: "listen" as const, listenAddress: transport.listenAddress, listenPort: transport.listenPort, listenPath: transport.listenPath }, activate: () => { activation = (async () => { await processStarted; outbound = connectRunnerPrpIngress({ authority, endpoint: transport.ingress }); })(); void activation.catch(() => {}); }, ready: async () => { await activation; if (!outbound) throw new Error("runner_ingress_unavailable"); await outbound.ready; await processIdentityReady; }, get failure() { return outbound?.failure; }, startupFailureCode: "runner_ingress_unavailable" as const, release: async () => { if (outbound) await outbound.close(); else await transport.ingress.close(); }, }; }; return { runnerBinary, runnerStateDirectory, runnerProcessLauncher, readProviderState, controlPlaneRegistration, adoptExistingRunner }; } async function createRunnerdBackendWithinSessionClaim( input: Parameters[0], sessionScopeId: string, retainedTransition?: VerifiedWarmTransitionBinding, ): Promise { let recoveryPending = retainedTransition !== undefined; const target = input.runnerExecutionTarget ?? { kind: "local" as const }; const remoteTarget = target.kind === "remote" ? target : null; const remoteCommandRunner = remoteTarget ? remoteTarget.transport === "ssh" ? createNativeSshCommandRunner({ spec: remoteTarget.spec, defaultCwd: remoteTarget.remoteCwd, }) : remoteTarget.transport === "computer" ? remoteTarget.processRunner : remoteTarget.runner : null; if (remoteTarget && !remoteCommandRunner) { throw new Error( "runner_transport_ineligible: remote process runner is unavailable", ); } const currentWakeComments = await resolveCurrentWakeCommentsBinding( input.db, input.execution.binding, ); const pinnedSkills = new Set("runtimeContext" in input.execution ? input.execution.runtimeContext.skills.map((skill) => skill.key) : []); const connectorAssignments = [...pinnedSkills].some(isConnectorSkill) ? await resolveConnectorAssignments(input.db, input.execution.binding) : []; const reviewRun = await input.db.select({ contextSnapshot: heartbeatRuns.contextSnapshot }) .from(heartbeatRuns).where(and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), )).limit(1).then((rows) => rows[0]); const nativeReview = readNativeReviewAssignmentContext(reviewRun?.contextSnapshot); if (nativeReview && !await getNativeReviewAssignment(input.db, { ...input.execution.binding, contextSnapshot: nativeReview, })) throw new Error("native_review_assignment_no_longer_available"); // Remote Codex already sends dynamic tool calls over authenticated PRP. Keep // the assigned gateway on the control plane instead of asking the sandbox to // reach the host's HTTP origin (which may be private or loopback-only). const relayAssignedMcp = (remoteTarget !== null && input.execution.provider.kind === "codex") || input.execution.provider.kind === "openai_dot"; const assignedMcpUrl = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_URL; const assignedMcpToken = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_TOKEN; const assignedMcpName = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_NAME; const hasAssignedMcp = Boolean(assignedMcpName || assignedMcpUrl || assignedMcpToken); if (relayAssignedMcp && hasAssignedMcp && (!assignedMcpName?.trim() || !assignedMcpUrl?.trim() || !assignedMcpToken?.trim())) { throw new Error("assigned native MCP launch binding is incomplete"); } const assignedGatewayPublicId = relayAssignedMcp && assignedMcpUrl ? new URL(assignedMcpUrl).pathname.match(/^\/mcp\/gateways\/([a-zA-Z0-9_-]+)$/)?.[1] : undefined; if (relayAssignedMcp && hasAssignedMcp && !assignedGatewayPublicId) { throw new Error("assigned native MCP gateway path is invalid"); } const assignedMcpTools = relayAssignedMcp && !nativeReview && assignedMcpUrl && assignedMcpToken ? await createAssignedMcpTools({ gateway: getAssignedMcpGateway(input.db), gatewayPublicId: assignedGatewayPublicId!, bearerToken: assignedMcpToken, workMode: input.execution.task.workMode, }) : undefined; const dotAttachmentActor = input.execution.provider.kind === "openai_dot" ? await input.db.select({ config: agents.adapterConfig }).from(agents).where(and( eq(agents.id, input.execution.binding.agentId), eq(agents.companyId, input.execution.binding.companyId), )).limit(1).then(rows => rows[0]) : undefined; const authority = new PaperclipRunnerToolAuthority(input.db, { ...(nativeReview ? { nativeReview } : {}), connectorAssignments: connectorAssignments.filter((assignment) => pinnedSkills.has(assignment.skillKey)), assignedMcpTools, companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, runId: input.execution.binding.runId, agentId: input.execution.binding.agentId, normalizedSessionId: nativeSessionKey(input.execution), pinnedMcpDigest: "runtimeContext" in input.execution ? input.execution.runtimeContext.mcp.digest : undefined, workMode: input.execution.task.workMode, runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : undefined, workspaceBridge: input.execution.provider.kind === "openai_dot" && !!input.dotWorkspaceRoot, taskAttachmentRead: dotAttachmentActor?.config.dotAttachmentAccess === true, dotRuntime: input.execution.provider.kind === "openai_dot", assertBridgeAuthority: input.execution.provider.kind === "openai_dot" ? () => dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6) : undefined, workspaceRoot: input.dotWorkspaceRoot ?? remoteTarget?.remoteCwd ?? input.execution.workspace.cwd ?? undefined, executionTargetKind: target.kind, readRemoteWorkspaceFile: remoteTarget && remoteCommandRunner ? (file) => readVerifiedRemoteWorkspaceFile({ runner: remoteCommandRunner, workspaceRoot: remoteTarget.remoteCwd, ...file }) : undefined, currentWakeComments: currentWakeComments ?? undefined, chatAttachmentReadScope: input.chatAttachmentReadScope, stopTaskForReassignment: input.stopTaskForReassignment, syncIssueExternalObjects: input.syncIssueExternalObjects, enqueueWakeup: input.enqueueWakeup, }); const authorityEpoch = new SessionToolAuthorityEpoch( input.execution.binding.runId, authority, input.toolTrace, ); let dynamicTools: Awaited< ReturnType >; try { dynamicTools = await authorityEpoch.definitions(); } catch (error) { authorityEpoch.revoke(); throw error; } const root = runnerdStateRoot(input.execution); const durableIdentity = readRunnerdDurableIdentity(root); const durableBinding = retainedTransition ? loadRunnerdDurableBinding(input.execution, retainedTransition) : durableIdentityMatchesSession(durableIdentity, input.execution) ? durableIdentity : null; const effectiveRunnerInstanceId = durableBinding?.runnerInstanceId ?? input.runnerInstanceId; const effectiveEnvironmentLeaseId = durableBinding?.environmentLeaseId ?? input.durableEnvironmentLeaseId ?? input.execution.binding.executionWorkspaceId; mkdirSync(root, { recursive: true, mode: 0o700 }); if (input.execution.schema === "paperclip.native-execution-input.v6") { if (target.kind === "remote" && input.dotWorkspaceRoot) throw new Error("dot_remote_workspace_access_unavailable"); const remoteDot = target.kind === "remote" ? await prepareRemoteDotRunner({ input, target, runner: remoteCommandRunner!, root, identity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` } }) : null; const recoveredProcess = input.restartRecovery?.kind === "reattach_existing_runner" ? input.restartRecovery.process : null; const backend = createNativeSessionBackend(input.execution, { onSpawn: input.onSpawn, dynamicTools, dynamicToolHandler: async call => { await dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6); return authorityEpoch.execute(call); }, completionFeedback: async result => { await dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6); await authorityEpoch.definitions(); return nativeCompletionFeedback(input.db, input.execution.binding.runId, result); }, dotRunnerOptions: { stateDirectory: root, runnerBinary: remoteDot?.runnerBinary ?? resolvePaperclipRunnerBinary(), ...(remoteDot ?? {}), identity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` }, port: dotRunnerBroker(input.db).port(input.execution), controlPlaneRegistration: remoteDot?.controlPlaneRegistration ?? (async authority => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, authority })), adoptExistingRunner: remoteDot?.adoptExistingRunner ?? (recoveredProcess ? { ...recoveredProcess, isAlive: () => verifiedRecoveryProcessIsAlive(recoveredProcess), signal: signal => signalVerifiedRecoveryProcess(recoveredProcess, signal) } : undefined), }, }); const prior = sessionToolAuthorityEpochs.get(sessionScopeId); if (prior) prior.revoke(); sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch); return { ...backend, descriptor: () => backend.descriptor(), openSession: input => backend.openSession(input), recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options), bindManagedSession: session => session }; } const remoteStorage = remoteTarget ? remoteRunnerStorageRoots(remoteTarget, nativeSessionKey(input.execution)) : null; const remoteRuntimeRoot = remoteStorage?.runtimeRoot ?? null; const requiresRemoteProviderPack = remoteTarget !== null && (input.execution.provider.kind === "opencode" || input.execution.provider.kind === "acpx"); // Pi's explicit operator import supplies the target-platform authority. Never // substitute a Mac controller daemon or trust an image's self-reported hashes. // Existing explicit overrides and every other provider keep their old path. const remotePiCompanion = await selectRemotePiCompanion({ provider: input.execution.provider, remote: remoteTarget !== null, binaryOverride: input.runnerRemoteBinaryPath, packOverride: input.runnerRemoteProviderPackPath, workspaceRoot: input.execution.workspace.cwd, }); const configuredProviderPackRoot = input.runnerRemoteProviderPackPath?.trim() || remotePiCompanion?.providerPack || null; let expectedProviderPackManifest: RemoteProviderPackManifest | null = null; const useBundledRemoteImageAssets = requiresRemoteProviderPack && !configuredProviderPackRoot && (input.execution.provider.kind === "opencode" || (input.execution.provider.kind === "acpx" && input.execution.provider.agent === "cursor")); const retainedComputerPack = remoteTarget?.transport === "computer" && input.execution.provider.kind === "acpx" && durableBinding && durableIdentity && remoteCommandRunner && remoteStorage ? pinnedComputerProviderPack({ agentHome: remoteTarget.fileAuthority.agentHome, identity: durableIdentity, provider: input.execution.provider, control: JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"), NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")), }) : null; if (retainedComputerPack && remoteCommandRunner && remoteStorage && input.execution.provider.kind === "acpx") { // The local attachment template chooses the digest. Remote content cannot // authorize a newer pack or rewrite the existing provider's launch identity. const [manifest, state] = await readPinnedComputerProviderMetadata({ runner: remoteCommandRunner, packRoot: retainedComputerPack.root, stateDirectory: posix.join(remoteStorage.sessionRoot, "runner") }); expectedProviderPackManifest = validateRemoteProviderPackIdentity(manifest as RemoteProviderPackManifest, "", false, { digest: retainedComputerPack.digest, agent: input.execution.provider.agent }); if (expectedProviderPackManifest.digest !== retainedComputerPack.digest || expectedProviderPackManifest.payload.target.platform !== "linux" || expectedProviderPackManifest.payload.target.architecture !== "x64") throw new Error("runner_remote_provider_pack_provenance_mismatch"); const launchDigest = computerAcpxLaunchProfileDigest({ authorityDigest: retainedComputerPack.digest, command: retainedComputerPack.command, sidecar: retainedComputerPack.sidecar, commandSha256: expectedProviderPackManifest.payload.artifacts.nodeCommand.sha256, sidecarSha256: expectedProviderPackManifest.payload.artifacts.acpxSidecar.sha256 }); assertPinnedComputerProviderState({ state, launchProfileDigest: launchDigest, normalizedSessionId: nativeSessionKey(input.execution), command: retainedComputerPack.command, sidecar: retainedComputerPack.sidecar }); } else if (useBundledRemoteImageAssets) { expectedProviderPackManifest = readBundledRemoteProviderPackManifest(); } else if (requiresRemoteProviderPack) { if ( !configuredProviderPackRoot || !existsSync(configuredProviderPackRoot) || !lstatSync(configuredProviderPackRoot).isDirectory() ) { throw new Error( "runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH with the build-owned provider pack", ); } expectedProviderPackManifest = readRemoteProviderPackManifest( configuredProviderPackRoot, ); } const stagedRemoteProviderPackRoot = remoteTarget?.transport === "computer" && expectedProviderPackManifest ? computerProviderPackCachePath(remoteTarget.fileAuthority.agentHome, expectedProviderPackManifest.digest) : remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "provider-pack") : null; let activeRemoteProviderPackRoot: string | null = null; const remoteBinary = remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "bin", "paperclip-runnerd") : null; // The transport hashes runnerBinary on the controller before an external // launcher starts runnerd. Keep that artifact identity in the controller's // filesystem; the remote launcher separately owns the sandbox command path. // When an explicit remote artifact is configured, prepareRemoteRunner stages // these exact bytes at remoteBinary before launch. const controllerRunnerBinary = remoteTarget ? input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? bundledRemoteRunnerBinary() : resolvePaperclipRunnerBinary()) : resolvePaperclipRunnerBinary(); const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || (remoteTarget?.transport === "computer" && !explicitRemoteCodex ? `@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}` : null); if (explicitRemoteCodex && remoteCodexNpmSpec) { throw new Error("runner_remote_codex_source_conflict"); } const remoteCodexBinary = remoteRuntimeRoot && input.execution.provider.kind === "codex" ? remoteCodexNpmSpec ? posix.join( remoteRuntimeRoot, "harnesses", "codex", "node_modules", ".bin", "codex", ) : posix.join(remoteRuntimeRoot, "bin", "codex") : null; const remoteSessionRoot = remoteStorage?.sessionRoot ?? null; const remoteStateDirectory = remoteSessionRoot ? posix.join(remoteSessionRoot, "runner") : undefined; const remoteRunnerFilesystemRoot = remoteSessionRoot ? posix.join(remoteSessionRoot, "filesystem") : null; const persistenceProfile = resolveNativeHarnessPersistenceProfile( input.execution, ); const sandboxLeaseAcquisition = remoteTarget?.transport === "sandbox" ? (remoteTarget.sandboxLeaseAcquisition ?? null) : null; const remotePersistencePath = ( directory: NativeHarnessPersistenceDirectory, ): string | null => directory.location === "runner" ? (remoteStateDirectory ?? null) : remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, directory.name) : null; const sourceRuntimeContext = "runtimeContext" in input.execution ? input.execution.runtimeContext : null; // Computer bundles are immutable. Each bootstrap attempt publishes new // references instead of overlaying a previous read-only snapshot. Old // snapshots remain until future bounded maintenance; provider HOME and user // files never participate in this materialization. const remoteContextRoot = remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, "context", ...(remoteTarget?.transport === "computer" ? ["snapshots", randomUUID()] : [])) : null; const remoteRuntimeContext: NativeRuntimeContextSnapshot | null = remoteRunnerFilesystemRoot && sourceRuntimeContext ? { ...sourceRuntimeContext, ...(remoteTarget?.transport === "computer" ? { persistentAgentHome: { rootPath: remoteTarget.fileAuthority.agentHome }, } : {}), instructions: { ...sourceRuntimeContext.instructions, bundle: { ...sourceRuntimeContext.instructions.bundle, rootPath: posix.join( remoteContextRoot!, "instructions", ), }, }, skills: sourceRuntimeContext.skills.map((skill, index) => ({ ...skill, bundle: { ...skill.bundle, rootPath: posix.join( remoteContextRoot!, "skills", `${index}-${skill.bundle.digest.slice(0, 12)}`, ), }, })), } : sourceRuntimeContext; let remotePrepared = false; let remoteHarnessStatePrepared = false; let selectedRemoteMode: "dial_wss" | "listen_ws" | null = null; let remoteCaBundleMapping: { sourcePath: string; targetPath: string } | null = null; let resolveRemoteRunnerProcessSpawned: (() => void) | null = null; const remoteRunnerProcessSpawned = remoteTarget ? new Promise((resolveSpawned) => { resolveRemoteRunnerProcessSpawned = resolveSpawned; }) : Promise.resolve(); const verifyRemoteRunner = async ( requiredMode: "dial_wss" | "listen_ws", executable = remoteBinary, ) => { if (!remoteTarget || !remoteCommandRunner || !executable) return; const metadataResult = await remoteCommandRunner.execute({ command: executable, args: ["--build-metadata"], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 30_000, }); if (metadataResult.exitCode !== 0 || metadataResult.timedOut) { throw new Error("runner_remote_artifact_verification_failed"); } let metadata: Record; try { metadata = JSON.parse(metadataResult.stdout) as Record; } catch (error) { throw new Error("runner_remote_artifact_metadata_invalid", { cause: error, }); } assertRemoteRunnerBuildMetadata(metadata, requiredMode); }; const reportedCodexVersions = new Set(); const verifyRemoteCodex = async (executable = remoteCodexBinary) => { if (!remoteTarget || !remoteCommandRunner || !executable) return; const versionResult = await remoteCommandRunner.execute({ command: executable, args: ["--version"], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 30_000, }); if (versionResult.exitCode !== 0 || versionResult.timedOut) { throw new Error("runner_remote_codex_artifact_verification_failed"); } const versionOutput = `${versionResult.stdout}\n${versionResult.stderr}`; const version = parseCodexCliVersion(versionOutput); if (!version || !isSupportedRemoteCodexVersion(version)) { throw new Error( `runner_remote_provider_artifact_incompatible: supported Codex versions ${REMOTE_CODEX_SUPPORTED_RANGE}, received ${version ?? "an unrecognized or prerelease version"}; install a supported stable Codex release or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`, ); } // A Codex inside the compatibility window can still be too old for the // configured model: the ChatGPT backend rejects a model from clients // below the model's floor on every turn. Fail before launch with the // exact gap, so a stale sandbox image is not reported as an account // problem. When a preinstalled Codex fails here and an npm spec is // configured, the caller falls back to installing the pinned release. const configuredModel = input.execution.provider.kind === "codex" ? input.execution.provider.model : null; const modelMinimum = minimumCodexCliVersionForModel(configuredModel); if (modelMinimum && !codexCliVersionAtLeast(version, modelMinimum)) { throw new Error( `runner_remote_provider_artifact_incompatible: ${normalizeCodexModel(configuredModel)} requires Codex ${modelMinimum} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex ${REMOTE_PROVIDER_PACK_PINS.codex} or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`, ); } if (version !== REMOTE_PROVIDER_PACK_PINS.codex && !reportedCodexVersions.has(version)) { reportedCodexVersions.add(version); await input.onLog?.( "stderr", `[paperclip-runner] using compatible Codex ${version} (supported ${REMOTE_CODEX_SUPPORTED_RANGE}; install pin ${REMOTE_PROVIDER_PACK_PINS.codex})\n`, ); } }; const verifyRemoteProviderPack = async (packRoot: string) => { if (!remoteTarget || !remoteCommandRunner || !expectedProviderPackManifest) return; const expected = Buffer.from( canonicalJson(expectedProviderPackManifest), "utf8", ).toString("base64"); const providerNodeCommand = posix.join( packRoot, expectedProviderPackManifest.payload.artifacts.nodeCommand.path, ); const verifyScript = [ "const fs=require('node:fs')", "const crypto=require('node:crypto')", "const path=require('node:path')", "const root=process.argv[1]", "const expected=Buffer.from(process.argv[2],'base64').toString('utf8')", "const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()", "const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)", "const manifest=JSON.parse(actual)", "if(canonical(manifest)!==expected)throw new Error('manifest mismatch')", "const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')", "const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}", "for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}", "if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')", "for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}", "const version=process.versions.node.split('.').map(Number)", `const minimums=[manifest.payload.pins.nodeMinimum,${JSON.stringify(REMOTE_PROVIDER_PACK_PINS.nodeMinimum)}]`, "for(const value of minimums){const minimum=value.split('.').map(Number);if(version[0]JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version", "const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}", "for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')", ].join(";"); const verified = await remoteCommandRunner.execute({ command: providerNodeCommand, args: ["-e", verifyScript, packRoot, expected], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 30_000, }); if (verified.exitCode !== 0 || verified.timedOut) { throw new Error( `runner_remote_provider_artifact_incompatible: provider pack verification failed (${verified.stderr.trim().slice(-1_024)})`, ); } const opencodeCommand = posix.join( packRoot, expectedProviderPackManifest.payload.artifacts.opencodeCommand.path, ); const opencodeVersion = await remoteCommandRunner.execute({ command: opencodeCommand, args: ["--version"], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 30_000, }); if ( opencodeVersion.exitCode !== 0 || opencodeVersion.timedOut || opencodeVersion.stdout.trim() !== expectedProviderPackManifest.payload.pins.opencode ) { throw new Error( "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", ); } }; const discoverPreinstalledProviderPack = async () => { if (!remoteTarget || !remoteCommandRunner) return null; const result = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", 'for candidate in /opt/paperclip-runner/provider-pack "$HOME/.local/share/paperclip-runner/provider-pack"; do if [ -f "$candidate/provider-pack.json" ]; then printf \'%s\\n\' "$candidate"; break; fi; done', ], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); if (result.exitCode !== 0 || result.timedOut) return null; return parseRemoteExecutableCandidate(result.stdout); }; // Image policy: keep one latest stable CLI installation shared by native and // local adapters. Preferred bin entries must point to that same installation; // never bake an older global CLI alongside a private runner-only version. const discoverPreinstalledExecutable = async ( name: "paperclip-runnerd" | "codex", ) => { if (!remoteTarget || !remoteCommandRunner) return null; return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name); }; const linkPreinstalledExecutable = async ( sourcePath: string, targetPath: string, ) => { if (!remoteTarget || !remoteCommandRunner) return; const escapedSource = sourcePath.replaceAll("'", "'\\''"); const escapedTarget = targetPath.replaceAll("'", "'\\''"); const escapedDirectory = posix.dirname(targetPath).replaceAll("'", "'\\''"); const result = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", targetPath === remoteCodexBinary ? buildRemoteCodexLauncherCommand(sourcePath, targetPath) : `umask 077; mkdir -p '${escapedDirectory}' && ` + `ln -sfn '${escapedSource}' '${escapedTarget}'`, ], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_remote_preinstalled_link_failed"); } }; const prepareRemoteRunner = async ( requiredMode: "dial_wss" | "listen_ws", ) => { if ( !remoteTarget || !remoteCommandRunner || !remoteBinary || remotePrepared ) return; selectedRemoteMode = requiredMode; if (input.restartRecovery?.kind === "reattach_remote_runner") { await verifyRemoteRunner(requiredMode); if (requiresRemoteProviderPack && stagedRemoteProviderPackRoot) { await verifyRemoteProviderPack(stagedRemoteProviderPackRoot); activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot; } remotePrepared = true; return; } // Compatibility metadata does not prove artifact identity. Reuse only the // exact controller-owned bytes when the controller artifact is available. const matchesControllerRunnerArtifact = async (executable: string): Promise => { const expected = createHash("sha256") .update(readFileSync(controllerRunnerBinary)) .digest("hex"); try { const probe = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", 'test -x "$1" || exit 1; if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi', "paperclip-runner-artifact", executable, ], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); return probe.exitCode === 0 && !probe.timedOut && /^[a-f0-9]{64}\s/.test(probe.stdout) && probe.stdout.trim().split(/\s+/)[0] === expected; } catch { // An unavailable checksum uses the verified staging path. return false; } }; let runnerArtifactPrepared = false; if ( sandboxLeaseAcquisition?.outcome === "resumed" && existsSync(controllerRunnerBinary) ) { runnerArtifactPrepared = await measureNativeRunnerSpan( input.trace, "runner.artifact.verify_retained", () => matchesControllerRunnerArtifact(remoteBinary), ); } const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null; if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) { const preinstalledRunner = await measureNativeRunnerSpan( input.trace, "runner.artifact.discover", () => discoverPreinstalledExecutable("paperclip-runnerd"), ); if (preinstalledRunner) { try { await measureNativeRunnerSpan( input.trace, "runner.artifact.verify_preinstalled", () => verifyRemoteRunner(requiredMode, preinstalledRunner), ); if (existsSync(controllerRunnerBinary) && !await matchesControllerRunnerArtifact(preinstalledRunner)) { throw new Error("runner_remote_preinstalled_artifact_mismatch"); } await measureNativeRunnerSpan( input.trace, "runner.artifact.link", () => linkPreinstalledExecutable(preinstalledRunner, remoteBinary), ); runnerArtifactPrepared = true; await input.onLog?.( "stderr", "[paperclip-runner] using preinstalled runnerd from the sandbox image\n", ); } catch { runnerArtifactPrepared = false; } } } if (!runnerArtifactPrepared) { // Upload the same artifact used for the controller identity. The server // vendors the runner under vendor/paperclip-runner/bin, so the package // development fallback cannot locate it in a deployed server. const sourceBinary = controllerRunnerBinary; if (!existsSync(sourceBinary)) { throw new Error("runner_remote_artifact_unavailable"); } if (!explicitRemoteBinary && !remotePiCompanion && !useBundledRemoteImageAssets) { const platform = await remoteCommandRunner.execute({ command: "sh", args: ["-c", "uname -s; uname -m"], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); const [remoteOs = "", remoteArch = ""] = platform.stdout .trim() .split(/\r?\n/); const localOs = process.platform === "darwin" ? "Darwin" : process.platform === "linux" ? "Linux" : process.platform; const localArch = process.arch === "x64" ? "x86_64" : process.arch === "arm64" ? "aarch64" : process.arch; const archMatches = remoteArch === localArch || (localArch === "aarch64" && remoteArch === "arm64"); if ( platform.exitCode !== 0 || platform.timedOut || remoteOs !== localOs || !archMatches ) { throw new Error( "runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture", ); } } await stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: sourceBinary, targetPath: remoteBinary, mode: 0o700, }); } await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () => verifyRemoteRunner(requiredMode), ); if (remoteCodexBinary && explicitRemoteCodex) { if (!existsSync(explicitRemoteCodex)) { throw new Error("runner_remote_codex_artifact_unavailable"); } await stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: explicitRemoteCodex, targetPath: remoteCodexBinary, mode: 0o700, }); await verifyRemoteCodex(); } if (remoteCodexBinary && remoteCodexNpmSpec) { let usedPreinstalledCodex = false; const preinstalledCodex = await measureNativeRunnerSpan( input.trace, "harness.artifact.discover", () => discoverPreinstalledExecutable("codex"), ); if (preinstalledCodex) { try { await measureNativeRunnerSpan( input.trace, "harness.artifact.verify_preinstalled", () => verifyRemoteCodex(preinstalledCodex), ); await measureNativeRunnerSpan( input.trace, "harness.artifact.link", () => linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), ); usedPreinstalledCodex = true; await input.onLog?.( "stderr", "[paperclip-runner] using preinstalled Codex from the sandbox image\n", ); } catch { usedPreinstalledCodex = false; } } if (!usedPreinstalledCodex) { const installRoot = posix.join( remoteRuntimeRoot!, "harnesses", "codex", ); const installResult = await remoteCommandRunner.execute({ command: "npm", args: [ "install", "--prefix", installRoot, "--no-audit", "--no-fund", remoteCodexNpmSpec, ], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 180_000, }); if (installResult.exitCode !== 0 || installResult.timedOut) { throw new Error(`runner_remote_codex_install_failed exitCode=${installResult.exitCode} timedOut=${installResult.timedOut}: ${redactSensitiveText(installResult.stderr).slice(-2_000).trim()}`); } } await measureNativeRunnerSpan( input.trace, "harness.artifact.verify", () => verifyRemoteCodex(), ); } if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) { const preinstalledCodex = await measureNativeRunnerSpan( input.trace, "harness.artifact.discover", () => discoverPreinstalledExecutable("codex"), ); if (!preinstalledCodex) { throw new Error( "runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC", ); } await measureNativeRunnerSpan( input.trace, "harness.artifact.verify_preinstalled", () => verifyRemoteCodex(preinstalledCodex), ); await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () => linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary), ); await measureNativeRunnerSpan( input.trace, "harness.artifact.verify", () => verifyRemoteCodex(), ); await input.onLog?.( "stderr", "[paperclip-runner] using preinstalled Codex from the sandbox image\n", ); } if ( requiresRemoteProviderPack && expectedProviderPackManifest && stagedRemoteProviderPackRoot ) { const packSource = remoteTarget.transport === "computer" ? await prepareComputerProviderPackCache({ cacheRoot: stagedRemoteProviderPackRoot, sessionKey: nativeSessionKey(input.execution), owner: remoteTarget.resourceAuthority, runner: remoteCommandRunner, verify: (root) => measureNativeRunnerSpan(input.trace, "provider_pack.verify", () => verifyRemoteProviderPack(root)), stage: async (root, runner) => { if (retainedComputerPack) throw new Error("runner_remote_provider_pack_retained_artifacts_unavailable"); if (!configuredProviderPackRoot) { throw new Error("runner_remote_provider_artifact_incompatible: configure the build-owned provider pack for first use on this computer"); } await input.onLog?.("stderr", "[paperclip-runner] Preparing the provider pack for first use.\n"); await stageRemoteRunnerDirectory({ target: remoteTarget, runner, sourcePath: configuredProviderPackRoot, targetPath: root, mode: 0o700, onProgress: async (completed, total) => { await input.onLog?.("stderr", `[paperclip-runner] Uploading provider pack: ${Math.round(completed / 1024 / 1024)} / ${Math.round(total / 1024 / 1024)} MiB.\n`); }, }); }, }) : await prepareVerifiedRemoteProviderPack({ verifyStaged: () => measureNativeRunnerSpan( input.trace, "provider_pack.verify", () => verifyRemoteProviderPack(stagedRemoteProviderPackRoot), ), usePreinstalled: async () => { let preinstalledProviderPack = await discoverPreinstalledProviderPack(); if (preinstalledProviderPack) { try { await measureNativeRunnerSpan( input.trace, "provider_pack.verify_preinstalled", () => verifyRemoteProviderPack(preinstalledProviderPack!), ); const escapedSource = preinstalledProviderPack.replaceAll( "'", "'\\''", ); const escapedTarget = stagedRemoteProviderPackRoot.replaceAll( "'", "'\\''", ); const escapedParent = posix .dirname(stagedRemoteProviderPackRoot) .replaceAll("'", "'\\''"); const linked = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", `umask 077; mkdir -p '${escapedParent}' && rm -rf '${escapedTarget}' && ln -s '${escapedSource}' '${escapedTarget}'`, ], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); if (linked.exitCode !== 0 || linked.timedOut) { throw new Error( "runner_remote_provider_artifact_incompatible: preinstalled provider pack could not be linked", ); } activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot; await input.onLog?.( "stderr", "[paperclip-runner] using manifest-matched provider pack from the sandbox image\n", ); } catch { preinstalledProviderPack = null; } } return preinstalledProviderPack !== null; }, stageAndVerify: async () => { if (!configuredProviderPackRoot) { throw new Error("runner_remote_provider_artifact_incompatible: install the matching Paperclip package and Daytona image; the image provider pack did not match the bundled release identity"); } const escapedPackRoot = stagedRemoteProviderPackRoot.replaceAll( "'", "'\\''", ); const cleared = await remoteCommandRunner.execute({ command: "sh", args: ["-c", `rm -rf '${escapedPackRoot}'`], cwd: remoteTarget.remoteCwd, bypassSession: true, timeoutMs: 10_000, }); if (cleared.exitCode !== 0 || cleared.timedOut) { throw new Error( "runner_remote_provider_artifact_incompatible: stale provider pack could not be replaced", ); } await input.onLog?.("stderr", "[paperclip-runner] Preparing the provider pack for first use.\n"); await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: configuredProviderPackRoot, targetPath: stagedRemoteProviderPackRoot, mode: 0o700, onProgress: async (completed, total) => { await input.onLog?.("stderr", `[paperclip-runner] Uploading provider pack: ${Math.round(completed / 1024 / 1024)} / ${Math.round(total / 1024 / 1024)} MiB.\n`); }, }); await measureNativeRunnerSpan(input.trace, "provider_pack.verify", () => verifyRemoteProviderPack(stagedRemoteProviderPackRoot), ); activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot; }, }); activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot; if (packSource === "staged" || packSource === "reused") { await input.onLog?.( "stderr", "[paperclip-runner] reusing manifest-matched provider pack from the workspace\n", ); } } remotePrepared = true; }; const inspectRemoteHarnessState = async (): Promise<{ complete: boolean; runnerState: Record | null; providerSessionIdentity: Record | null; incompleteReason: "unavailable" | "not_suspended" | null; }> => { if (!remoteCommandRunner || !remoteStateDirectory) { return { complete: false, runnerState: null, providerSessionIdentity: null, incompleteReason: "unavailable", }; } const requirements = persistenceProfile.directories.flatMap((directory) => { const path = remotePersistencePath(directory); if (!path) return []; const escaped = path.replaceAll("'", "'\\''"); return directory.name === "runner" ? [`test -f '${escaped}/runner-state.json'`] : [`test -d '${escaped}'`]; }); const escapedRunnerState = posix .join(remoteStateDirectory, "runner-state.json") .replaceAll("'", "'\\''"); const escapedProviderState = posix .join(remoteStateDirectory, runnerProviderStateFilename(input.execution)) .replaceAll("'", "'\\''"); const inspected = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", `${requirements.join(" && ")} && test -f '${escapedProviderState}' && base64 < '${escapedRunnerState}'`, ], bypassSession: true, timeoutMs: 10_000, }); if (inspected.exitCode !== 0 || inspected.timedOut) { return { complete: false, runnerState: null, providerSessionIdentity: null, incompleteReason: "unavailable", }; } let runnerState: Record; try { runnerState = record( JSON.parse( Buffer.from(inspected.stdout.replace(/\s+/g, ""), "base64").toString( "utf8", ), ), ); } catch { throw new Error("runner_harness_state_mismatch: runner_state_invalid_json"); } if ( runnerState.runnerInstanceId !== input.runnerInstanceId || runnerState.normalizedSessionId !== nativeSessionKey(input.execution) ) { throw new Error("runner_harness_state_mismatch: runner_identity"); } if (runnerState.lifecycle !== "suspended") { return { complete: false, runnerState: null, providerSessionIdentity: null, incompleteReason: "not_suspended", }; } let providerState: Record; try { providerState = await readRemoteRunnerProviderState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory, execution: input.execution, }); } catch { throw new Error("runner_harness_state_mismatch: provider_state_unreadable"); } const providerSessionIdentity = providerSessionIdentityFromDurableProviderState({ execution: input.execution, providerState, }); if (!providerSessionIdentityIsPresent(providerSessionIdentity)) { throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)})`); } const previousManifest = compatibleNativeHarnessBackupManifests({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId, })[0]?.manifest; if ( previousManifest && !providerSessionIdentityTransitionIsAllowed({ execution: input.execution, previous: previousManifest.providerSessionIdentity, current: providerSessionIdentity, }) ) { throw new Error("runner_harness_state_mismatch: provider_identity_changed"); } return { complete: true, runnerState, providerSessionIdentity, incompleteReason: null, }; }; const claimUntouchedSessionInResumedLease = async (): Promise => { if (!remoteCommandRunner || !remoteRuntimeRoot || !remoteSessionRoot) return false; const identity = readRunnerdDurableIdentity(root); if (!durableIdentityMatchesExecution(identity, input.execution) || identity?.runnerInstanceId !== effectiveRunnerInstanceId || identity?.environmentLeaseId !== effectiveEnvironmentLeaseId || !runnerdStateProvesIncompleteBootstrap(root)) return false; // A reusable workspace may have failed before any harness was created. // Claim this exact new session atomically under readable real directories. // Missing files inside an existing session never authorize a fresh start. const probe = await remoteCommandRunner.execute({ command: "sh", args: ["-c", 'set -eu; umask 077; test -d "$1" && test ! -L "$1" && test -r "$1" && test -x "$1" || exit 1; if test ! -e "$2" && test ! -L "$2"; then mkdir -- "$2"; fi; test -d "$2" && test ! -L "$2" && test -r "$2" && test -x "$2" || exit 1; mkdir -- "$3"', "paperclip-runner-claim-unstarted-session", remoteRuntimeRoot, posix.dirname(remoteSessionRoot), remoteSessionRoot], bypassSession: true, timeoutMs: 10_000, }); return probe.exitCode === 0 && !probe.timedOut; }; const recordInPlaceHarnessReuse = async ( providerSessionIdentity: Record, startedAtMs = Date.now(), ) => { const now = Date.now(); const attributes = { provider: input.execution.provider.kind, harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, stateSource: "sandbox_filesystem", bytesTransferred: 0, }; await input.trace?.record({ name: "harness_state.reuse", startedAtMs, endedAtMs: now, attributes, }); await input.trace?.record({ name: "provider.session.resume", startedAtMs: now, endedAtMs: now, attributes: { provider: input.execution.provider.kind, harness: input.execution.session.driverKind, identityPresent: providerSessionIdentityIsPresent( providerSessionIdentity, ), }, }); }; const recordHarnessBackupStampForCurrentLease = async ( backup: VerifiedHarnessBackup, ) => { if (remoteTarget?.transport !== "sandbox" || !remoteTarget.leaseId) { return; } const leaseRow = await input.db .select({ metadata: environmentLeases.metadata, providerLeaseId: environmentLeases.providerLeaseId, }) .from(environmentLeases) .where(eq(environmentLeases.id, remoteTarget.leaseId)) .limit(1) .then((rows) => rows[0] ?? null); if (!leaseRow?.providerLeaseId) { throw new Error("runner_harness_backup_lease_missing"); } if ( sandboxLeaseAcquisition?.providerLeaseId && sandboxLeaseAcquisition.providerLeaseId !== leaseRow.providerLeaseId ) { throw new Error("runner_harness_backup_lease_mismatch"); } const stamp = createNativeHarnessBackupStamp({ manifestPath: resolve(backup.root, "manifest.json"), sessionScopeId, authorizedProviderLeaseId: leaseRow.providerLeaseId, normalizedSessionId: backup.manifest.normalizedSessionId, runnerInstanceId: backup.manifest.runnerInstanceId, completedAt: backup.manifest.completedAt, }); const updated = await input.db .update(environmentLeases) .set({ metadata: { ...(leaseRow.metadata ?? {}), nativeHarnessBackup: stamp, }, updatedAt: new Date(), }) .where(eq(environmentLeases.id, remoteTarget.leaseId)) .returning({ id: environmentLeases.id }) .then((rows) => rows[0] ?? null); if (!updated) throw new Error("runner_harness_backup_lease_missing"); }; const restoreVerifiedHarnessBackup = async () => { if (!remoteTarget || !remoteCommandRunner) { throw new Error("runner_harness_backup_unavailable"); } const backup = verifyNativeHarnessBackup({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId, }); if (!backup) throw new Error("runner_harness_backup_unavailable"); await measureNativeRunnerSpan( input.trace, "harness_state.failover_restore", async () => { for (const directory of persistenceProfile.directories) { const targetPath = remotePersistencePath(directory); if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable"); await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: resolve(backup.root, directory.name), targetPath, mode: 0o700, }); } }, { attributes: { provider: input.execution.provider.kind, harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, stateSource: "verified_failover_backup", bytesTransferred: backup.bytes, }, }, ); const restored = await inspectRemoteHarnessState(); if ( !restored.complete || !restored.runnerState || !restored.providerSessionIdentity || canonicalJson(restored.providerSessionIdentity) !== canonicalJson(backup.manifest.providerSessionIdentity) ) { throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed"); } // A deliberately non-reusable environment receives a fresh provider lease // for every turn. Stamp that new lease as soon as the verified host backup // has been restored so a later provider/bootstrap failure can still clean // the ephemeral sandbox up without discarding the only durable copy. await recordHarnessBackupStampForCurrentLease(backup); return backup; }; const materializeRemoteHarnessLaunchState = async () => { if (!remoteTarget || !remoteCommandRunner) return; for (const directory of persistenceProfile.directories) { if (directory.location !== "filesystem") continue; const targetPath = remotePersistencePath(directory); if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable"); const escapedTarget = targetPath.replaceAll("'", "'\\''"); const created = await remoteCommandRunner.execute({ command: "sh", args: ["-c", `umask 077; install -d -m 0700 '${escapedTarget}'`], bypassSession: true, timeoutMs: 10_000, }); if (created.exitCode !== 0 || created.timedOut) { throw new Error(`runner_remote_directory_staging_failed: exit=${created.exitCode} timedOut=${created.timedOut}${created.stderr.trim() ? ` ${redactSensitiveText(created.stderr).trim().slice(-512)}` : ""}`); } // Codex launch credentials are intentionally excluded from failover // backups. Re-materialize only those launch-time files into a fresh or // replacement sandbox after the durable history has been restored. if (directory.name !== "codex-home") continue; const localDirectory = resolve(root, directory.name); for (const name of ["auth.json", "config.toml"] as const) { const sourcePath = resolve(localDirectory, name); if (!existsSync(sourcePath)) continue; await stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, sourcePath, targetPath: posix.join(targetPath, name), mode: 0o600, }); } } }; const ensureRemoteRunner = async (stageLaunchAssets = true) => { await measureNativeRunnerSpan( input.trace, "stage.sync", async () => { if (!selectedRemoteMode) { throw new Error("runner_remote_transport_mode_unresolved"); } try { await verifyRemoteRunner(selectedRemoteMode); await verifyRemoteCodex(); if (requiresRemoteProviderPack) { if (!activeRemoteProviderPackRoot) { throw new Error( "runner_remote_provider_artifact_incompatible: provider pack was not prepared", ); } await verifyRemoteProviderPack(activeRemoteProviderPackRoot); } } catch { remotePrepared = false; await prepareRemoteRunner(selectedRemoteMode); } if (!remoteHarnessStatePrepared) { await measureNativeRunnerSpan(input.trace, "stage.asset.home", () => measureNativeRunnerSpan( input.trace, "session.checkpoint.restore", async () => { if ( remoteTarget?.transport === "sandbox" && remoteCommandRunner ) { const acquisitionRecordedAtMs = Date.now(); const backupAvailable = harnessBackupCandidates(root).some( (candidate) => existsSync(resolve(candidate, "manifest.json")), ); const restoreIntoCreatedSandbox = shouldRestoreNativeHarnessBackupIntoSandbox({ acquisitionOutcome: sandboxLeaseAcquisition?.outcome ?? null, reusableLeaseConfigured: remoteTarget.reusableLeaseConfigured, backupAvailable, }); await input.trace?.record({ name: "sandbox.lease.acquisition", startedAtMs: acquisitionRecordedAtMs, endedAtMs: acquisitionRecordedAtMs, attributes: { provider: remoteTarget.providerKey ?? "sandbox", harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, outcome: sandboxLeaseAcquisition?.outcome ?? "unknown", stateSource: sandboxLeaseAcquisition?.outcome === "replacement" || restoreIntoCreatedSandbox ? "verified_failover_backup" : sandboxLeaseAcquisition?.outcome === "resumed" ? "sandbox_filesystem" : "new_sandbox", bytesTransferred: 0, }, }); if (sandboxLeaseAcquisition?.outcome === "resumed") { const reuseStartedAtMs = Date.now(); const state = await measureNativeRunnerSpan( input.trace, "sandbox.lease.resume", inspectRemoteHarnessState, { attributes: { provider: remoteTarget.providerKey ?? "sandbox", harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, outcome: "resumed", }, }, ); if ( !state.complete || !state.runnerState || !state.providerSessionIdentity ) { if (state.incompleteReason !== "unavailable" || backupAvailable || !(await claimUntouchedSessionInResumedLease())) { throw new Error(`runner_harness_state_mismatch: resumed_${state.incompleteReason}`); } } else { await recordInPlaceHarnessReuse( state.providerSessionIdentity, reuseStartedAtMs, ); } } else if ( sandboxLeaseAcquisition?.outcome === "replacement" ) { await measureNativeRunnerSpan( input.trace, "sandbox.lease.replacement", restoreVerifiedHarnessBackup, { attributes: { provider: remoteTarget.providerKey ?? "sandbox", harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, outcome: "replacement", reason: sandboxLeaseAcquisition.reason ?? "unknown", }, }, ); } else if (restoreIntoCreatedSandbox) { await measureNativeRunnerSpan( input.trace, "sandbox.lease.replacement", restoreVerifiedHarnessBackup, { attributes: { provider: remoteTarget.providerKey ?? "sandbox", harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, outcome: "created", reason: "reuse_disabled", }, }, ); } else { const reuseStartedAtMs = Date.now(); const state = await inspectRemoteHarnessState(); if ( state.complete && state.runnerState && state.providerSessionIdentity ) { // Re-entry while this newly-created lease is already running (for // example a transport reconnect) still uses the in-place state. await recordInPlaceHarnessReuse( state.providerSessionIdentity, reuseStartedAtMs, ); } else if (backupAvailable) { // A continuation that has a durable backup but no recorded reusable // lease was not provider-confirmed lost. Never silently create a new // provider session from that ambiguous state. throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease"); } } } else if (remoteTarget && remoteCommandRunner) { // Local and generic SSH execution retain their existing checkpoint // behavior. The manifest-only failover gate applies to managed sandbox // replacement, where provider lease provenance is available. for (const directory of persistenceProfile.directories) { const localDirectory = resolve(root, directory.name); const remoteDirectory = remotePersistencePath(directory); if ( remoteDirectory && existsSync(localDirectory) && !(await remoteRunnerPathExists({ runner: remoteCommandRunner, path: directory.name === "runner" ? posix.join(remoteDirectory, "runner-state.json") : remoteDirectory, kind: directory.name === "runner" ? "file" : "directory", })) ) { await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: localDirectory, targetPath: remoteDirectory, mode: 0o700, excludeEntries: directory.excludeEntries, }); } } } }, { attributes: { mode: remoteTarget?.transport ?? "local", lifecycleMode: input.execution.session.lifecyclePolicy.mode, }, }, ), ); remoteHarnessStatePrepared = true; } // Authority rotation needs durable history, before the transport writes // this turn's launch files. Stage assets only at the actual launch so we // neither upload stale context nor transfer every bundle twice on resume. if (!stageLaunchAssets) return; // Resume can prepare/rotate durable state before the transport creates // this invocation's isolated Codex auth/config. The later launch must // still stage those fresh files even when history was already restored. // Launch material is never recovered from a failover backup. await materializeRemoteHarnessLaunchState(); if ( remoteTarget && remoteCommandRunner && remoteRunnerFilesystemRoot && sourceRuntimeContext && remoteRuntimeContext ) { await measureNativeRunnerSpan( input.trace, "stage.asset.runtime_context", async () => { await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: sourceRuntimeContext.instructions.bundle.rootPath, targetPath: remoteRuntimeContext.instructions.bundle.rootPath, mode: 0o555, }); for ( let index = 0; index < sourceRuntimeContext.skills.length; index += 1 ) { await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: sourceRuntimeContext.skills[index]!.bundle.rootPath, targetPath: remoteRuntimeContext.skills[index]!.bundle.rootPath, mode: 0o555, }); } await stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: resolve(root, "runtime-context.json"), targetPath: posix.join( remoteRunnerFilesystemRoot, "runtime-context.json", ), mode: 0o600, }); }, ); } if (remoteTarget && remoteCommandRunner && remoteCaBundleMapping) { const caBundleMapping = remoteCaBundleMapping; await measureNativeRunnerSpan( input.trace, "stage.asset.ca_bundle", () => stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, sourcePath: caBundleMapping.sourcePath, targetPath: caBundleMapping.targetPath, mode: 0o600, }), ); } }, { attributes: { target: remoteTarget?.transport ?? "local", lifecycleMode: input.execution.session.lifecyclePolicy.mode, }, }, ); }; const checkpointRemoteRunner = async ( settlement: "settled" | "unsettled", ) => { if ( !remoteCommandRunner || !remoteStateDirectory || !remoteRunnerFilesystemRoot ) return; // Transport release also runs when provider bootstrap failed. In that case // runnerd has no durable provider identity (and may not have created the // provider persistence directory at all), so attempting a failover backup // would replace the original provider error with // `runner_harness_state_mismatch`. Only checkpoint a harness that runnerd // has proved complete. A malformed or identity-conflicting state still // throws from inspectRemoteHarnessState and therefore fails closed. let checkpointable = await inspectRemoteHarnessState(); // The remote runner writes its suspended lifecycle and provider state // before the outer process-owner RPC necessarily observes completion. // Allow a very small bounded visibility window without ever accepting an // active, incomplete, malformed, or identity-conflicting checkpoint. for (let attempt = 1; !checkpointable.complete && attempt < 3; attempt++) { await new Promise((resolve) => setTimeout(resolve, 100)); checkpointable = await inspectRemoteHarnessState(); } if (!checkpointable.complete) { const incompleteFailure = remoteCheckpointIncompleteFailure( settlement, checkpointable.incompleteReason, ); await input.onLog?.( "stderr", `[paperclip-runner] remote checkpoint ${incompleteFailure ? "failed" : "skipped"}: exact suspended harness state unavailable (process=${settlement} reason=${checkpointable.incompleteReason})\n`, ); if (incompleteFailure) throw incompleteFailure; return; } // The computer ledger and exact remote placement are the recovery authority. // Its durable provider state stays on the persistent machine; importing the // sandbox archive would create a second authority and delay bounded shutdown. if (remoteTarget?.transport === "computer") return; const backupSpanAttributes = { provider: input.execution.provider.kind, harness: input.execution.session.driverKind, lifecycleMode: input.execution.session.lifecyclePolicy.mode, stateSource: "sandbox_filesystem", bytesTransferred: 0, }; try { await measureNativeRunnerSpan( input.trace, "session.checkpoint.persist", () => measureNativeRunnerSpan( input.trace, "harness_state.backup.persist", async () => { const verified = await inspectRemoteHarnessState(); if ( !verified.complete || !verified.runnerState || !verified.providerSessionIdentity ) { throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete"); } const providerSessionIdentity = verified.providerSessionIdentity; const backupRoot = harnessBackupRoot(root); mkdirSync(backupRoot, { recursive: true, mode: 0o700 }); const pendingRoot = resolve( backupRoot, `.pending-${randomUUID()}`, ); mkdirSync(pendingRoot, { recursive: true, mode: 0o700 }); try { for (const directory of persistenceProfile.directories) { const sourcePath = remotePersistencePath(directory); if (!sourcePath) throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable"); const targetPath = resolve(pendingRoot, directory.name); await syncRemoteRunnerDirectoryOut({ runner: remoteCommandRunner, sourcePath, targetPath, mode: 0o700, excludeEntries: directory.excludeEntries, }); if (!existsSync(targetPath)) { throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing"); } } const manifest = buildNativeHarnessBackupManifest({ backupRoot: pendingRoot, execution: input.execution, runnerInstanceId: input.runnerInstanceId, providerSessionIdentity, sourceProviderLeaseId: sandboxLeaseAcquisition?.providerLeaseId ?? remoteTarget?.leaseId ?? input.durableEnvironmentLeaseId ?? "unknown", }); backupSpanAttributes.bytesTransferred = manifest.directories.reduce( (total, directory) => total + directory.bytes, 0, ); const temporaryManifest = resolve( pendingRoot, "manifest.json.tmp", ); const manifestPath = resolve(pendingRoot, "manifest.json"); writeFileSync(temporaryManifest, JSON.stringify(manifest), { encoding: "utf8", mode: 0o600, }); renameSync(temporaryManifest, manifestPath); const currentRoot = resolve(backupRoot, "current"); const previousRoot = resolve(backupRoot, "previous"); removeNativeHarnessBackup(previousRoot); let movedCurrent = false; if (existsSync(currentRoot)) { renameSync(currentRoot, previousRoot); movedCurrent = true; } try { renameSync(pendingRoot, currentRoot); if ( remoteTarget?.transport === "sandbox" && remoteTarget.leaseId ) { await recordHarnessBackupStampForCurrentLease({ root: currentRoot, manifest, bytes: backupSpanAttributes.bytesTransferred, }); } } catch (error) { if (existsSync(currentRoot)) { removeNativeHarnessBackup(currentRoot); } if ( movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot) ) { renameSync(previousRoot, currentRoot); } throw error; } removeNativeHarnessBackup(previousRoot); } finally { removeNativeHarnessBackup(pendingRoot); } }, { attributes: backupSpanAttributes, }, ), { parentName: "task.settle" }, ); } catch (error) { const detail = redactSensitiveText( error instanceof Error ? error.message : String(error), ) .replace(/[\r\n]+/g, " ") .slice(0, 512); await input.onLog?.( "stderr", `[paperclip-runner] remote checkpoint failed: ${detail || "unknown failure"}\n`, ); throw error; } }; const prepareExternalRunnerState = remoteTarget && remoteCommandRunner ? async () => { selectedRemoteMode ??= resolveRemoteRunnerTransportMode({ target: remoteTarget, runnerIngressAuthorized: input.runnerIngressAuthorized === true, }); await ensureRemoteRunner(false); if (remoteTarget.transport === "computer" && remoteStateDirectory) { await sealRetiredComputerRunnerState({ db: input.db, execution: input.execution, root, target: remoteTarget, runner: remoteCommandRunner, stateDirectory: remoteStateDirectory }); } } : undefined; const archiveExternalRunnerState = remoteCommandRunner && remoteStateDirectory && remoteSessionRoot ? async (archive: { archiveKey: string }) => { if (!/^[0-9a-f]{24}$/.test(archive.archiveKey)) { throw new Error("runner_remote_authority_archive_invalid"); } const sourcePath = posix.join( remoteStateDirectory, "runner-state.json", ); const archiveDirectory = posix.join( remoteSessionRoot, "authority-epochs", `epoch-${archive.archiveKey}`, ); const archivedStatePath = posix.join( archiveDirectory, "runner-state.json", ); const result = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", 'set -eu; if test -L "$2" || { test -e "$2" && test ! -d "$2"; }; then exit 1; fi; if test -f "$1" && test ! -L "$1" && test ! -e "$3" && test ! -L "$3"; then umask 077; install -d -m 0700 "$2"; mv -- "$1" "$3"; elif test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"; then :; else exit 1; fi; base64 < "$3"', "paperclip-runner-authority-archive", sourcePath, archiveDirectory, archivedStatePath, ], bypassSession: true, timeoutMs: 10_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_remote_authority_archive_failed"); } try { return record( JSON.parse( Buffer.from( result.stdout.replace(/\s+/g, ""), "base64", ).toString("utf8"), ), ); } catch { throw new Error("runner_remote_authority_archive_failed"); } } : undefined; const remoteProcessLauncher = remoteTarget && remoteCommandRunner && remoteBinary && remoteStateDirectory ? createRemoteRunnerProcessLauncher({ target: remoteTarget, runner: remoteCommandRunner, remoteBinary, processIdentityPath: posix.join( remoteStateDirectory, "runner-process.identity", ), stateDirectory: remoteStateDirectory, diagnosticsDirectory: posix.join(remoteSessionRoot!, "diagnostics"), runnerInstanceId: input.runnerInstanceId, ensureArtifact: ensureRemoteRunner, onSpawn: input.onSpawn, onLog: input.onLog, trace: input.trace, onRunnerProcessSpawned: () => resolveRemoteRunnerProcessSpawned?.(), onLaunchAttempt: input.onRemoteLaunchAttempt, }) : undefined; const runnerExecution: NativeExecutionInput = remoteTarget ? { ...input.execution, workspace: { ...input.execution.workspace, cwd: remoteTarget.remoteCwd, }, } : input.execution; const isGrok = input.execution.provider.kind === "acpx" && input.execution.provider.agent === "grok"; let effectiveRunnerEnvironmentBase: NodeJS.ProcessEnv = { ...(isGrok ? input.runnerEnvironment ?? {} : resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment)), }; const grokCredential = isGrok ? await prepareGrokRunnerCredentials({ companyId: input.execution.binding.companyId, environment: effectiveRunnerEnvironmentBase, remote: Boolean(remoteTarget), managedHome: input.managedAiCredentialHome, }) : null; if (grokCredential) effectiveRunnerEnvironmentBase = grokCredential.environment; if (relayAssignedMcp) { // The server-held tool authority owns this credential. Do not deliver a // duplicate HTTP MCP server or its bearer token to the remote provider. delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_NAME; delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_URL; delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_TOKEN; } // This authority bit is derived only from the selected execution target. // Never let an agent, environment binding, or host variable disable the // Codex sandbox for a local runner by supplying the same key. delete effectiveRunnerEnvironmentBase.PAPERCLIP_RUNNER_EXTERNAL_SANDBOX; const effectiveRunnerEnvironment: NodeJS.ProcessEnv = remoteRuntimeRoot ? { ...effectiveRunnerEnvironmentBase, // The provider home is runner-owned state, not the execution workspace. // Codex's permission profile explicitly denies HOME and CODEX_HOME. If // either points at remoteCwd, that deny rule shadows the workspace write // grant and the provider cannot initialize its shell sandbox or edit. HOME: posix.join(remoteRunnerFilesystemRoot!, "codex-home"), CODEX_HOME: posix.join(remoteRunnerFilesystemRoot!, "codex-home"), PAPERCLIP_WORKSPACE_CWD: remoteTarget!.remoteCwd, ...(adapterExecutionTargetIsCommandBacked(remoteTarget) ? { PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1" } : {}), } : { ...effectiveRunnerEnvironmentBase, PAPERCLIP_WORKSPACE_CWD: input.execution.workspace.cwd, }; const archiveContinuityState = async () => { if (hasRetainedWarmTransitionEvidence(root)) { throw new Error("native_runner_warm_transition_recovery_unproven"); } const archiveToken = `${Date.now()}-${randomUUID()}`; const archiveRoot = resolve(root, "continuity-breaks", archiveToken); mkdirSync(archiveRoot, { recursive: true, mode: 0o700 }); for (const name of [ "control-plane", "runner", "codex-home", "opencode", "acpx", // Backups belong to the retired provider session. Leaving them active // makes the fresh replacement look like ambiguous lost harness state. // Keep their evidence inside the same continuity-break archive. "failover-backups", ]) { const source = resolve(root, name); if (existsSync(source)) renameSync(source, resolve(archiveRoot, name)); } if (remoteCommandRunner && remoteSessionRoot) { const escapedSource = remoteSessionRoot.replaceAll("'", "'\\''"); const archivedRemote = `${remoteSessionRoot}.continuity-break-${archiveToken}`; const escapedArchive = archivedRemote.replaceAll("'", "'\\''"); const result = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", `if test -d '${escapedSource}'; then mv '${escapedSource}' '${escapedArchive}'; fi`, ], bypassSession: true, timeoutMs: 30_000, }); if (result.exitCode !== 0 || result.timedOut) { throw new Error("runner_continuity_break_archive_failed"); } } remotePrepared = false; }; const localRecoveryProcess = input.restartRecovery?.kind === "reattach_existing_runner" ? input.restartRecovery.process : null; const adoptedProcess = input.restartRecovery?.kind === "reattach_remote_runner" ? await verifyRemoteRunnerReattachment({ claim: input.restartRecovery, target, identity: durableIdentity ?? {}, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), }) : target.kind === "local" && localRecoveryProcess ? { ...localRecoveryProcess, isAlive: () => verifiedRecoveryProcessIsAlive(localRecoveryProcess), signal: (signal: NodeJS.Signals) => signalVerifiedRecoveryProcess(localRecoveryProcess, signal), } : undefined; if (adoptedProcess && remoteTarget) { const markSpawned = resolveRemoteRunnerProcessSpawned as (() => void) | null; markSpawned?.(); } const executeCurrentToolAuthority = ( call: Parameters[0], ) => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_tool_authority_unavailable"); return current.execute(call); }; const backend = createNativeSessionBackend(runnerExecution, { runnerInstanceId: retainedTransition?.runnerInstanceId ?? input.runnerInstanceId, environment: effectiveRunnerEnvironment, workingDirectoryAuthority: remoteTarget ? "remote_runner" : "local_filesystem", onSpawn: input.onSpawn, dynamicTools, completionFeedback: async (result) => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_tool_authority_unavailable"); await current.definitions(); // Reject a revoked run authority before reading task state. return nativeCompletionFeedback(input.db, current.runId, result); }, dynamicToolHandler: executeCurrentToolAuthority, acpxDynamicToolHandler: executeCurrentToolAuthority, opencodeRuntimeDirectory: resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "opencode", ), acpxRuntimeDirectory: resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "acpx", ), codexTransportFactory: (recoveryContext) => createRunnerdCodexTransport({ onSpawn: input.onSpawn, provider: input.execution.provider.kind === "codex" ? "codex" : input.execution.provider.kind === "opencode" ? "opencode" : input.execution.provider.kind === "claude_managed" ? "claude_managed" : input.execution.provider.kind === "aws_agentcore" ? "aws_agentcore" : input.execution.provider.kind === "acpx" ? "acpx" : undefined, ...(input.execution.provider.kind === "acpx" ? { acpxAgent: input.execution.provider.agent, // Read only the server operator environment, never agent/runtime env. acpxCandidateProfile: input.execution.provider.agent === "pi" ? undefined : resolveAcpxQualification(input.execution.provider, process.env), acpxPermissionMode: input.execution.provider.permissionMode, acpxMode: input.execution.provider.mode, piThinkingLevel: input.execution.provider.piThinkingLevel, acpxPermissionModePinned: input.execution.schema === "paperclip.native-execution-input.v4" || input.execution.schema === "paperclip.native-execution-input.v5", acpxRuntimeDirectory: remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, "acpx") : resolve( resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "acpx", ), } : {}), ...(input.execution.provider.kind === "opencode" ? { opencodePermissionMode: input.execution.provider.permissionMode, } : {}), ...(input.execution.provider.kind === "claude_managed" ? { managedProfile: { ...input.execution.provider.managedProfile, maxSessionListCostUsd: input.execution.provider.maxSessionListCostUsd, model: input.execution.provider.model, }, } : {}), ...(input.execution.provider.kind === "aws_agentcore" ? { agentCoreProfile: { ...input.execution.provider.agentCoreProfile, maxEstimatedSessionCostUsd: input.execution.provider.maxEstimatedSessionCostUsd, maxIterations: input.execution.provider.invocationLimits.maxIterations, maxOutputTokens: input.execution.provider.invocationLimits.maxOutputTokens, timeoutSeconds: input.execution.provider.invocationLimits.timeoutSeconds, model: input.execution.provider.model, }, } : {}), ...(expectedProviderPackManifest && stagedRemoteProviderPackRoot ? { providerNodeCommand: posix.join( stagedRemoteProviderPackRoot, expectedProviderPackManifest.payload.artifacts.nodeCommand.path, ), providerNodeCommandSha256: expectedProviderPackManifest.payload.artifacts.nodeCommand .sha256, providerPackAuthorityDigest: expectedProviderPackManifest.digest, opencodeCommand: posix.join( stagedRemoteProviderPackRoot, expectedProviderPackManifest.payload.artifacts .opencodeExecutable.path, ), opencodeCommandSha256: expectedProviderPackManifest.payload.artifacts .opencodeExecutable.sha256, opencodeProxyPath: posix.join( stagedRemoteProviderPackRoot, expectedProviderPackManifest.payload.artifacts.opencodeProxy .path, ), opencodeProxySha256: expectedProviderPackManifest.payload.artifacts.opencodeProxy .sha256, acpxSidecarPath: posix.join( stagedRemoteProviderPackRoot, expectedProviderPackManifest.payload.artifacts.acpxSidecar.path, ), acpxSidecarSha256: expectedProviderPackManifest.payload.artifacts.acpxSidecar .sha256, } : {}), stateDirectory: root, runnerStateDirectory: remoteStateDirectory, readRunnerState: remoteStateDirectory && remoteCommandRunner ? () => readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory, }) : undefined, prepareExternalRunnerState, archiveExternalRunnerState, runnerBinary: controllerRunnerBinary, codexCommand: remoteCodexBinary ?? undefined, sourceCodexHome: remoteTarget ? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment)) : undefined, runnerProcessLauncher: remoteProcessLauncher, runnerReconnectGraceMs: remoteTarget ? 120_000 : undefined, adoptExistingRunner: adoptedProcess, environment: effectiveRunnerEnvironment, onDiagnostic: (message) => { void input.onLog?.( "stderr", `[paperclip-runner] runnerd diagnostic: ${redactSensitiveText(message).slice(-4_096)}\n`, ); }, lifecyclePolicy: input.execution.session.lifecyclePolicy, runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : null, runnerRuntimeContext: remoteRuntimeContext, baseInstructions: recoveryContext?.baseInstructions, runnerFilesystemRoot: remoteRunnerFilesystemRoot ?? undefined, resumeWorkingDirectory: runnerExecution.workspace.cwd, externallySandboxed: adapterExecutionTargetIsCommandBacked(remoteTarget), computerTool: nativeComputerToolForExecution(input.execution, remoteTarget), persistentAgentHome: remoteTarget?.transport === "computer" ? remoteTarget.fileAuthority.agentHome : undefined, opencodeRuntimeDirectory: remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, "opencode") : undefined, resumeDynamicTools: dynamicTools, resumeCompletionContract: { revision: input.execution.completionContract.contract.revision, criterionIds: input.execution.completionContract.contract.criteria.map( (criterion) => criterion.id, ), }, resumeActiveTurnId: recoveryContext?.persistedSession?.activeTurnId ?? null, resumeProviderSession: recoveryContext?.persistedSession, providerRecoveryPolicy: recoveryContext?.providerRecoveryPolicy ?? (input.execution.provider.kind === "acpx" && input.execution.interactionResponses.length > 0 ? "allow_replacement_after_governed_wait" : undefined), prpIdentity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, runId: input.execution.binding.runId, normalizedSessionId: input.execution.session.normalizedSessionId ?? `session-${input.execution.binding.runId}`, turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}`, }, warmTransitionRegistrationMode: retainedTransition ? "routed_connect" : undefined, authorizeWarmTransitionRecovery: retainedTransition ? async ( stage: "before_bootstrap" | "before_spawn" | "before_authentication", ) => { if (!recoveryPending) return; const current = await verifyWarmTransitionRestart(input); if ( current.transitionId !== retainedTransition.transitionId || (stage === "before_bootstrap" && current.stateFingerprint !== retainedTransition.stateFingerprint) ) { throw new Error( "native_runner_warm_transition_recovery_unproven", ); } } : undefined, onWarmTransitionRecoveryCompleted: retainedTransition ? (completed: { transitionId: string }) => { // Only the package's fresh completed new-authority snapshot can // reach this callback. A tombstone or caller hint cannot retire // the server's pending-recovery admission checks. if (completed?.transitionId !== retainedTransition.transitionId) { throw new Error( "native_runner_warm_transition_recovery_unproven", ); } recoveryPending = false; } : undefined, controlPlaneRegistration: async (authority, attachmentIdentity) => { if (retainedTransition && recoveryPending) { const current = await verifyWarmTransitionRestart(input); if ( current.stateFingerprint !== retainedTransition.stateFingerprint ) { throw new Error( "native_runner_warm_transition_recovery_unproven", ); } } return measureNativeRunnerSpan( input.trace, "runner.transport.connect", async () => { if (target.kind === "local") { const selectedAtMs = Date.now(); await input.trace?.record({ name: "runner.transport.selected", parentName: "runner.transport.connect", startedAtMs: selectedAtMs, endedAtMs: selectedAtMs, attributes: { mode: "local_loopback", connectionOwner: "runnerd", }, }); await input.onLog?.( "stderr", "[paperclip-runner] transport mode=local_loopback state=connecting\n", ); const registration = await measureNativeRunnerSpan( input.trace, "runner.prp.route.register", () => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, runId: attachmentIdentity?.runId ?? input.execution.binding.runId, authority, }), ); return { ...registration, startupFailureCode: "runner_local_connect_failed" as const, }; } const requiredMode = resolveRemoteRunnerTransportMode({ target, runnerIngressAuthorized: input.runnerIngressAuthorized === true, }); let transport: PaperclipRunnerTransport; if (requiredMode === "dial_wss") { // Validate eligibility before staging any artifact. transport = await measureNativeRunnerSpan( input.trace, "runner.transport.resolve", () => resolvePaperclipRunnerTransport({ target, runId: attachmentIdentity?.runId ?? input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath, runnerIngressAuthorized: input.runnerIngressAuthorized === true, }), ); if ( transport.mode === "direct_outbound" && transport.caBundlePath && !existsSync(transport.caBundlePath) ) { throw new Error( "runner_direct_wss_failed: configured runner CA bundle is unavailable", ); } await measureNativeRunnerSpan( input.trace, "runner.artifact.prepare", () => prepareRemoteRunner(requiredMode), ); } else { await measureNativeRunnerSpan( input.trace, "runner.artifact.prepare", () => prepareRemoteRunner(requiredMode), ); // Provider endpoint acquisition happens only after runnerd is staged // and its listener capability has been verified. transport = await measureNativeRunnerSpan( input.trace, "runner.ingress.acquire", () => resolvePaperclipRunnerTransport({ target, runId: attachmentIdentity?.runId ?? input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath, runnerIngressAuthorized: true, }), ); } const selectedAtMs = Date.now(); await input.trace?.record({ name: "runner.transport.selected", parentName: "runner.transport.connect", startedAtMs: selectedAtMs, endedAtMs: selectedAtMs, attributes: { mode: transport.mode, connectionOwner: transport.mode === "provider_ingress" ? "paperclip" : "runnerd", }, }); await input.onLog?.( "stderr", `[paperclip-runner] transport mode=${transport.mode} state=connecting\n`, ); if (transport.mode === "direct_outbound") { const inbound = await measureNativeRunnerSpan( input.trace, "runner.prp.route.register", () => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId, runId: attachmentIdentity?.runId ?? input.execution.binding.runId, authority, }), { parentName: "runner.transport.connect" }, ); let caBundlePath = transport.caBundlePath; if (caBundlePath && remoteBinary) { const remoteCaBundlePath = posix.join( posix.dirname(remoteBinary), "runner-ca-bundle.pem", ); remoteCaBundleMapping = { sourcePath: caBundlePath, targetPath: remoteCaBundlePath, }; caBundlePath = remoteCaBundlePath; } return { connection: { mode: "connect" as const, connectUrl: transport.connectUrl, ...(caBundlePath ? { caBundlePath } : {}), }, startupFailureCode: "runner_direct_wss_failed" as const, checkpoint: checkpointRemoteRunner, release: () => inbound.release(), }; } if (transport.mode !== "provider_ingress") { throw new Error("runner_transport_mode_changed_after_dispatch"); } let outbound: ReturnType | null = null; let activation: Promise | null = null; return { connection: { mode: "listen" as const, listenAddress: transport.listenAddress, listenPort: transport.listenPort, listenPath: transport.listenPath, }, activate: () => { activation = measureNativeRunnerSpan( input.trace, "runner.transport.activation", async () => { await measureNativeRunnerSpan( input.trace, "runner.ingress.wait_for_process", () => remoteRunnerProcessSpawned, ); outbound = connectRunnerPrpIngress({ authority, endpoint: transport.ingress, onStateChange: (state, failureCode) => { void input.onLog?.( "stderr", `[paperclip-runner] transport mode=provider_ingress state=${state}${failureCode ? ` failure=${failureCode}` : ""}\n`, ); }, }); }, { parentName: "runner.session.startup" }, ); // Cancellation can release the sandbox while activation is // still waiting for the remote runner process. Attach a // rejection observer immediately; `ready()` still awaits the // original promise and reports the same startup failure, but // an early teardown can no longer crash the controller with // an unhandled plugin RPC rejection. void activation.catch(() => undefined); }, ready: async () => { await measureNativeRunnerSpan( input.trace, "runner.transport.ready", async () => { await activation; if (!outbound) throw new Error("runner_ingress_unavailable"); await measureNativeRunnerSpan( input.trace, "runner.prp.authenticate", () => outbound!.ready, ); }, { parentName: "runner.session.startup" }, ); }, get failure() { return outbound?.failure; }, startupFailureCode: "runner_ingress_unavailable" as const, checkpoint: checkpointRemoteRunner, release: async () => { if (outbound) await outbound.close(); else await transport.ingress.close(); }, }; }, ); }, }).transport, }); const boundManagedSessions = new WeakSet(); const wrapManagedSession = (session: NativeSession): NativeSession => { if (isGrok && grokCredential?.home && !boundManagedSessions.has(session)) { boundManagedSessions.add(session); // The launch runtime directory already ends in "acpx"; ACPX adds its // own namespace beneath it in resolveAcpxRuntimeRoot. const relativeHome = `acpx/acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(input.execution))}/grok-home`; const localHome = resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", relativeHome); const remoteHome = remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, relativeHome) : null; const readAuth = async (name: string): Promise => { if (!remoteHome || !remoteCommandRunner) return Buffer.from(await readLocalAiCredentialFile(join(localHome, name))); const script = `const fs=require('node:fs'),path=require('node:path');let fd;try{const file=process.argv[1];let parent=path.dirname(file);while(true){if(!fs.lstatSync(parent).isDirectory())throw Error('directory');const next=path.dirname(parent);if(next===parent)break;parent=next;}fd=fs.openSync(file,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);const st=fs.fstatSync(fd);if(!st.isFile()||st.uid!==process.getuid()||(st.mode&511)!==384||st.size>65536)throw Error('credential');const b=Buffer.alloc(65537);let n=0;while(n65536)throw Error('size');process.stdout.write(b.subarray(0,n).toString('base64'));b.fill(0);}catch(e){process.exitCode=e.code==='ENOENT'?66:1;}finally{if(fd!==undefined)fs.closeSync(fd);}`; const result = await remoteCommandRunner.execute({ command: "node", args: ["-e", script, posix.join(remoteHome, name)], bypassSession: true, timeoutMs: 10000 }); if (result.exitCode !== 0 || result.timedOut) throw Object.assign(new Error("Grok credential refresh handoff unavailable"), { code: result.exitCode === 66 ? "ENOENT" : "INVALID_CREDENTIAL" }); return Buffer.from(result.stdout, "base64"); }; return bindManagedNativeCredentialTurn(session, { copyBack: async () => { await copyBackGrokAuth({ hostHomeDir: grokCredential.home!, log: () => {}, readSandboxAuth: () => readAuth("auth.json").catch((error) => { if (error.code !== "ENOENT") throw error; return readAuth("auth-refresh.json"); }), }); }, remove: async () => { for (const name of ["auth.json", "auth-refresh.json", "auth-refresh.json.tmp"]) { rmSync(join(localHome, name), { force: true }); if (remoteHome && remoteCommandRunner) { const result = await remoteCommandRunner.execute({ command: "rm", args: ["-f", "--", posix.join(remoteHome, name)], bypassSession: true, timeoutMs: 10000 }); if (result.exitCode !== 0 || result.timedOut) throw new Error("Grok credential cleanup failed"); } } }, }); } if (!input.managedAiCredentialHome || input.execution.provider.kind !== "codex" || boundManagedSessions.has(session)) return session; boundManagedSessions.add(session); const remoteAuth = remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, "codex-home", "auth.json") : null; const localAuth = join(root, "codex-home", "auth.json"); return bindManagedNativeCredentialTurn(session, { copyBack: async () => { await copyBackCodexAuth({ hostAuthPath: join(input.managedAiCredentialHome!, "auth.json"), readSandboxAuth: async () => { if (!remoteAuth || !remoteCommandRunner) return readFileSync(localAuth); const result = await remoteCommandRunner.execute({ command: "base64", args: [remoteAuth], bypassSession: true, timeoutMs: 10000 }); if (result.exitCode !== 0 || result.timedOut) throw new Error("AI credential copy-back failed"); return Buffer.from(result.stdout, "base64"); }, log: () => {}, }); }, remove: async () => { rmSync(localAuth, { force: true }); if (remoteAuth && remoteCommandRunner) await remoteCommandRunner.execute({ command: "rm", args: ["-f", "--", remoteAuth], bypassSession: true, timeoutMs: 10000 }); }, }); }; const priorAuthorityEpoch = sessionToolAuthorityEpochs.get(sessionScopeId); if (priorAuthorityEpoch && priorAuthorityEpoch !== authorityEpoch) { priorAuthorityEpoch.revoke(); } sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch); return { bindManagedSession: wrapManagedSession, descriptor: () => backend.descriptor(), openSession: async (sessionInput) => wrapManagedSession(await backend.openSession(sessionInput)), recoverSession: async (snapshot, options) => { const result = backend.recoverSession ? await backend.recoverSession(snapshot, options) : { recovered: false, reason: "driver does not support recovery" }; return result.session ? { ...result, session: wrapManagedSession(result.session) } : result; }, openReplacementSession: async (sessionInput) => { await measureNativeRunnerSpan( input.trace, "provider.session.archive_before_replacement", archiveContinuityState, { parentName: "native.session.execute" }, ); return wrapManagedSession(await backend.openSession(sessionInput)); }, } satisfies NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession }; }