Files
PaperClipAI/server/src/services/native-runtime/native-session-executor.ts
T

14305 lines
588 KiB
TypeScript

import { computerService } from "../../modules/computers/index.js";
import { readNativeComputerWorkspaceReference } from "./native-workspace-sync.js";
import { adapterExecutionTargetIsCommandBacked, type AdapterComputerExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { configuredEnvironment } from "../../vendor/paperclip-runner/index.js";
import { agents } from "@paperclipai/db";
import { isCloudManagedInstance } from "../cloud-instance.js";
import { dotRunnerBroker } from "../dot-runner-broker.js";
import { CURSOR_DISTRIBUTION_PINS, QUALIFIED_ACPX_PROFILES, QUALIFIED_ACPX_VERSION } from "../../vendor/paperclip-runner/index.js";
import { isProviderMode } from "../../vendor/paperclip-runner/index.js";
import { bundledRemoteProviderPackManifestPath, bundledRemoteRunnerBinary } from "../../vendor/paperclip-runner/index.js";
import { nativeRetryCancellationEligible, rethrowNativeCancellationLockConflict, assertCancellationRequest, cancellationIntentId as callerCancellationIntentId, cancellationRequestId } from "./native-cancellation-request.js";
import { readNativePlanWait } from "./native-plan-wait.js";
import { nativeProviderLifecycle } from "./provider-lifecycle.js";
import { NativePermissionDeclinedError, readCompletedNativePermissionDecline } from "./native-permission-decline.js";
import { activeIssueInteractionCondition } from "../issue-question-context.js";
import { resolveAcpxQualification } from "./acpx-qualification.js";
import { readLocalAiCredentialFile } from "../local-ai-credential-file.js";
import { prepareGrokRunnerCredentials } from "./grok-runner-credentials.js";
import { copyBackGrokAuth } from "@paperclipai/adapter-grok-local/server";
import { inferOpenAiCompatibleBiller, type AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import {
isSupportedRemoteCodexVersion,
parseCodexCliVersion,
REMOTE_CODEX_SUPPORTED_RANGE,
} from "./codex-runtime-compatibility.js";
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js";
import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js";
import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } from "./managed-native-credentials.js";
import { createLocalNativeQuestionBridge } from "./local-native-question-bridge.js";
import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js";
import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server";
import {
codexCliVersionAtLeast,
minimumCodexCliVersionForModel,
normalizeCodexModel,
} from "@paperclipai/adapter-codex-local";
import { nativeCompletionFeedback } from "./native-completion-feedback.js";
import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js";
import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js";
import { computerProviderPackCachePath, prepareComputerProviderPackCache, pinnedComputerProviderPack, computerAcpxLaunchProfileDigest, readPinnedComputerProviderMetadata, assertPinnedComputerProviderState } from "./remote-provider-pack-cache.js";
import { selectRemotePiCompanion } from "./remote-pi-companion.js";
import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js";
import { remoteLeaseCleanupScope } from "../remote-execution-termination.js";
import { resolveConnectorAssignments, isConnectorSkill } from "../connector-runtime.js";
import {
boundedExecutionCleanup,
EXECUTION_CONTROL_DEADLINE_MS,
} from "../execution-control-deadline.js";
import { createHash, randomUUID } from "node:crypto";
import { execFile, execFileSync } from "node:child_process";
import {
chmodSync,
createReadStream,
copyFileSync,
closeSync,
constants,
existsSync,
fstatSync,
lstatSync,
mkdirSync,
mkdtempSync,
openSync,
opendirSync,
readFileSync,
readdirSync,
readlinkSync,
readSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { DatabaseSync } from "node:sqlite";
import { basename, dirname, join, posix, resolve } from "node:path";
import type {
AdapterExecutionResult,
AdapterRuntimeEvent,
} from "../../adapters/index.js";
import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared";
import type {
HarnessRuntimeRequestResolution,
NativeExecutionInput,
NativeRuntimeContextSnapshot,
NativeSession,
NativeSessionBackend,
PaperclipQuestionSet,
PersistedNativeSession,
PrpEvent,
PrpStructuredRunResult,
} from "../../vendor/paperclip-runner/index.js";
import {
NativeProviderTerminalFailure,
NativeSessionCleanupQuarantinedError,
NativeSessionProtocolIntegrityError,
completeRetainedNativeSessionCleanup,
completeTerminatedLocalNativeSessionCleanup,
acpxRuntimeSessionDirectoryName,
createNativeSessionBackend,
createRunnerdCodexTransport,
executeNativeSession,
applyNativeSessionGoalControl,
inspectWarmRunTransition,
parseNativeExecutionInput,
parsePaperclipQuestionSet,
resolveSourceCodexHome,
readRunnerdArtifactBinding,
retainedRunnerdMaintenanceIsIdle,
settleRetainedRunnerdSession,
validatePrpEvent,
validatePrpStructuredRunResult,
type RunnerProcessHandle,
type RunnerProcessLaunchSpec,
type NativeSessionGoalControl,
} from "../../vendor/paperclip-runner/index.js";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { createNativeSshCommandRunner } from "./native-ssh-command-runner.js";
import { discoverRemoteExecutable, parseRemoteExecutableCandidate } from "./codex-model-fallback.js";
export { parseRemoteExecutableCandidate } from "./codex-model-fallback.js";
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
import {
resolvePaperclipRunnerTransport,
type PaperclipRunnerTransport,
} from "@paperclipai/adapter-utils/runner-connectivity";
import type { Db } from "@paperclipai/db";
import {
and,
desc,
eq,
gt,
inArray,
isNull,
like,
notInArray,
or,
sql,
} from "drizzle-orm";
import {
agentWakeupRequests,
documentRevisions,
environmentLeases,
heartbeatRunEvents,
heartbeatRuns,
issueDocuments,
issueThreadInteractions,
issues,
nativeRunFinalizations,
nativeRunResults,
} from "@paperclipai/db";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { nativeSha256 } from "./canonical.js";
import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js";
import { createAssignedMcpTools, getAssignedMcpGateway } from "./assigned-mcp-tools.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
import { NativeChatAttachmentReadScope } from "./chat-attachment-read.js";
import {
assertCurrentWakeCommentsRead,
resolveCurrentWakeCommentsBinding,
} from "./current-wake-comments.js";
import {
renderNativeRunnerStagedAttachmentPrompt,
stageNativeRunnerWakeAttachments,
} from "./native-runner-file-handoff.js";
import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js";
import { registerRunnerPrpAuthority } from "../../realtime/runner-prp-ws.js";
import { connectRunnerPrpIngress } from "../../realtime/runner-prp-outbound.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { reportRunFailure } from "../run-failure-report.js";
import { persistActivity, publishActivity } from "../activity-log.js";
import { commitNativeStatusDecision } from "./status-decision-committer.js";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { documentService } from "../documents.js";
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
import { issueService } from "../issues.js";
import {
NATIVE_STATUS_ARBITER_POLICY_VERSION,
type NativeAuthoritativeIssueStatus,
type NativeStatusDecision,
} from "./status-arbiter.js";
import { conflict, HttpError } from "../../errors.js";
import { redactSensitiveText } from "../../redaction.js";
import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js";
import {
createNativeRunTrace,
isNativeRunRootHistoricalSpan,
nativeRunPreparationStarts,
type NativeRunHistoricalSpan,
type NativeRunSpanScope,
type NativeRunTrace,
} from "./native-run-trace.js";
import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js";
import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js";
import { registerLiveRunnerGoalController } from "../runner-goal-control-broker.js";
import { applyRunnerGoalPrpEvent } from "../runner-goals.js";
import { readProcessStartedAt } from "../hot-restart.js";
import {
NativeRunnerOwnershipUnverifiedError,
isNativeRunnerOwnershipHeld,
NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT,
} from "./native-runner-ownership.js";
import {
currentNativeControllerIdentity,
nextNativeProviderAttempt,
type NativeControllerIdentity,
type NativeRestartRecoveryClaim,
} from "./native-restart-recovery.js";
type ActiveNativeSession = {
session: NativeSession;
cancelRequested: boolean;
restartDetach?: Promise<void>;
};
class NativeResultPendingFinalizationError extends Error {
constructor() {
super("native_result_pending_finalization");
this.name = "NativeResultPendingFinalizationError";
}
}
export class NativeCancellationPendingRecoveryError extends Error {
constructor() {
super("native_cancellation_pending_recovery");
this.name = "NativeCancellationPendingRecoveryError";
}
}
export class NativeControllerDetachedForRestartError extends Error {
constructor() {
super("native_controller_detached_for_restart");
this.name = "NativeControllerDetachedForRestartError";
}
}
const activeNativeSessions = new Map<string, ActiveNativeSession>();
// Shutdown can race provider startup before onSession publishes its handle.
// Retain the request for the remainder of this controller's lifetime so that
// the late publication detaches before it can dispatch another turn.
const nativeRunsDetachingForRestart = new Set<string>();
// A restart must not cut authority rotation between archiving the previous
// runner state and authenticating its replacement. Keep the startup owner
// alive until onSession can detach it, or until startup itself has settled.
type NativeSessionStartup = {
promise: Promise<ActiveNativeSession | null>;
resolve: (session: ActiveNativeSession | null) => void;
stopRequested?: boolean;
cancellationSettled?: Promise<void>;
governedWait?: boolean;
settled: Promise<void>;
settle: () => void;
};
const nativeSessionStartups = new Map<string, NativeSessionStartup>();
function detachActiveNativeSessionForRestart(active: ActiveNativeSession) {
active.restartDetach ??= Promise.resolve().then(() => active.session.detachControllerForRestart!());
return active.restartDetach;
}
async function waitForNativeSessionStartup(
runId: string,
timeoutError: () => Error = () => new Error("native_restart_startup_not_ready"),
) {
const startup = nativeSessionStartups.get(runId);
if (!startup) return null;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
startup.promise,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(timeoutError()), 30_000);
}),
]);
} finally {
if (timer !== undefined) clearTimeout(timer);
}
}
export async function detachNativeSessionsForRestart(
runIds: readonly string[],
): Promise<{
detachedRunIds: string[];
inactiveRunIds: string[];
unsupportedRunIds: string[];
}> {
const detachedRunIds: string[] = [];
const inactiveRunIds: string[] = [];
const unsupportedRunIds: string[] = [];
const settlementDeadline = Date.now() + 20_000;
for (const runId of new Set(runIds)) {
// A governed stop already revoked new work and is collecting terminal
// accounting. Give that exact owner a bounded chance to persist it before
// fencing callbacks and relinquishing the runner. Timeout is not success.
const settling = nativeSessionStartups.get(runId);
if (settling?.governedWait) {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([settling.settled, new Promise<void>(resolve => {
timer = setTimeout(resolve, Math.max(0, settlementDeadline - Date.now()));
})]);
} finally { clearTimeout(timer); }
}
nativeRunsDetachingForRestart.add(runId);
const active = activeNativeSessions.get(runId) ?? await waitForNativeSessionStartup(runId);
if (!active) {
inactiveRunIds.push(runId);
continue;
}
if (active.session.detachControllerForRestart === undefined) {
unsupportedRunIds.push(runId);
continue;
}
await detachActiveNativeSessionForRestart(active);
detachedRunIds.push(runId);
}
return { detachedRunIds, inactiveRunIds, unsupportedRunIds };
}
const MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES = 64 * 1024 * 1024;
const MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES = 64 * 1024 * 1024;
const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000;
// Identity is read from the complete control-plane state, including its PRP
// event window. Match the transport's bounded reader so verbose valid turns do
// not lose continuation authority merely because their history exceeds 64 MiB.
const NATIVE_CONTROL_PLANE_STATE_MAX_BYTES = 256 * 1024 * 1024;
const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024;
const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024;
const CODEX_HOME_NON_PERSISTENT_ENTRIES = [
"tmp",
".tmp",
"auth.json",
"config.toml",
] as const;
const RUNNERD_CONTROL_PLANE_STATE_SCHEMA =
"paperclip.runner.durable.control-plane-state.v1";
const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1";
const CODEX_PROVIDER_STATE_SCHEMA = "paperclip.runner.codex-provider-state.v1";
const ACPX_PROVIDER_STATE_SCHEMA = "paperclip.runner.acpx-provider-state.v3";
const MANAGED_PROVIDER_STATE_SCHEMA =
"paperclip.runner.managed-provider-state.v1";
const RUNNERD_STATE_LIFECYCLES = new Set([
"connecting",
"ready",
"backpressure",
"recoverable_failure",
"unrecoverable",
"suspended",
"stopped",
"revoked",
]);
const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([
"succeeded",
"interrupted",
"failed",
"cancelled",
"timed_out",
]);
const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000;
const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000;
const NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS = 2_000;
const NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX = 256;
type NativeRuntimeRequestResolution = {
runId: string;
fingerprint: string;
commandId: string;
pending: Promise<void>;
completedAt: number | null;
};
const nativeRuntimeRequestResolutions = new Map<
string,
NativeRuntimeRequestResolution
>();
async function verifiedRecoveryProcessIsAlive(input: {
pid: number;
startedAt: string;
}): Promise<boolean> {
try {
process.kill(input.pid, 0);
} catch (error) {
if ((error as NodeJS.ErrnoException | undefined)?.code !== "EPERM")
return false;
}
try {
const observed = await readProcessStartedAt(input.pid);
return (
observed !== null &&
new Date(observed).getTime() === new Date(input.startedAt).getTime()
);
} catch {
return false;
}
}
async function signalVerifiedRecoveryProcess(
input: { pid: number; startedAt: string },
signal: NodeJS.Signals,
): Promise<boolean> {
if (!(await verifiedRecoveryProcessIsAlive(input))) return false;
try {
return process.kill(input.pid, signal);
} catch {
return false;
}
}
function pruneNativeRuntimeRequestResolutionCache(): void {
const completed = [...nativeRuntimeRequestResolutions.entries()]
.filter(([, resolution]) => resolution.completedAt !== null)
.sort(
([, left], [, right]) =>
(left.completedAt ?? 0) - (right.completedAt ?? 0),
);
for (
let index = 0;
index < completed.length - NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX;
index += 1
) {
nativeRuntimeRequestResolutions.delete(completed[index]![0]);
}
}
function clearNativeRuntimeRequestResolutions(runId: string): void {
for (const [key, resolution] of nativeRuntimeRequestResolutions) {
if (resolution.runId === runId) {
nativeRuntimeRequestResolutions.delete(key);
}
}
}
export type NativeInstructionWorkingCopy = {
/** Present only for a complete registered agent_files materialization. */
runId?: string;
preparationKey?: string;
root?: string;
checkpointWarm?: () => Promise<boolean>;
hasChanges: () => Promise<boolean>;
collectStopped: () => Promise<void>;
retirementFailed?: () => Promise<void>;
};
type WarmNativeSession = {
computerTarget?: AdapterComputerExecutionTarget;
runId: string;
instructionWorkingCopy?: NativeInstructionWorkingCopy;
// Keep the admitted physical root independent of the per-run collection
// capability, which instruction preparation adopts before final admission.
instructionRoot?: string;
instructionPreparationRunId?: string;
agentId: string;
instructionCopy?: { runId: string; root?: string; targetIdentity: string; collectStopped: () => Promise<void> };
preparingRunId?: string;
managedAiCredentialIdentity?: string;
credentialRunId?: string;
githubAccess?: NativeGitHubAccess;
githubAuthenticationMode?: string;
networkAccess: boolean;
session: NativeSession;
ownerToken: symbol;
configDigest: string;
configuredEnvironmentDigest: string;
ownerScope: string;
companyId: string;
environmentId: string | null;
busy: boolean;
closeOnReleaseReason?: string;
idleTimer: ReturnType<typeof setTimeout> | null;
lastActivityAt: string;
};
/** A scoped preparation failure is retryable only after exact retirement fences
* queued commands. A process claim or invoked launcher keeps normal recovery. */
export async function retireUnlaunchedComputerAttempt(
target: AdapterExecutionTarget | null | undefined,
launchAttempted: boolean,
): Promise<void> {
if (target?.kind !== "remote" || target.transport !== "computer" || launchAttempted) return;
const process = await target.inspectProcess();
if (process.running || process.claim !== null) return;
if (!(await target.retire())) throw new NativeSessionCleanupQuarantinedError();
}
class ComputerOwnerSupersededError extends Error {
constructor() { super("computer_owner_retirement_superseded"); }
}
async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) {
let closeFailure: unknown;
let closeCompleted = false;
const close = async () => {
try { await entry.session.close({ reason }); closeCompleted = true; }
catch (error) { closeFailure = error; throw error; }
};
// Reserve the exact generation before protocol close. A winning admission
// must never receive a stale controller's suspend/kill command. The module
// always hard-retires descendants after the bounded graceful window.
if (entry.computerTarget && !(await entry.computerTarget.retire({ beforeStop: close }))) {
throw new ComputerOwnerSupersededError();
}
const stopping = entry.githubAccess?.stop();
try {
try {
if (!entry.computerTarget) await close();
if (closeFailure || !closeCompleted) {
await entry.session.detachControllerForRestart?.();
throw closeFailure ?? new NativeSessionCleanupQuarantinedError();
}
} catch (error) {
try { await entry.instructionWorkingCopy?.retirementFailed?.(); }
catch (receiptError) { throw new AggregateError([error, receiptError], "Warm instruction retirement and receipt both failed"); }
throw error;
}
await entry.instructionWorkingCopy?.collectStopped();
if (entry.instructionCopy?.runId !== preserveInstructionsForRunId) await entry.instructionCopy?.collectStopped();
} finally { await stopping; }
}
const warmNativeSessions = new Map<string, WarmNativeSession>();
// Closing a remote owner saves its checkpoint asynchronously. A new turn must
// not inspect or quarantine that owner's state until the save has finished.
const closingWarmNativeSessions = new Map<string, Promise<void>>();
/** Reserve the existing physical copy before heartbeat composes AGENT_HOME and
* task guidance. The final execution/configuration fence still owns admission. */
export async function claimWarmNativeInstructionCopy(input: {
priorExecution: NativeExecutionInput;
companyId: string;
agentId: string;
executionWorkspaceId: string;
workspace: NativeExecutionInput["workspace"];
environmentId: string | null;
runId: string;
preparationKey: string;
forceRetirement?: boolean;
adopt: (previousRunId: string) => Promise<NativeInstructionWorkingCopy | null>;
}): Promise<(() => Promise<void>) | null> {
const prior = input.priorExecution;
if (prior.binding.companyId !== input.companyId || prior.binding.agentId !== input.agentId
|| JSON.stringify(nativeSessionWorkspaceScope(prior)) !== JSON.stringify(nativeSessionWorkspaceScope({
binding: { ...prior.binding, runId: input.runId, executionWorkspaceId: input.executionWorkspaceId }, workspace: input.workspace,
})) || prior.workspace.cwd !== input.workspace.cwd) return null;
const scope = nativeSessionScopeKey(prior);
const entry = warmNativeSessions.get(scope);
if (!entry || !entry.instructionWorkingCopy?.runId || entry.environmentId !== input.environmentId) return null;
if (entry.busy) throw new Error("native_session_supervisor_busy");
entry.busy = true;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
const retire = async (reason: string) => {
if (warmNativeSessions.get(scope) !== entry) return;
entry.closeOnReleaseReason = reason;
try {
await closeWarmNativeSession(entry, reason);
if (warmNativeSessions.get(scope) === entry) warmNativeSessions.delete(scope);
} finally { entry.busy = false; }
};
try {
const preparationChanged = input.forceRetirement === true || entry.instructionWorkingCopy.preparationKey !== input.preparationKey;
// The prior capability may close over an expired remote lease. Rebind
// collection authority first; adoption itself grants no reuse permission.
const adopted = await input.adopt(entry.instructionWorkingCopy.runId);
if (!adopted) {
await retire("warm instruction materialization cannot be handed off");
throw new Error("native_instruction_materialization_handoff_unavailable");
}
if (adopted.runId !== input.runId || adopted.preparationKey !== input.preparationKey) throw new Error("native_instruction_materialization_handoff_mismatch");
entry.instructionWorkingCopy = adopted;
entry.instructionPreparationRunId = input.runId;
if (entry.closeOnReleaseReason !== undefined || preparationChanged || await adopted.hasChanges()) {
await retire("warm instruction materialization changed before preparation");
return null;
}
return async () => {
if (entry.instructionPreparationRunId === input.runId) await retire("warm instruction preparation abandoned");
};
} catch (error) {
if (entry.closeOnReleaseReason === undefined) await retire("warm instruction preparation failed");
throw error;
}
}
function instructionTargetIdentity(target?: AdapterExecutionTarget | null): string {
return JSON.stringify(target?.kind === "remote" ? {
environmentId: target.environmentId, cwd: target.remoteCwd,
providerLeaseId: target.transport === "sandbox" ? target.sandboxLeaseAcquisition?.providerLeaseId : target.transport === "computer" ? [target.resourceAuthority.computerId, target.resourceAuthority.ownerId] : target.spec,
} : { kind: "local", environmentId: target?.environmentId });
}
/** Reserve the actual live owner before handing its directory to another run.
* A persisted conversation alone is not proof that a working tree is reusable. */
export async function reserveWarmNativeInstructionDirectory(input: {
companyId: string; agentId: string; previousRunId: string; runId: string; target?: AdapterExecutionTarget | null;
canReuse: () => Promise<boolean>;
}): Promise<{ reuseRunId: string; adopt: (root: string, collectStopped: () => Promise<void>) => void; release: () => Promise<void> } | null> {
for (const [id, entry] of warmNativeSessions) {
if (entry.companyId !== input.companyId || entry.agentId !== input.agentId || entry.instructionCopy?.runId !== input.previousRunId) continue;
if (entry.busy || entry.preparingRunId) throw new Error("native_session_supervisor_busy");
if (input.target?.kind === "remote" && input.target.transport === "computer" &&
entry.computerTarget?.resourceAuthority.computerId === input.target.resourceAuthority.computerId &&
entry.computerTarget.resourceAuthority.ownerId === input.target.resourceAuthority.ownerId &&
entry.computerTarget.resourceAuthority.generation < input.target.resourceAuthority.generation) {
entry.computerTarget = input.target;
}
if (entry.idleTimer) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
entry.preparingRunId = input.runId;
const retire = async (reason: string) => {
entry.closeOnReleaseReason = reason;
try {
await closeWarmNativeSession(entry, reason);
if (warmNativeSessions.get(id) === entry) warmNativeSessions.delete(id);
} finally {
// Failed containment remains registered and cannot be adopted. The next
// admission retries retirement instead of launching over a live owner.
entry.preparingRunId = undefined;
}
};
let reusable = false;
const targetProven = input.target?.kind !== "remote" || input.target.transport !== "sandbox" || Boolean(input.target.sandboxLeaseAcquisition?.providerLeaseId);
try { reusable = targetProven && !entry.closeOnReleaseReason && entry.instructionCopy.targetIdentity === instructionTargetIdentity(input.target) && await input.canReuse(); }
finally {
if (!reusable) {
await retire("managed agent files changed or environment replaced");
}
}
if (!reusable) return null;
return { reuseRunId: input.previousRunId, adopt: (root, collectStopped) => {
entry.instructionCopy = { runId: input.runId, root, targetIdentity: instructionTargetIdentity(input.target), collectStopped };
}, release: async () => {
if (warmNativeSessions.get(id) === entry && entry.preparingRunId === input.runId) {
await retire("managed warm turn preparation ended without attachment");
}
} };
}
return null;
}
/**
* Close idle native sessions before an operator destroys their remote
* environment. A warm runner owns a long-lived sandbox command stream, so the
* provider cannot safely delete that sandbox until the session has closed the
* stream. Busy sessions are reported instead of interrupted; the environment
* delete guard can then fail closed while their heartbeat run is still live.
*/
export async function closeWarmNativeSessionsForEnvironment(input: {
environmentId: string;
reason: string;
}): Promise<{ closed: number; busy: number; failed: number }> {
return closeIdleWarmNativeSessions(input);
}
/** Cancel one attempt without closing other agents on its computer. */
export async function closeWarmNativeSessionsForRun(input: { runId: string; reason: string }) {
return closeIdleWarmNativeSessions(input);
}
/** Suspend idle owners and persist their remote backup before a controller
* exits. Active turns keep their separate authenticated restart handoff. */
export async function closeIdleWarmNativeSessionsForRestart(): Promise<{
closed: number; busy: number; failed: number;
}> {
return closeIdleWarmNativeSessions({
reason: "controller restart",
closeBusyOnRelease: true,
});
}
async function closeIdleWarmNativeSessions(input: {
environmentId?: string;
runId?: string;
reason: string;
closeBusyOnRelease?: boolean;
}): Promise<{ closed: number; busy: number; failed: number }> {
let closed = 0;
let busy = 0;
let failed = 0;
for (const [sessionId, entry] of [...warmNativeSessions]) {
if (input.runId !== undefined && entry.runId !== input.runId) continue;
if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) {
continue;
}
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) {
// A busy turn can complete while another idle session is checkpointing.
// Fence that entry now so its eventual release cannot leave a new idle
// owner behind after the shutdown sweep has already passed it.
if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason;
busy += 1;
continue;
}
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
// Keep a fenced owner until both retirement and stopped collection succeed.
// Failure must not make a live directory eligible for a replacement owner.
entry.busy = true;
entry.closeOnReleaseReason = input.reason;
const closing = Promise.resolve().then(() =>
closeWarmNativeSession(entry, input.reason),
);
closingWarmNativeSessions.set(sessionId, closing);
try {
await closing;
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
closed += 1;
} catch {
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
failed += 1;
} finally {
entry.busy = false;
if (closingWarmNativeSessions.get(sessionId) === closing) {
closingWarmNativeSessions.delete(sessionId);
}
}
}
return { closed, busy, failed };
}
function readBoundedNativeFile(
path: string,
maxBytes: number,
errorCode: string,
): Buffer {
const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const before = fstatSync(descriptor);
if (!before.isFile() || before.size > maxBytes) throw new Error(errorCode);
const output = Buffer.allocUnsafe(before.size);
let offset = 0;
while (offset < output.length) {
const bytesRead = readSync(
descriptor,
output,
offset,
output.length - offset,
offset,
);
if (bytesRead === 0) break;
offset += bytesRead;
}
const after = fstatSync(descriptor);
if (
offset !== before.size ||
after.size !== before.size ||
after.mtimeMs !== before.mtimeMs ||
after.ino !== before.ino
) {
throw new Error("native_state_file_changed");
}
return output;
} finally {
closeSync(descriptor);
}
}
const NATIVE_PROVIDER_HOST_ENV_KEYS = [
"PATH",
"HOME",
"USER",
"LOGNAME",
"SHELL",
"TMPDIR",
"TEMP",
"TMP",
"CODEX_HOME",
"XDG_CONFIG_HOME",
"XDG_CACHE_HOME",
"XDG_DATA_HOME",
"SystemRoot",
"PATHEXT",
] as const;
async function measureNativeRunnerSpan<T>(
trace: NativeRunTrace | undefined,
name: string,
fn: () => Promise<T>,
options:
| string
| {
parentName?: string;
attributes?: Record<string, string | number | boolean>;
} = {},
): Promise<T> {
return trace
? trace.measure(
name,
fn,
typeof options === "string" ? { parentName: options } : options,
)
: fn();
}
/**
* Provider bootstrap needs a small amount of host process context even when
* the agent has no configured env. In particular, an empty environment makes
* a bare `codex` command unresolvable. Agent-configured values remain
* authoritative and may intentionally override the host defaults, while the
* server-selected workspace remains an immutable containment boundary.
*/
export function buildNativeProviderEnvironment(
configured: NodeJS.ProcessEnv,
host: NodeJS.ProcessEnv = process.env,
assignedWorkspaceCwd?: string,
): NodeJS.ProcessEnv {
const inherited = Object.fromEntries(
NATIVE_PROVIDER_HOST_ENV_KEYS.flatMap((key) => {
const value = host[key];
return typeof value === "string" && value.length > 0
? [[key, value]]
: [];
}),
);
const environment = { ...inherited, ...configured };
if (assignedWorkspaceCwd?.trim()) {
environment.PAPERCLIP_WORKSPACE_CWD = assignedWorkspaceCwd;
}
return environment;
}
/** Missing candidate bindings must not turn the controller's ambient credentials into explicit input. */
export function resolveNativeProviderEnvironment(
provider: NativeExecutionInput["provider"],
configured: NodeJS.ProcessEnv | undefined,
host: NodeJS.ProcessEnv = process.env,
): NodeJS.ProcessEnv {
if (configured !== undefined) return configured;
if (provider.kind === "acpx" && ["pi", "cursor", "copilot"].includes(provider.agent)) {
return buildNativeProviderEnvironment({}, host);
}
return host;
}
type PlanSynchronization = {
eventId: string;
planId: string;
providerRevision: number;
status:
| "synchronized"
| "already_synchronized"
| "conflict"
| "invalid"
| "approval_failed";
baseRevisionId: string | null;
digest: string;
documentRevision: number | null;
currentRevisionId: string | null;
confirmationId: string | null;
};
type RuntimeQuestionFallback = {
kind: "ask_user_questions";
idempotencyKey: string;
sourceRunId: string;
title: string | null;
summary: string | null;
continuationPolicy: "wake_assignee";
payload: {
version: 1;
title?: string;
submitLabel?: string;
supersedeOnUserComment: false;
runtimeRequestId: string;
questionSet: PaperclipQuestionSet;
questions: Array<{
id: string;
prompt: string;
helpText?: string;
selectionMode: "single" | "multi";
required: boolean;
options: Array<{
id: string;
label: string;
description?: string;
freeText?: boolean;
}>;
}>;
};
};
/** Translate non-replayable live-input expirations into one durable interaction. */
export function runtimeQuestionFallbackFromEvent(
event: Pick<PrpEvent, "eventType" | "payload" | "runId">,
): RuntimeQuestionFallback | null {
if (event.eventType !== "runtime_request.expired") return null;
const payload = record(event.payload);
if (
!["durable_handoff", "provider_process_lost"].includes(
String(payload.reason),
) ||
payload.replayAllowed !== false
)
return null;
const request = record(payload.request);
if (
payload.requestKind !== "runtime" ||
payload.requestType !== "input" ||
request.schema !== "paperclip.runtime_request.v2" ||
request.requestKind !== "runtime" ||
request.type !== "input" ||
typeof request.requestId !== "string" ||
payload.requestId !== request.requestId ||
typeof request.turnId !== "string" ||
typeof request.itemId !== "string"
)
return null;
let questionSet: PaperclipQuestionSet;
try {
questionSet = parsePaperclipQuestionSet(request.input);
} catch {
return null;
}
const questions = questionSet.questions.map((question) => ({
id: question.id,
prompt: question.prompt,
...(question.helpText ? { helpText: question.helpText } : {}),
selectionMode:
question.answerMode === "multi_select"
? ("multi" as const)
: ("single" as const),
required: question.required,
options:
question.answerMode === "text"
? [
{
id: "__paperclip_text__",
label:
question.textValidation?.inputType === "integer"
? "Enter an integer"
: question.textValidation?.inputType === "number"
? "Enter a number"
: "Enter your answer",
freeText: true,
},
]
: (question.options ?? []).map((option) => ({
id: option.id,
label: option.label,
...(option.description ? { description: option.description } : {}),
})),
}));
return {
kind: "ask_user_questions",
idempotencyKey: `runtime-input-durable:v1:${event.runId}:${request.requestId}`,
sourceRunId: event.runId,
title: questionSet.title?.slice(0, 240) ?? null,
summary: questionSet.description?.slice(0, 1000) ?? null,
continuationPolicy: "wake_assignee",
payload: {
version: 1,
...(questionSet.title ? { title: questionSet.title.slice(0, 240) } : {}),
...(questionSet.submitLabel
? { submitLabel: questionSet.submitLabel.slice(0, 120) }
: {}),
supersedeOnUserComment: false,
runtimeRequestId: request.requestId,
questionSet,
questions,
},
};
}
/**
* Materialize the durable replacement for a non-replayable runtime question.
*
* The interaction service enforces the fallback's stable idempotency key, so
* this is safe both immediately after the event commit and while recovering an
* exact duplicate whose original post-commit callback did not finish.
*/
export async function materializeRuntimeQuestionFallback(input: {
db: Db;
binding: {
companyId: string;
issueId: string;
runId: string;
agentId: string;
};
event: Pick<PrpEvent, "eventType" | "payload" | "runId">;
}): Promise<{
fallback: RuntimeQuestionFallback;
interaction: { id: string };
} | null> {
const fallback = runtimeQuestionFallbackFromEvent(input.event);
if (!fallback) return null;
const interaction = await issueThreadInteractionService(input.db).create(
{
id: input.binding.issueId,
companyId: input.binding.companyId,
},
fallback as never,
{
agentId: input.binding.agentId,
runId: input.binding.runId,
systemId: "native-runtime-question-handoff",
},
);
return { fallback, interaction };
}
export function runtimeInputLifecycleMetric(
event: Pick<PrpEvent, "eventType" | "payload">,
): {
outcome:
| "normalized"
| "rejected"
| "resolved"
| "expired"
| "durable_handoff"
| "provider_loss_handoff"
| "cancelled";
adapter: string;
requestId: string | null;
} | null {
const payload = record(event.payload);
const request = record(payload.request);
if (
event.eventType === "runtime_request.created" &&
request.type === "input"
) {
const origin = record(request.origin);
return {
outcome: "normalized",
adapter: typeof origin.adapter === "string" ? origin.adapter : "unknown",
requestId:
typeof request.requestId === "string" ? request.requestId : null,
};
}
if (
event.eventType === "harness.diagnostic" &&
payload.code === "runtime_input_rejected"
) {
return {
outcome: "rejected",
adapter:
typeof payload.adapter === "string" ? payload.adapter : "unknown",
requestId: null,
};
}
const terminalOutcome =
event.eventType === "runtime_request.resolved"
? "resolved"
: event.eventType === "runtime_request.expired" &&
payload.reason === "durable_handoff"
? "durable_handoff"
: event.eventType === "runtime_request.expired" &&
payload.reason === "provider_process_lost"
? "provider_loss_handoff"
: event.eventType === "runtime_request.expired"
? "expired"
: event.eventType === "runtime_request.cancelled"
? "cancelled"
: null;
const requestType = payload.requestType ?? request.type;
if (!terminalOutcome || requestType !== "input") return null;
const origin = record(request.origin);
return {
outcome: terminalOutcome,
adapter:
typeof payload.adapter === "string"
? payload.adapter
: typeof origin.adapter === "string"
? origin.adapter
: "unknown",
requestId:
typeof payload.requestId === "string"
? payload.requestId
: typeof request.requestId === "string"
? request.requestId
: null,
};
}
export function providerPlanMarkdown(payload: Record<string, unknown>): string {
const completedMarkdown =
typeof payload.markdown === "string" ? payload.markdown.trim() : "";
if (completedMarkdown) return completedMarkdown.slice(0, 256_000);
const explanation =
typeof payload.explanation === "string" ? payload.explanation.trim() : "";
const steps = Array.isArray(payload.steps) ? payload.steps : [];
const lines = steps.slice(0, 256).flatMap((value) => {
if (!value || typeof value !== "object" || Array.isArray(value)) return [];
const step = value as Record<string, unknown>;
const body =
typeof step.body === "string" ? step.body.trim().slice(0, 4_000) : "";
if (!body) return [];
const status = step.status === "completed" ? "x" : " ";
const suffix =
step.status === "blocked"
? " _(blocked)_"
: step.status === "in_progress"
? " _(in progress)_"
: "";
return [`- [${status}] ${body}${suffix}`];
});
return [explanation, lines.join("\n")]
.filter(Boolean)
.join("\n\n")
.slice(0, 256_000);
}
export function semanticProviderPlanMarkdown(
result: Record<string, unknown>,
): string | null {
const artifacts = Array.isArray(result.artifacts) ? result.artifacts : [];
for (const value of artifacts) {
const artifact = record(value);
if (
artifact.kind !== "native_provider_plan" ||
typeof artifact.ref !== "string"
)
continue;
const match = artifact.ref.match(
/<proposed_plan>\s*([\s\S]*?)\s*<\/proposed_plan>/i,
);
const completedMarkdown = match?.[1]?.trim();
if (completedMarkdown) return completedMarkdown.slice(0, 256_000);
const embedded = artifact.ref.match(
/^native-provider-plan:([^\n]+)\n([\s\S]+)$/i,
);
if (embedded) {
const title = embedded[1]!
.replace(/^(?:DOT-\d+-)?/i, "")
.replace(/-v\d+$/i, "")
.replace(/-/g, " ")
.trim();
const body = embedded[2]!.trim();
if (title && body) {
return [`# ${title.charAt(0).toUpperCase()}${title.slice(1)}`, "", body]
.join("\n")
.slice(0, 256_000);
}
}
if (/^\s*1\.\s+/.test(artifact.ref)) {
const numberedPlan = artifact.ref
.split(/\s+\|\s+(?=\d+\.\s+)/)
.join("\n")
.trim();
if (numberedPlan) return `# Plan\n\n${numberedPlan}`.slice(0, 256_000);
}
// Some qualified Codex builds use the artifact reference itself as a
// compact, human-readable plan. Accept only an explicitly numbered form;
// arbitrary opaque artifact references must never become plan documents.
const inlineNumbered = artifact.ref.trim();
if (/\(1\)\s+.+\(2\)\s+/s.test(inlineNumbered)) {
const body = inlineNumbered
.replace(/^DOT-\d+\s+plan:\s*/i, "")
.replace(/^\(1\)\s*/, "1. ")
.replace(/;\s*\((\d+)\)\s*/g, "\n$1. ")
.trim();
if (body) return `# Plan\n\n${body}`.slice(0, 256_000);
}
const compact =
artifact.ref.match(/^native-provider-plan:([^#]+)#(.+)$/i) ??
artifact.ref.match(/^native-plan:\/\/[^/]+\/([^#]+)#(.+)$/i);
if (!compact) continue;
const humanize = (slug: string) =>
slug
.replace(
/\b(GET|POST|PUT|PATCH|DELETE)-([a-z0-9][a-z0-9-]*)/gi,
(_whole, method: string, path: string) =>
`${method.toUpperCase()} /${path}`,
)
.replace(/-/g, " ")
.replace(/\bjson\b/gi, "JSON")
.replace(/\bapi\b/gi, "API")
.replace(/\s+/g, " ")
.trim();
const title = humanize(compact[1]!.replace(/-v\d+$/i, ""));
const steps = compact[2]!.split(";").flatMap((encoded) => {
const parsed = encoded.match(/^\d+-(.+)$/);
const sentence = humanize(parsed?.[1] ?? encoded);
return sentence
? [sentence.charAt(0).toUpperCase() + sentence.slice(1)]
: [];
});
if (!title || steps.length === 0) continue;
return [
`# ${title.charAt(0).toUpperCase()}${title.slice(1)}`,
"",
...steps.map((step, index) => `${index + 1}. ${step}`),
]
.join("\n")
.slice(0, 256_000);
}
const hasNativePlanArtifact = artifacts.some(
(value) => record(value).kind === "native_provider_plan",
);
const summary =
typeof result.summary === "string" ? result.summary.trim() : "";
const summaryPlan = hasNativePlanArtifact
? summary.match(
/(?:^|:\s*)(1\)\s+[\s\S]+;\s*2\)\s+[\s\S]+;\s*3\)\s+[\s\S]+)$/,
)
: null;
if (summaryPlan) {
const body = summaryPlan[1]!
.replace(/^1\)\s*/, "1. ")
.replace(/;\s*(\d+)\)\s*/g, "\n$1. ")
.trim();
if (body) return `# Plan\n\n${body}`.slice(0, 256_000);
}
return null;
}
/**
* Convert a server-owned pending interaction into the semantic wait that a
* provider omitted. This is not a fabricated final response: it records that
* the current turn intentionally yielded to a durable governance surface.
*/
export function nativeGovernedWaitResult(input: {
interaction: { id: string; title: string | null; summary: string | null };
completionContract: NativeExecutionInput["completionContract"]["contract"];
}): PrpStructuredRunResult {
const interactionRef = `interaction:${input.interaction.id}`;
const label =
input.interaction.title?.trim() ||
input.interaction.summary?.trim() ||
"the requested response";
return {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "yielded",
summary: `Waiting for ${label}.`,
completionClaim: {
contractRevision: input.completionContract.revision,
objectiveSatisfied: false,
criteria: input.completionContract.criteria.map((criterion) => ({
criterionId: criterion.id,
status: "unknown",
evidenceRefs: [interactionRef],
})),
remainingWork: [
{
description: "Resume after the durable interaction is resolved.",
blocksCompletion: true,
},
],
},
evidence: [{ ref: interactionRef }],
verification: [],
attentionRequests: [],
artifacts: [{ kind: "issue_thread_interaction", ref: interactionRef }],
continuation: {
kind: "response_wake",
summary:
"Resume from the resolved interaction response without repeating prior work.",
idempotencyKey: `interaction-response:${input.interaction.id}`,
},
};
}
/** A completed chat reply yields to the next message without claiming task completion. */
export function nativeConversationReplyResult(input: {
conversation: boolean;
terminalEvent: PrpEvent;
replyEvent: PrpEvent | null;
completionContract: NativeExecutionInput["completionContract"]["contract"];
}): PrpStructuredRunResult | null {
const reply = input.replyEvent;
const payload = record(reply?.payload);
const text = typeof payload.text === "string" ? payload.text.trim() : "";
if (!input.conversation || input.terminalEvent.eventType !== "turn.completed" ||
!reply || reply.eventType !== "item.completed" || payload.kind !== "agentMessage" ||
payload.channel !== "final" || !text || reply.runId !== input.terminalEvent.runId ||
reply.turnId !== input.terminalEvent.turnId ||
reply.normalizedSessionId !== input.terminalEvent.normalizedSessionId) return null;
const ref = `run-event:${reply.sourceEventId}`;
return {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "yielded",
summary: text.slice(0, 12_000),
completionClaim: {
contractRevision: input.completionContract.revision,
objectiveSatisfied: false,
criteria: input.completionContract.criteria.map((criterion) => ({
criterionId: criterion.id, status: "unknown", evidenceRefs: [ref],
})),
remainingWork: [],
},
evidence: [{ ref }],
verification: [],
attentionRequests: [],
artifacts: [],
continuation: {
kind: "response_wake",
summary: "Wait for the next user message in this conversation.",
idempotencyKey: `conversation-reply:${reply.sourceEventId}`,
},
};
}
/**
* Bridge an asynchronous durable-interaction lookup to the runner package's
* synchronous governed-wait boundary. Observations are single-use and bound
* to one exact source event so a delayed or replayed lookup cannot leak into a
* later provider event.
*/
export function createGovernedWaitEventObservation(
resolvePending: () => Promise<PrpStructuredRunResult | null>,
) {
const pendingTools = new Set<string>();
let providerLost = false;
let generation = 0;
let observation: {
sourceInstanceId: string;
sourceEventId: string;
sourceSeq: number;
result: PrpStructuredRunResult;
} | null = null;
return {
async observe(event: PrpEvent, eligible: boolean): Promise<void> {
const currentGeneration = ++generation;
observation = null;
const payload = record(event.payload);
providerLost ||= event.eventType === "runtime_request.expired" && payload.reason === "provider_process_lost";
const kind = payload.kind;
const tool = ["dynamicToolCall", "mcpToolCall", "commandExecution"].includes(String(kind));
if (event.itemId) {
if (tool && event.eventType === "item.started") pendingTools.add(event.itemId);
// Terminal error events can omit kind; the tracked ID owns cleanup.
if (event.eventType === "item.completed" || event.eventType === "item.failed") {
pendingTools.delete(event.itemId);
}
}
// Usage/model messages can arrive while the tool creating the card is
// still awaiting its response. Parking then interrupts that in-flight
// response and cannot produce a durable suspension checkpoint.
if (event.eventType === "item.completed" && (
pendingTools.size > 0 || (!tool && !(kind === "agentMessage" && payload.channel === "final"))
)) return;
// A replacement question preserves human input after a provider crash;
// it does not authorize a successful suspension of the failed execution.
if (providerLost || !eligible) return;
const result = await resolvePending();
if (generation !== currentGeneration || result === null) return;
// If the interaction is answered after this read, parking remains the
// fail-closed outcome: the durable answer owns the response-wake path.
// Continuing provider work on a possibly stale authorization does not.
observation = {
sourceInstanceId: event.sourceInstanceId,
sourceEventId: event.sourceEventId,
sourceSeq: event.sourceSeq,
result,
};
},
consume(event: PrpEvent): PrpStructuredRunResult | null {
generation += 1;
const current = observation;
observation = null;
if (
current === null ||
current.sourceInstanceId !== event.sourceInstanceId ||
current.sourceEventId !== event.sourceEventId ||
current.sourceSeq !== event.sourceSeq
) {
return null;
}
return current.result;
},
};
}
export function nativeToolsRefreshWaitResult(input: {
wakeId: string;
key: string;
completionContract: NativeExecutionInput["completionContract"]["contract"];
}): PrpStructuredRunResult {
const ref = `wakeup:${input.wakeId}`;
return {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "yielded",
summary: "Continuing with the newly installed connection tools.",
completionClaim: {
contractRevision: input.completionContract.revision,
objectiveSatisfied: false,
criteria: input.completionContract.criteria.map((criterion) => ({
criterionId: criterion.id,
status: "unknown",
evidenceRefs: [ref],
})),
remainingWork: [
{
description: "Continue in the queued session with updated tools.",
blocksCompletion: true,
},
],
},
evidence: [{ ref }],
verification: [],
attentionRequests: [],
artifacts: [],
continuation: {
kind: "same_agent",
summary: "Use the updated connection tools in a fresh session.",
idempotencyKey: input.key,
},
};
}
/**
* Partial item-verdict responses deliberately leave their original durable
* interaction pending. They are already authority-checked before entering the
* closed native envelope, so that exact interaction may park the continuation
* run without requiring the model to recreate a second request.
*/
export function continuingPendingInteractionIds(
execution: NativeExecutionInput,
): string[] {
return execution.interactionResponses
.filter(
(response) =>
response.kind === "request_item_verdicts" &&
response.response.status === "pending",
)
.map((response) => response.interactionId);
}
export async function synchronizeCompletedProviderPlan(input: {
db: Db;
execution: NativeExecutionInput;
event: {
sourceEventId: string;
turnId?: string;
eventType: string;
payload: Record<string, unknown>;
};
}): Promise<PlanSynchronization | null> {
if (
input.event.eventType !== "plan.updated" ||
input.event.payload.complete !== true
)
return null;
if (
!("executionMode" in input.execution) ||
input.execution.executionMode !== "plan"
)
return null;
const planningContext = input.execution.planningContext;
if (!planningContext) return null;
const planId =
typeof input.event.payload.planId === "string"
? input.event.payload.planId
: "";
const providerRevision = Number.isSafeInteger(input.event.payload.revision)
? Number(input.event.payload.revision)
: 0;
const body = providerPlanMarkdown(input.event.payload);
const digest = createHash("sha256").update(body).digest("hex");
if (!planId || providerRevision < 1 || !body) {
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status: "invalid",
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: null,
currentRevisionId: null,
confirmationId: null,
};
}
const provenance = `runner-plan-sync:v2 run=${input.execution.binding.runId} turn=${input.event.turnId ?? "unknown"} provider=${input.execution.provider.kind} plan=${planId} revision=${providerRevision} digest=${digest}`;
const existingRevision = await input.db
.select({
revisionNumber: documentRevisions.revisionNumber,
id: documentRevisions.id,
})
.from(documentRevisions)
.innerJoin(
issueDocuments,
eq(issueDocuments.documentId, documentRevisions.documentId),
)
.where(
and(
eq(issueDocuments.issueId, input.execution.binding.issueId),
eq(issueDocuments.key, "plan"),
eq(documentRevisions.changeSummary, provenance),
),
)
.orderBy(desc(documentRevisions.revisionNumber))
.limit(1)
.then((rows) => rows[0] ?? null);
const documents = documentService(input.db);
let revision = existingRevision;
let status: PlanSynchronization["status"] = existingRevision
? "already_synchronized"
: "synchronized";
if (!revision) {
const latest = await documents.getIssueDocumentByKey(
input.execution.binding.issueId,
"plan",
);
if (latest?.latestRevisionId && latest.body === body) {
const sameRunRevision = await input.db
.select({
id: documentRevisions.id,
revisionNumber: documentRevisions.revisionNumber,
createdByRunId: documentRevisions.createdByRunId,
})
.from(documentRevisions)
.where(eq(documentRevisions.id, latest.latestRevisionId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (sameRunRevision?.createdByRunId === input.execution.binding.runId) {
revision = sameRunRevision;
status = "already_synchronized";
}
}
}
try {
if (!revision) {
const write = await documents.upsertIssueDocument({
issueId: input.execution.binding.issueId,
key: "plan",
title: "Plan",
format: "markdown",
body,
baseRevisionId: planningContext.baseRevisionId,
changeSummary: provenance,
createdByAgentId: input.execution.binding.agentId,
createdByRunId: input.execution.binding.runId,
});
revision = {
revisionNumber: write.document.latestRevisionNumber,
id: write.document.latestRevisionId!,
};
}
} catch (error) {
if (!(error instanceof HttpError) || error.status !== 409) throw error;
const latest = await documents.getIssueDocumentByKey(
input.execution.binding.issueId,
"plan",
);
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status: "conflict",
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: latest?.latestRevisionNumber ?? null,
currentRevisionId: latest?.latestRevisionId ?? null,
confirmationId: null,
};
}
const current = await documents.getIssueDocumentByKey(
input.execution.binding.issueId,
"plan",
);
if (!current || !revision?.id || current.latestRevisionId !== revision.id) {
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status: "conflict",
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: current?.latestRevisionNumber ?? null,
currentRevisionId: current?.latestRevisionId ?? null,
confirmationId: null,
};
}
let confirmationId: string;
let confirmationPending = false;
try {
const confirmation = await issueThreadInteractionService(input.db).create(
{
id: input.execution.binding.issueId,
companyId: input.execution.binding.companyId,
},
{
kind: "request_confirmation",
idempotencyKey: `runner-plan-approval:v1:${input.execution.binding.runId}:${planId}:${providerRevision}:${digest}`,
sourceRunId: input.execution.binding.runId,
title: `Review plan revision ${revision.revisionNumber}`,
summary: "Review the synchronized Paperclip plan.",
continuationPolicy: "wake_assignee",
payload: {
version: 1,
prompt: `Approve plan revision ${revision.revisionNumber}?`,
detailsMarkdown:
"The completed provider plan has been synchronized to the canonical Plan document.",
acceptLabel: "Approve plan",
rejectLabel: "Request changes",
rejectRequiresReason: true,
supersedeOnUserComment: false,
target: {
type: "issue_document",
issueId: input.execution.binding.issueId,
documentId: current.id,
key: "plan",
revisionId: revision.id,
revisionNumber: revision.revisionNumber,
label: `Plan v${revision.revisionNumber}`,
},
},
} as never,
{
agentId: input.execution.binding.agentId,
runId: input.execution.binding.runId,
},
);
confirmationId = confirmation.id;
confirmationPending = confirmation.status === "pending";
} catch {
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status: "approval_failed",
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: revision.revisionNumber,
currentRevisionId: revision.id,
confirmationId: null,
};
}
if (confirmationPending) {
try {
const currentIssue = await input.db
.select({ status: issues.status })
.from(issues)
.where(eq(issues.id, input.execution.binding.issueId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (currentIssue && currentIssue.status !== "in_review") {
await issueService(input.db).update(input.execution.binding.issueId, {
status: "in_review",
actorAgentId: input.execution.binding.agentId,
});
}
} catch {
const settledIssue = await input.db
.select({ status: issues.status })
.from(issues)
.where(eq(issues.id, input.execution.binding.issueId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (settledIssue?.status !== "in_review") {
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status: "approval_failed",
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: revision.revisionNumber,
currentRevisionId: revision.id,
confirmationId,
};
}
}
}
return {
eventId: input.event.sourceEventId,
planId,
providerRevision,
status,
baseRevisionId: planningContext.baseRevisionId,
digest,
documentRevision: revision.revisionNumber,
currentRevisionId: revision.id,
confirmationId,
};
}
class SessionToolAuthorityEpoch {
readonly runId: string;
#authority: PaperclipRunnerToolAuthority;
#revoked = false;
constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) {
this.runId = runId;
this.#authority = authority;
}
revoke(): void {
this.#revoked = true;
this.#authority.close();
}
#assertCurrent(): void {
if (this.#revoked) {
throw new Error("native_tool_authority_epoch_revoked");
}
}
definitions() {
this.#assertCurrent();
return this.#authority.definitions();
}
async execute(call: Parameters<PaperclipRunnerToolAuthority["execute"]>[0]) {
this.#assertCurrent();
return this.toolTrace
? await this.toolTrace.execute(call, () => this.#authority.execute(call))
: await this.#authority.execute(call);
}
}
const sessionToolAuthorityEpochs = new Map<string, SessionToolAuthorityEpoch>();
const initializingSessionToolAuthorities = new Set<string>();
const executingRunnerdSessionScopes = new Map<string, string>();
const executingNativeOwnerScopes = new Map<string, symbol>();
function nativeSessionKey(execution: NativeExecutionInput): string {
return (
execution.session.normalizedSessionId ??
`session-${execution.binding.runId}`
);
}
/** Persistent computer artifacts belong to an agent session, independent of a shared repo cwd. */
export function remoteRunnerStorageRoots(
target: Extract<AdapterExecutionTarget, { kind: "remote" }>,
sessionKey: string,
) {
const home = target.transport === "computer" ? target.fileAuthority?.agentHome : target.remoteCwd;
if (!home || !posix.isAbsolute(home)) throw new Error("runner_remote_storage_home_missing");
const base = posix.join(home, ".paperclip-runtime", "paperclip-runner");
const sessionRoot = posix.join(base, "sessions", createHash("sha256").update(sessionKey).digest("hex"));
return { runtimeRoot: target.transport === "computer" ? sessionRoot : base, sessionRoot };
}
export function nativeSessionWorkspaceScope(execution: { binding: Pick<NativeExecutionInput["binding"], "runId" | "executionWorkspaceId">; workspace: NativeExecutionInput["workspace"] }) {
if ("access" in execution.workspace) return { kind: "none" as const };
// Projectless local runs use the heartbeat run id as a durable placeholder
// rather than fabricating an execution_workspaces row. Do not let that
// per-run placeholder break continuity for the same provider session; the
// immutable workspace descriptor is the stable identity in that case.
const transientWorkspace =
execution.binding.executionWorkspaceId === execution.binding.runId;
return transientWorkspace
? {
kind: "transient" as const,
cwd: execution.workspace.cwd,
repoUrl: execution.workspace.repoUrl,
repoRef: execution.workspace.repoRef,
branchName: execution.workspace.branchName,
}
: {
kind: "managed" as const,
executionWorkspaceId: execution.binding.executionWorkspaceId,
};
}
function nativeProviderSessionScope(execution: NativeExecutionInput) {
switch (execution.provider.kind) {
case "openai_dot": return { kind: "openai_dot", bindingId: execution.provider.binding.bindingId, generation: execution.provider.binding.bindingGeneration };
case "claude_managed":
return {
kind: execution.provider.kind,
profileId: execution.provider.managedProfile.profileId,
};
case "aws_agentcore":
return {
kind: execution.provider.kind,
profileId: execution.provider.agentCoreProfile.profileId,
};
case "acpx":
return {
kind: execution.provider.kind,
agent: execution.provider.agent,
profile: execution.provider.profile,
};
case "codex":
case "opencode":
// These local providers have no separate persisted profile id. Company,
// agent, and workspace bind their credential/session context; mutable
// model and permission settings remain in nativeSessionConfigDigest so
// they rotate an incompatible warm session within the same scope.
return { kind: execution.provider.kind };
}
}
function nativeSessionScopeKey(execution: NativeExecutionInput): string {
return canonicalJson({
schema: "paperclip.native-session-scope.v2",
companyId: execution.binding.companyId,
agentId: execution.binding.agentId,
workspace: nativeSessionWorkspaceScope(execution),
provider: {
driverKind: execution.session.driverKind,
identity: nativeProviderSessionScope(execution),
},
normalizedSessionId: nativeSessionKey(execution),
});
}
// A plan acceptance or explicit reset can rotate the provider/session identity
// while retaining the same task sandbox and its fixed ingress port.
function nativeSessionOwnerScope(
execution: NativeExecutionInput,
environmentId: string | null,
): string {
return canonicalJson({
companyId: execution.binding.companyId,
agentId: execution.binding.agentId,
issueId: execution.binding.issueId,
workspace: nativeSessionWorkspaceScope(execution),
environmentId,
});
}
async function retireSupersededWarmNativeSessions(
ownerScope: string,
nextSessionScope: string,
): Promise<void> {
for (const [scope, entry] of warmNativeSessions) {
if (scope === nextSessionScope || entry.ownerScope !== ownerScope) continue;
if (entry.busy || executingRunnerdSessionScopes.has(scope)) {
throw new Error("native_session_supervisor_busy");
}
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
entry.busy = true;
entry.closeOnReleaseReason = "warm native session identity changed";
try {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
if (warmNativeSessions.get(scope) === entry) warmNativeSessions.delete(scope);
} finally {
// A failed close retains the owner and prevents launch. A later attempt
// must retry retirement, never adopt this partially closed transport.
entry.busy = false;
}
}
}
function legacyCompanyNativeSessionScopeKey(
execution: NativeExecutionInput,
): string {
return JSON.stringify([
execution.binding.companyId,
nativeSessionKey(execution),
]);
}
function runnerdStateBase(): string {
return (
process.env.PAPERCLIP_RUNNER_STATE_DIR ??
resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"durable-sessions",
)
);
}
function scopedRunnerdStateRoot(execution: NativeExecutionInput): string {
return resolve(
runnerdStateBase(),
createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex"),
);
}
function scrubRunnerdQuarantineLaunchState(root: string): void {
const codexHome = resolve(root, "codex-home");
const codexHomeStats = lstatSync(codexHome, { throwIfNoEntry: false });
if (!codexHomeStats) return;
if (!codexHomeStats.isDirectory() || codexHomeStats.isSymbolicLink()) {
throw new Error("runner_state_directory_unsafe");
}
for (const name of CODEX_HOME_NON_PERSISTENT_ENTRIES) {
const entry = resolve(codexHome, name);
const stats = lstatSync(entry, { throwIfNoEntry: false });
if (!stats) continue;
// Remove symlinks themselves, never their targets. Real temporary
// directories are safe to remove recursively inside the verified home.
rmSync(entry, {
recursive: stats.isDirectory() && !stats.isSymbolicLink(),
force: true,
});
}
}
function quarantineRunnerdStateRoot(
root: string,
reason: "identity_indeterminate" | "identity_mismatch",
): string {
const stateBase = resolve(runnerdStateBase());
const resolvedRoot = resolve(root);
const stateKey = basename(resolvedRoot);
// Callers pass only roots derived by this module. Keep that boundary explicit
// so a future call site cannot turn quarantine into an arbitrary filesystem
// move, and never follow or move a symlink in place of the state directory.
if (
dirname(resolvedRoot) !== stateBase ||
!/^[a-f0-9]{64}$/.test(stateKey) ||
!isSafeNativeStateDirectory(resolvedRoot)
) {
throw new Error("runner_state_directory_unsafe");
}
// Quarantine retains durable session history for diagnosis and recovery, but
// launch credentials and transient files are re-materializable and must not
// survive after this state loses authority.
scrubRunnerdQuarantineLaunchState(resolvedRoot);
const quarantineRoot = resolve(stateBase, "quarantine");
mkdirSync(quarantineRoot, { recursive: true, mode: 0o700 });
if (!isSafeNativeStateDirectory(quarantineRoot)) {
throw new Error("runner_state_directory_unsafe");
}
const destination = resolve(
quarantineRoot,
`${stateKey}.${reason}.${Date.now()}.${process.pid}.${randomUUID()}`,
);
renameSync(resolvedRoot, destination);
return destination;
}
function legacyCompanyRunnerdStateRoot(
execution: NativeExecutionInput,
): string {
return resolve(
runnerdStateBase(),
createHash("sha256")
.update(legacyCompanyNativeSessionScopeKey(execution))
.digest("hex"),
);
}
function legacyRunnerdStateRoot(execution: NativeExecutionInput): string {
return resolve(
runnerdStateBase(),
createHash("sha256").update(nativeSessionKey(execution)).digest("hex"),
);
}
function migrateLegacyRunnerdStateRoot(input: {
legacy: string;
scoped: string;
execution: NativeExecutionInput;
verifiedPriorRunId?: string;
}): string | null {
if (!existsSync(input.legacy)) return null;
if (!isSafeNativeStateDirectory(input.legacy)) {
throw new Error("runner_state_directory_unsafe");
}
const legacyIdentity = readRunnerdDurableIdentity(input.legacy);
const exactRun = durableIdentityMatchesExecution(
legacyIdentity,
input.execution,
);
const verifiedSettledPriorRun = Boolean(
input.verifiedPriorRunId &&
legacyIdentity?.runId === input.verifiedPriorRunId &&
durableIdentityMatchesSession(legacyIdentity, input.execution),
);
if (!exactRun && !verifiedSettledPriorRun) {
// A legacy path does not encode the full session scope. A mismatch may be
// valid live state owned by another agent/workspace, so refusing the claim
// is safe but moving that ambiguous directory is not.
throw new Error("runner_state_identity_mismatch: legacy_owner_unverified");
}
if (
exactRun &&
legacyIdentity &&
["absent", "indeterminate"].includes(
runnerdAuthorityLifecycle(
input.legacy,
legacyIdentity as RunnerdDurableIdentity,
),
)
) {
quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate");
throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate");
}
try {
renameSync(input.legacy, input.scoped);
} catch (error) {
// Another in-process recovery may have won the atomic rename. The scoped
// directory is authoritative once it exists; otherwise preserve the
// original migration failure rather than silently starting empty state.
if (!isSafeNativeStateDirectory(input.scoped)) throw error;
const scopedIdentity = readRunnerdDurableIdentity(input.scoped);
const exactScopedRun = durableIdentityMatchesExecution(
scopedIdentity,
input.execution,
);
const sameVerifiedPriorRun = Boolean(
input.verifiedPriorRunId &&
scopedIdentity &&
scopedIdentity.runId === input.verifiedPriorRunId &&
scopedIdentity.runnerInstanceId === legacyIdentity?.runnerInstanceId &&
scopedIdentity.environmentLeaseId ===
legacyIdentity?.environmentLeaseId &&
durableIdentityMatchesSession(scopedIdentity, input.execution),
);
if (!exactScopedRun && !sameVerifiedPriorRun) {
throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed");
}
}
return input.scoped;
}
function runnerdAuthorityLifecycle(
root: string,
identity: RunnerdDurableIdentity,
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
const runnerRoot = resolve(root, "runner");
if (!existsSync(runnerRoot)) return "absent";
if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate";
const statePath = resolve(runnerRoot, "runner-state.json");
if (!existsSync(statePath)) {
try {
// Older remote transports created this controller-side placeholder even
// though runner state was owned by the sandbox. It carries no authority,
// so a verified failover backup may be consulted. Any non-empty direct
// directory remains indeterminate and therefore blocks fallback.
return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate";
} catch {
return "indeterminate";
}
}
try {
const state = record(
JSON.parse(
readBoundedNativeFile(
statePath,
NATIVE_RUNNER_STATE_MAX_BYTES,
"runner_state_too_large",
).toString("utf8"),
),
);
if (
state.schema !== RUNNERD_STATE_SCHEMA ||
typeof state.lifecycle !== "string" ||
!RUNNERD_STATE_LIFECYCLES.has(state.lifecycle)
) {
return "indeterminate";
}
if (
state.runnerInstanceId === identity.runnerInstanceId &&
state.environmentLeaseId === identity.environmentLeaseId &&
state.runId === identity.runId &&
state.normalizedSessionId === identity.normalizedSessionId
) {
if (state.lifecycle === "suspended") return "suspended";
return "not_suspended";
}
return "indeterminate";
} catch {
return "indeterminate";
}
}
function runnerdAuthorityLifecycleWithVerifiedBackup(input: {
root: string;
identity: RunnerdDurableIdentity;
execution: NativeExecutionInput;
allowVerifiedBackup: boolean;
}): "suspended" | "not_suspended" | "indeterminate" {
const direct = runnerdAuthorityLifecycle(input.root, input.identity);
if (direct !== "absent") return direct;
if (!input.allowVerifiedBackup) return "indeterminate";
const backup = verifyNativeHarnessBackup({
root: input.root,
execution: input.execution,
runnerInstanceId: input.identity.runnerInstanceId,
});
if (!backup) return "indeterminate";
const backupLifecycle = runnerdAuthorityLifecycle(
backup.root,
input.identity,
);
return backupLifecycle === "absent" ? "indeterminate" : backupLifecycle;
}
type PriorRunnerdStateVerification =
| "verified"
| "retained_warm_runner"
| "active"
| "authority_indeterminate"
| "scope_mismatch"
| "terminal_state_indeterminate"
| "unavailable";
const CLEANUP_CANONICAL_FILES = [
"control-plane/control-plane-state.json",
"runner/runner-state.json",
"runner/codex-provider-state.json",
] as const;
const CLEANUP_ACTIVATION_FILE = "cleanup-activation.json";
function cleanupStateSnapshot(root: string, providerFile = "codex-provider-state.json") {
if (
![root, resolve(root, "runner"), resolve(root, "control-plane")].every(
isSafeNativeStateDirectory,
)
) {
throw new Error("native_cleanup_maintenance_unproven");
}
const files = [...CLEANUP_CANONICAL_FILES.slice(0, 2), `runner/${providerFile}`];
const bytes = files.map((file) =>
readBoundedNativeFile(
resolve(root, file),
file === "control-plane/control-plane-state.json"
? NATIVE_CONTROL_PLANE_STATE_MAX_BYTES
: NATIVE_RUNNER_STATE_MAX_BYTES,
"native_cleanup_maintenance_unproven",
),
);
const [control, runner, provider] = bytes.map((value) =>
record(JSON.parse(value.toString("utf8"))),
);
const fileSha256 = bytes.map((value) =>
createHash("sha256").update(value).digest("hex"),
) as [string, string, string];
return {
control: control!,
runner: runner!,
provider: provider!,
fileSha256,
fingerprint: createHash("sha256")
.update(JSON.stringify(fileSha256))
.digest("hex"),
};
}
function cleanupProcessAbsent(pid: unknown): pid is number {
if (
process.platform === "win32" ||
!Number.isSafeInteger(pid) ||
Number(pid) <= 0
)
return false;
return [Number(pid), -Number(pid)].every((target) => {
try {
process.kill(target, 0);
return false;
} catch (error) {
return (error as NodeJS.ErrnoException).code === "ESRCH";
}
});
}
function cleanupCanonicalVacancy(root: string) {
const stat = lstatSync(root, { throwIfNoEntry: false });
if (!stat) return null;
if (!isSafeNativeStateDirectory(root) || readdirSync(root).length !== 0) {
throw new Error("native_cleanup_maintenance_unproven");
}
return {
device: stat.dev,
inode: stat.ino,
mode: stat.mode,
modifiedAt: stat.mtimeMs,
};
}
function cleanupArchiveRootIdentity(root: string) {
if (!isSafeNativeStateDirectory(root))
throw new Error("native_cleanup_maintenance_unproven");
const stat = lstatSync(root);
return { device: stat.dev, inode: stat.ino, mode: stat.mode };
}
/** Raw runner events and normalized driver events are distinct streams. Bind
* the retained raw journal to the server-accepted result, not a guessed shared
* event identifier. This is read-only and never interprets a tool as a request. */
export function retainedNativeCleanupJournalMatches(input: {
run: Pick<
typeof heartbeatRuns.$inferSelect,
| "id"
| "companyId"
| "agentId"
| "nativeIssueId"
| "nativeSessionId"
| "runnerInstanceId"
| "completionContractId"
| "completionContractSha256"
>;
execution: NativeExecutionInput;
accepted: Pick<
typeof nativeRunResults.$inferSelect,
| "schemaStatus"
| "resultJson"
| "turnId"
| "canonicalSha256"
| "serverFingerprint"
>;
control: Record<string, unknown>;
providerSessionId: string;
providerAccountSessionId?: string | null;
persistedEvents: Array<
Pick<
typeof heartbeatRunEvents.$inferSelect,
| "eventType"
| "payload"
| "sourceInstanceId"
| "sourceEventId"
| "sourceSeq"
| "sourcePayloadSha256"
>
>;
}): boolean {
const { run, accepted, control } = input;
const envelope = record(accepted.resultJson);
const terminal = record(envelope.terminal);
if (
accepted.schemaStatus !== "accepted" ||
!accepted.turnId ||
terminal.turnTerminalState !== "completed" ||
terminal.runTerminalState !== "succeeded" ||
!Array.isArray(control.committedEvents) ||
!Array.isArray(control.commands)
)
return false;
const canonical = {
result: envelope.result,
terminal: envelope.terminal,
turnId: accepted.turnId,
};
const fingerprint = nativeSha256({
runId: run.id,
completionContractSha256: run.completionContractSha256,
canonicalSha256: accepted.canonicalSha256,
});
const validPersisted = input.persistedEvents.filter((row) => {
const parsed = validatePrpEvent(record(row.payload).prpEvent);
return (
parsed.ok &&
parsed.event.runId === run.id &&
parsed.event.normalizedSessionId === run.nativeSessionId &&
parsed.event.sourceInstanceId === row.sourceInstanceId &&
parsed.event.sourceEventId === row.sourceEventId &&
parsed.event.sourceSeq === row.sourceSeq &&
row.sourcePayloadSha256 === nativeSha256(parsed.event)
);
});
const identities = validPersisted.filter(
(row) =>
["session.started", "session.resumed"].includes(row.eventType) &&
row.sourceInstanceId === run.runnerInstanceId,
);
if (identities.length !== 1) return false;
const identityEvent = record(record(identities[0]!.payload).prpEvent);
const identityPayload = record(identityEvent.payload);
if (
identityPayload.providerSessionId !==
(input.providerAccountSessionId ?? input.providerSessionId) ||
identityPayload.driverSessionId !== input.providerSessionId ||
identityEvent.sourceKind !== "runner" ||
identityEvent.eventType !== identities[0]!.eventType ||
identityEvent.sourceEventId !==
`${run.runnerInstanceId}:${run.id}:${identityEvent.sourceSeq}`
)
return false;
const boundDigest = validPersisted.some((row) => {
const event = record(record(row.payload).prpEvent);
return (
event.sourceKind === "control_plane" &&
nativeSha256({
binding: {
companyId: run.companyId,
issueId: run.nativeIssueId,
agentId: run.agentId,
runId: run.id,
sessionId: run.nativeSessionId,
sourceInstanceId: run.runnerInstanceId,
controlPlaneSourceInstanceId: row.sourceInstanceId,
completionContractId: run.completionContractId,
completionContractSha256: run.completionContractSha256,
},
...canonical,
}) === accepted.canonicalSha256
);
});
if (!(
(accepted.canonicalSha256 === `sha256:${nativeSha256(canonical)}` &&
accepted.serverFingerprint === `sha256:${fingerprint}`) ||
(accepted.serverFingerprint === fingerprint && boundDigest)
))
return false;
const events = control.committedEvents.map((entry) =>
record(record(record(entry).envelope).payload),
);
const durableIdentity = record(control.identity);
const boundRaw = (event: Record<string, unknown>) =>
validatePrpEvent(event).ok &&
event.runId === run.id &&
event.sourceInstanceId === run.runnerInstanceId &&
event.sourceKind === "runner" &&
event.normalizedSessionId === run.nativeSessionId &&
event.turnId === durableIdentity.turnId &&
event.itemId === durableIdentity.itemId;
const commands = control.commands.map(record);
const contract = input.execution.completionContract.contract;
if (
!commands.some(
(command) =>
["run.prepare", "run.attach"].includes(String(command.type)) &&
command.status === "completed" &&
canonicalJson(record(command.payload).completionContract) ===
canonicalJson({
revision: contract.revision,
criterionIds: contract.criteria.map((criterion) => criterion.id),
}),
)
)
return false;
if (
!commands.some(
(command) =>
command.type === "turn.start" &&
command.status === "completed" &&
record(record(command.result).result).providerTurnId ===
accepted.turnId,
)
)
return false;
if (
!events.some(
(event) =>
event.eventType === "turn.accepted" &&
boundRaw(event) &&
record(event.payload).providerTurnId === accepted.turnId &&
record(event.payload).providerSessionId === input.providerSessionId,
)
)
return false;
const submissions = events.filter(
(event) =>
event.eventType === "semantic_tool.input" &&
record(event.payload).semantic_tool &&
record(record(event.payload).semantic_tool).operationId ===
"paperclip_finish",
);
if (submissions.length !== 1) return false;
const event = submissions[0]!;
const semantic = record(record(event.payload).semantic_tool);
const correlation = record(semantic.correlation);
const result = validatePrpStructuredRunResult(semantic.input);
if (
!boundRaw(event) ||
!result.ok ||
canonicalJson(result.result) !== canonicalJson(envelope.result) ||
semantic.schema !== "paperclip.prp.semantic_tool.v1" ||
semantic.schemaVersion !== 1 ||
semantic.phase !== "input" ||
typeof semantic.callId !== "string" ||
!semantic.callId ||
record(semantic.content).digest !==
`sha256:${nativeSha256(semantic.input)}` ||
event.runId !== run.id ||
event.sourceInstanceId !== run.runnerInstanceId ||
event.normalizedSessionId !== run.nativeSessionId ||
correlation.runId !== run.id ||
correlation.normalizedSessionId !== run.nativeSessionId ||
correlation.turnId !== event.turnId ||
correlation.itemId !== event.itemId
)
return false;
return (
events.some((candidate) => {
const outcome = record(record(candidate.payload).semantic_tool);
return (
candidate.eventType === "semantic_tool.result" &&
boundRaw(candidate) &&
candidate.runId === run.id &&
candidate.sourceInstanceId === run.runnerInstanceId &&
candidate.normalizedSessionId === run.nativeSessionId &&
outcome.schema === semantic.schema &&
outcome.schemaVersion === 1 &&
outcome.phase === "result" &&
outcome.operationId === semantic.operationId &&
outcome.callId === semantic.callId &&
outcome.outcome === "succeeded" &&
outcome.code === "semantic_tool_succeeded" &&
outcome.operationReceiptId === `operation_${semantic.callId}` &&
canonicalJson(outcome.correlation) ===
canonicalJson(semantic.correlation)
);
}) &&
commands.some((command) => {
const payload = record(command.payload);
return (
command.type === "semantic_tool.result" &&
command.status === "completed" &&
payload.callId === semantic.callId &&
payload.operationId === semantic.operationId &&
payload.isError === false &&
payload.sourceEventId === event.sourceEventId &&
payload.sourceEventType === event.eventType &&
// Current receipts retain a digest; older journals retain the input.
// If both exist, both must bind to the accepted semantic input.
(payload.inputDigest === undefined
? canonicalJson(payload.input) === canonicalJson(semantic.input)
: payload.inputDigest === nativeSha256(semantic.input) &&
(payload.input === undefined ||
canonicalJson(payload.input) === canonicalJson(semantic.input))) &&
canonicalJson(payload.correlation) ===
canonicalJson(semantic.correlation) &&
record(record(command.result).result).callId === semantic.callId
);
})
);
}
/** Durable, content-free evidence for an authenticated maintenance event.
* Never feed raw cleanup output into the normal driver/progress namespace. */
export async function appendRetainedNativeCleanupEvent(
db: Db,
input: {
companyId: string;
agentId: string;
runId: string;
nativeSessionId: string;
runnerInstanceId: string;
requestId: string;
event: PrpEvent;
},
): Promise<void> {
const parsed = validatePrpEvent(input.event);
if (
!parsed.ok ||
parsed.event.runId !== input.runId ||
parsed.event.normalizedSessionId !== input.nativeSessionId ||
parsed.event.sourceInstanceId !== input.runnerInstanceId ||
parsed.event.sourceKind !== "runner" ||
!input.requestId
) {
throw new Error("native_cleanup_maintenance_unproven");
}
const event = parsed.event;
const receipt = {
schema: "paperclip.native_cleanup_event.v1",
requestId: input.requestId,
rawSourceInstanceId: event.sourceInstanceId,
rawSourceEventId: event.sourceEventId,
rawSourceSeq: event.sourceSeq,
rawEventType: event.eventType,
rawCanonicalSha256: nativeSha256(event),
};
await appendHeartbeatRunEvent(db, {
companyId: input.companyId,
agentId: input.agentId,
runId: input.runId,
eventType: "native.cleanup.event",
stream: "system",
level: "info",
payload: { nativeCleanupEvent: receipt },
nativeSource: {
sourceInstanceId: `${input.runnerInstanceId}:cleanup:${input.requestId}`,
sourceEventId: `cleanup:${input.requestId}:${event.sourceEventId}`,
sourceSeq: event.sourceSeq,
protocolSchemaVersion: 1,
canonicalPayload: receipt,
},
});
}
function cleanupProviderHomeSnapshot(home: string, content: boolean) {
const entries: Array<{
path: string;
directory: boolean;
dev: number;
ino: number;
size: number;
mtimeMs: number;
ctimeMs: number;
sha256?: string;
}> = [];
let bytes = 0;
const visit = (relative: string, depth: number) => {
if (depth > 32 || entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES)
throw new Error("native_cleanup_maintenance_unproven");
const path = resolve(home, relative);
const metadata = lstatSync(path);
if (
metadata.isSymbolicLink() ||
(!metadata.isFile() && !metadata.isDirectory())
)
throw new Error("native_cleanup_maintenance_unproven");
const entry = {
path: relative,
directory: metadata.isDirectory(),
dev: metadata.dev,
ino: metadata.ino,
size: metadata.isDirectory() ? 0 : metadata.size,
mtimeMs: metadata.mtimeMs,
ctimeMs: metadata.ctimeMs,
};
entries.push(entry);
if (entry.directory) {
const directory = opendirSync(path);
const children: string[] = [];
try {
for (
let child = directory.readSync();
child !== null;
child = directory.readSync()
) {
if (
!relative &&
(CODEX_HOME_NON_PERSISTENT_ENTRIES as readonly string[]).includes(
child.name,
)
)
continue;
if (children.length + entries.length >= MAX_REMOTE_CHECKPOINT_ENTRIES)
throw new Error("native_cleanup_maintenance_unproven");
children.push(child.name);
}
} finally {
directory.closeSync();
}
for (const name of children.sort())
visit(relative ? `${relative}/${name}` : name, depth + 1);
} else {
bytes += metadata.size;
if (bytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES)
throw new Error("native_cleanup_maintenance_unproven");
if (content) {
const value = readBoundedNativeFile(
path,
metadata.size,
"native_cleanup_maintenance_unproven",
);
(entry as (typeof entries)[number]).sha256 = createHash("sha256")
.update(value)
.digest("hex");
}
}
const after = lstatSync(path);
if (
after.dev !== metadata.dev ||
after.ino !== metadata.ino ||
after.mtimeMs !== metadata.mtimeMs ||
after.ctimeMs !== metadata.ctimeMs ||
after.size !== metadata.size
)
throw new Error("native_cleanup_maintenance_unproven");
};
visit("", 0);
return {
entries,
bytes,
metadataFingerprint: nativeSha256(
entries.map(({ sha256: _sha, ...entry }) => entry),
),
fingerprint: content
? nativeSha256(
entries.map(({ path, directory, size, sha256 }) => ({
path,
directory,
size,
...(sha256 ? { sha256 } : {}),
})),
)
: null,
};
}
function copyCleanupProviderHome(
source: string,
destination: string,
snapshot: ReturnType<typeof cleanupProviderHomeSnapshot>,
) {
for (const entry of snapshot.entries) {
const target = resolve(destination, entry.path);
if (entry.directory) mkdirSync(target, { mode: 0o700 });
else {
const value = readBoundedNativeFile(
resolve(source, entry.path),
entry.size,
"native_cleanup_maintenance_unproven",
);
if (createHash("sha256").update(value).digest("hex") !== entry.sha256)
throw new Error("native_cleanup_maintenance_unproven");
writeFileSync(target, value, { flag: "wx", mode: 0o600 });
}
}
if (
cleanupProviderHomeSnapshot(source, false).metadataFingerprint !==
snapshot.metadataFingerprint ||
cleanupProviderHomeSnapshot(destination, true).fingerprint !==
snapshot.fingerprint
)
throw new Error("native_cleanup_maintenance_unproven");
}
export function rebaseRetainedNativeCleanupProviderHome(
home: string,
canonicalHome: string,
threadId: string,
destination: "staging" | "canonical",
) {
if (resolve(home) === resolve(canonicalHome))
throw new Error("native_cleanup_maintenance_unproven");
const snapshot = cleanupProviderHomeSnapshot(home, true);
const rollouts = snapshot.entries.filter(
(entry) =>
!entry.directory &&
entry.path.startsWith("sessions/") &&
entry.path.endsWith(`-${threadId}.jsonl`),
);
if (rollouts.length !== 1)
throw new Error("native_cleanup_maintenance_unproven");
const rollout = rollouts[0]!;
const bytes = readBoundedNativeFile(
resolve(home, rollout.path),
rollout.size,
"native_cleanup_maintenance_unproven",
);
const newline = bytes.indexOf(10);
if (newline < 0 || newline > 64 * 1024)
throw new Error("native_cleanup_maintenance_unproven");
const first = record(JSON.parse(bytes.subarray(0, newline).toString("utf8")));
if (first.type !== "session_meta" || record(first.payload).id !== threadId)
throw new Error("native_cleanup_maintenance_unproven");
// Only open the NEW private copy. Codex 0.153.4 deliberately does not fall
// back to a filesystem scan for paginated threads: SQLite selects the exact
// immutable rollout, which can differ after thread/revert. Relocate only
// that already-proven path, never choose another rollout or change its mode.
const sqlite = resolve(home, "state_5.sqlite");
if (
snapshot.entries.some(
(entry) =>
/^state_\d+\.sqlite$/.test(entry.path) &&
entry.path !== "state_5.sqlite",
)
)
throw new Error("native_cleanup_maintenance_unproven");
if (!existsSync(sqlite)) return;
const database = new DatabaseSync(sqlite);
try {
database.exec("BEGIN IMMEDIATE");
// The pinned schema has insert and timestamp-only triggers. None fire
// for this column-only update; unknown/general update triggers deny it.
if (
database
.prepare(
"SELECT sql FROM sqlite_schema WHERE type = 'trigger' AND tbl_name COLLATE NOCASE = 'threads'",
)
.all()
.some(
(trigger) =>
typeof trigger.sql !== "string" ||
!/^CREATE\s+TRIGGER\s+[a-z_][a-z0-9_]*\s+AFTER\s+(?:INSERT|UPDATE\s+OF\s+(?:created_at|updated_at))\s+ON\s+threads\s/i.test(
trigger.sql,
),
)
)
throw new Error("native_cleanup_maintenance_unproven");
// Foreign-key actions can also mutate other tables without an explicit
// trigger. Unknown cascading topology is not an exact path relocation.
if (
database
.prepare(
`SELECT 1 FROM sqlite_schema AS s
JOIN pragma_foreign_key_list(s.name) AS f
WHERE s.type = 'table' AND f."table" COLLATE NOCASE = 'threads'
AND f.on_update NOT IN ('NO ACTION', 'RESTRICT') LIMIT 1`,
)
.get()
)
throw new Error("native_cleanup_maintenance_unproven");
const row = database
.prepare("SELECT * FROM threads WHERE id = ?")
.get(threadId);
if (
!row ||
typeof row.rollout_path !== "string" ||
![
resolve(home, rollout.path),
resolve(canonicalHome, rollout.path),
].includes(row.rollout_path) ||
!["legacy", "paginated"].includes(String(row.history_mode)) ||
(record(first.payload).history_mode ?? "legacy") !== row.history_mode
)
throw new Error("native_cleanup_maintenance_unproven");
const target = resolve(
destination === "staging" ? home : canonicalHome,
rollout.path,
);
if (row.rollout_path !== target) {
const changed = database
.prepare(
"UPDATE threads SET rollout_path = ? WHERE id = ? AND rollout_path = ? AND history_mode = ?",
)
.run(target, threadId, row.rollout_path, row.history_mode);
if (changed.changes !== 1)
throw new Error("native_cleanup_maintenance_unproven");
}
const after = database
.prepare("SELECT * FROM threads WHERE id = ?")
.get(threadId);
if (nativeSha256(after) !== nativeSha256({ ...row, rollout_path: target }))
throw new Error("native_cleanup_maintenance_unproven");
database.exec("COMMIT");
} catch (error) {
if (database.isTransaction) database.exec("ROLLBACK");
throw error;
} finally {
database.close();
}
}
/** Physical cleanup for an explicitly authorized NEW conversation turn. Unlike
* automatic replacement, this does not certify or replay the interrupted actions.
* The caller holds the issue/controller/run locks and preserves their history.
* Retain the old durable root permanently; only the exact in-memory cleanup
* owner is retired, and the successor must use a fresh normalized session.
*/
export async function verifyStoppedNativeSessionForContinuation(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
): Promise<{ evidence: Record<string, unknown>; retire: () => boolean } | null> {
try {
if (run.runtimeMode !== "native" || !["failed", "cancelled", "interrupted", "timed_out"].includes(run.status) ||
!run.finishedAt || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null;
const execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput);
if (!(["codex", "acpx"] as string[]).includes(execution.provider.kind) ||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
nativeSessionKey(execution) !== run.nativeSessionId ||
record(run.runnerProfileJson).nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget("local")) return null;
const scope = nativeSessionScopeKey(execution);
const idle = () => !activeNativeSessions.has(run.id) && !executingRunnerdSessionScopes.has(scope) &&
!initializingSessionToolAuthorities.has(scope) && !warmNativeSessions.has(scope);
if (!idle()) return null;
const leases = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id)));
if (leases.some(lease => lease.provider !== "local" || !lease.releasedAt || lease.cleanupStatus === "failed")) return null;
const stopped = await readNativeLocalProcessStop(db, run.companyId, run.id);
if (!stopped) return null;
const root = scopedRunnerdStateRoot(execution);
const providerFile = runnerProviderStateFilename(execution);
const snapshot = cleanupStateSnapshot(root, providerFile);
const identity = record(snapshot.control.identity);
if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId ||
!["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key =>
typeof identity[key] === "string" && identity[key] && snapshot.runner[key] === identity[key]) ||
!Array.isArray(snapshot.control.committedEvents)) return null;
// An incomplete provider launch can own a process that never emitted its
// session identity. It cannot be certified from an earlier owner's receipt.
if (snapshot.control.schema !== "paperclip.runner.durable.control-plane-state.v1" ||
snapshot.runner.schema !== RUNNERD_STATE_SCHEMA ||
snapshot.provider.schema !== (execution.provider.kind === "codex" ? "paperclip.runner.codex-provider-state.v1" : ACPX_PROVIDER_STATE_SCHEMA) ||
!["turn_active", "prepared", "suspended"].includes(String(snapshot.provider.lifecycle)) ||
snapshot.provider.startupAttempt != null) return null;
const pending = JSON.stringify([snapshot.runner.outbox, snapshot.provider.pendingEvents, snapshot.provider.queuedEvents]);
if (/session\.(started|resumed|reconciled)/.test(pending)) return null;
const events = snapshot.control.committedEvents.map(entry => record(record(record(entry).envelope).payload));
const providerEvents = events.filter(event => ["session.started", "session.resumed", "session.reconciled"].includes(String(event.eventType)));
if (!providerEvents.length) return null;
const receipts = await db.select().from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId),
inArray(heartbeatRunEvents.eventType, ["session.started", "session.resumed", "session.reconciled", "harness.diagnostic"])));
const providerPids = new Set<number>();
for (const event of providerEvents) {
const provider = record(event.payload);
if (!validatePrpEvent(event).ok || event.sourceKind !== "runner" || event.sourceInstanceId !== run.runnerInstanceId ||
event.runId !== run.id || event.normalizedSessionId !== run.nativeSessionId ||
typeof provider.providerSessionId !== "string" || !provider.providerSessionId ||
!cleanupProcessAbsent(provider.processId) || provider.processId === stopped.processPid) return null;
if (event.eventType === "session.reconciled" &&
(!providerPids.has(Number(provider.previousProcessId)) || !cleanupProcessAbsent(provider.previousProcessId))) return null;
const receipt = receipts.find(row => row.sourceEventId === `${run.runnerInstanceId}:${run.id}:${event.sourceSeq}`);
const durable = record(record(receipt?.payload).prpEvent);
const durableProvider = record(durable.payload);
if (!receipt || !validatePrpEvent(durable).ok || durable.sourceInstanceId !== event.sourceInstanceId ||
durable.runId !== run.id || durable.normalizedSessionId !== run.nativeSessionId ||
// Normalized session-open receipts omit turn/item IDs. Those belong
// to the durable runner identity checked above, not the open event.
durable.sourceSeq !== event.sourceSeq || durable.eventType !== event.eventType ||
receipt.sourcePayloadSha256 !== nativeSha256(durable) ||
(durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) ||
(durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null;
providerPids.add(provider.processId);
}
if (receipts.filter(row => record(record(row.payload).prpEvent).eventType !== "harness.diagnostic").length !== providerEvents.length) return null;
// ACPX's sidecar is not its agent process. Include separately recorded
// owners from both the durable journal and the independent DB/checkpoint.
// Extra evidence can only add a stop requirement, never waive one.
const owners: Record<string, unknown>[] = [record(record(run.runnerProfileJson?.sessionCheckpoint).process),
snapshot.provider, record(snapshot.provider.identity), record(snapshot.provider.descriptor)];
for (const event of [...events, ...receipts.map(row => record(record(row.payload).prpEvent))]) {
const payload = record(event.payload);
if (["session.started", "session.resumed", "session.reconciled"].includes(String(event.eventType))) {
owners.push(payload, record(payload.providerDescriptor), record(payload.providerIdentity), record(payload.runtimeIdentity));
} else if (event.eventType === "harness.diagnostic" && payload.providerMethod === "acpx/process") {
owners.push({ agentPid: payload.pid });
}
}
for (const owner of owners) for (const key of [
"processId", "process_id", "processGroupId", "providerPid", "codexPid", "sidecarPid", "agentPid", "agentProcessId",
]) {
const pid = owner[key];
if (pid === null || pid === undefined) continue;
if (!cleanupProcessAbsent(pid)) return null;
providerPids.add(pid);
}
const unchanged = () => idle() && cleanupProcessAbsent(stopped.processPid) &&
[...providerPids].every(cleanupProcessAbsent) && cleanupStateSnapshot(root, providerFile).fingerprint === snapshot.fingerprint;
return {
evidence: { schema: "paperclip.stopped_native_conversation.v1", runId: run.id,
nativeSessionId: run.nativeSessionId, runnerInstanceId: run.runnerInstanceId,
processPid: stopped.processPid, providerPids: [...providerPids], stateFingerprint: snapshot.fingerprint },
retire: () => {
try { return unchanged() && completeTerminatedLocalNativeSessionCleanup({
companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId!,
}); } catch { return false; }
},
};
} catch { return null; }
}
/** Prove that a crashed local Codex turn ended without external effects. No provider
* is launched and no retained state is rewritten. The successor must use a fresh
* normalized session; the old directory remains available for investigation. */
export async function verifyStoppedNativeSessionForReplacement(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
): Promise<{ evidence: Record<string, unknown>; retire: () => boolean } | null> {
try {
if (run.runtimeMode !== "native" || run.status !== "failed" || !run.finishedAt ||
!run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return null;
const execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput);
if (execution.provider.kind !== "codex" || execution.session.driverKind !== "codex_app_server" ||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
nativeSessionKey(execution) !== run.nativeSessionId ||
record(run.runnerProfileJson).nativeToolContractFingerprint !== nativeToolContractFingerprintForTarget("local")) return null;
const scope = nativeSessionScopeKey(execution);
const idle = () => !activeNativeSessions.has(run.id) && !executingRunnerdSessionScopes.has(scope) &&
!initializingSessionToolAuthorities.has(scope) && !warmNativeSessions.has(scope);
if (!idle()) return null;
const leases = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id)));
if (leases.some(lease => lease.provider !== "local" || !lease.releasedAt)) return null;
const stopped = await readNativeLocalProcessStop(db, run.companyId, run.id);
if (!stopped) return null;
const root = scopedRunnerdStateRoot(execution);
const snapshot = cleanupStateSnapshot(root);
const identity = record(snapshot.control.identity);
if (!durableIdentityMatchesExecution(identity, execution) || identity.runnerInstanceId !== run.runnerInstanceId ||
!["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"].every(key =>
typeof identity[key] === "string" && identity[key] && snapshot.runner[key] === identity[key]) ||
snapshot.runner.lifecycle !== "ready" || snapshot.provider.lifecycle !== "turn_active" ||
record(snapshot.provider.config).provider !== "codex" || record(snapshot.provider.config).cwd !== execution.workspace.cwd ||
!Array.isArray(snapshot.control.committedEvents) || !Array.isArray(snapshot.control.commands)) return null;
const events = snapshot.control.committedEvents.map(entry => record(record(record(entry).envelope).payload));
const providerEvent = events.filter(event => ["session.started", "session.resumed"].includes(String(event.eventType))).at(-1);
const provider = record(providerEvent?.payload);
const bound = (event: Record<string, unknown>) => validatePrpEvent(event).ok && event.sourceKind === "runner" &&
event.sourceInstanceId === run.runnerInstanceId && event.runId === run.id &&
event.normalizedSessionId === run.nativeSessionId && event.turnId === identity.turnId && event.itemId === identity.itemId;
if (!providerEvent || !bound(providerEvent) || !cleanupProcessAbsent(provider.processId) ||
provider.processId === stopped.processPid || typeof provider.providerSessionId !== "string" ||
snapshot.provider.threadId !== provider.providerSessionId ||
typeof snapshot.provider.activeProviderTurnId !== "string") return null;
const receipts = await db.select().from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId),
inArray(heartbeatRunEvents.eventType, ["session.started", "session.resumed"]))).limit(2);
const receipt = receipts.length === 1 ? receipts[0] : undefined;
const durableEvent = record(record(receipt?.payload).prpEvent);
// The persisted adapter enriches identity payloads with driverSessionId. Bind
// the normalized and provider identities explicitly instead of comparing raw JSON.
const durableProvider = record(durableEvent.payload);
if (!receipt || !validatePrpEvent(durableEvent).ok || durableEvent.runId !== run.id ||
durableEvent.sourceInstanceId !== run.runnerInstanceId || durableEvent.normalizedSessionId !== run.nativeSessionId ||
receipt.sourceEventId !== `${run.runnerInstanceId}:${run.id}:${durableEvent.sourceSeq}` ||
receipt.sourcePayloadSha256 !== nativeSha256(durableEvent) ||
(durableProvider.processId !== undefined && durableProvider.processId !== provider.processId) ||
(durableProvider.driverSessionId ?? durableProvider.providerSessionId) !== provider.providerSessionId) return null;
const turnId = snapshot.provider.activeProviderTurnId;
if (!snapshot.control.commands.map(record).some(command => command.type === "turn.start" && command.status === "completed" &&
record(record(command.result).result).providerTurnId === turnId) ||
!events.some(event => bound(event) && event.eventType === "turn.accepted" &&
record(event.payload).providerTurnId === turnId && record(event.payload).providerSessionId === provider.providerSessionId)) return null;
// Completion bookkeeping may precede the final answer. Its exact accepted
// receipt is safe to preserve; arbitrary provider tools still prevent replay.
const completedTaskControlCalls: Array<{ callId: string; input: unknown }> = [];
for (const event of events.filter(event => event.eventType === "semantic_tool.input")) {
const semantic = record(record(event.payload).semantic_tool);
const correlation = record(semantic.correlation);
if (!bound(event) || semantic.operationId !== "paperclip_finish" || semantic.phase !== "input" ||
typeof semantic.callId !== "string" || !validatePrpStructuredRunResult(semantic.input).ok ||
correlation.runId !== run.id || correlation.normalizedSessionId !== run.nativeSessionId ||
correlation.turnId !== identity.turnId || correlation.itemId !== identity.itemId ||
record(semantic.content).digest !== `sha256:${nativeSha256(semantic.input)}` ||
!events.some(resultEvent => {
const result = record(record(resultEvent.payload).semantic_tool);
return bound(resultEvent) && resultEvent.eventType === "semantic_tool.result" &&
result.operationId === semantic.operationId && result.callId === semantic.callId &&
result.outcome === "succeeded" && result.operationReceiptId === `operation_${semantic.callId}` &&
canonicalJson(result.correlation) === canonicalJson(semantic.correlation);
}) || !snapshot.control.commands.map(record).some(command => {
const payload = record(command.payload);
return command.type === "semantic_tool.result" && command.status === "completed" &&
payload.callId === semantic.callId && payload.operationId === semantic.operationId &&
payload.sourceEventId === event.sourceEventId && payload.isError === false &&
record(payload.result).success === true && canonicalJson(payload.input) === canonicalJson(semantic.input) &&
canonicalJson(payload.correlation) === canonicalJson(semantic.correlation);
})) return null;
completedTaskControlCalls.push({ callId: semantic.callId, input: semantic.input });
}
if (completedTaskControlCalls.length > 1) return null;
const pendingInventory = JSON.stringify([snapshot.runner.outbox, snapshot.provider.pendingEvents, snapshot.provider.queuedEvents]);
if (/semantic_tool\.input|mcp_app\.tool_input|runtime\.input\.requested|runtime_request\.created/.test(pendingInventory) ||
events.some(event => ["mcp_app.tool_input", "runtime.input.requested", "runtime_request.created"].includes(String(event.eventType)))) return null;
if (snapshot.control.commands.map(record).some(command => command.status === "pending" &&
!["turn.stop", "runner.drain", "runner.suspend"].includes(String(command.type)))) return null;
const home = cleanupProviderHomeSnapshot(resolve(root, "codex-home"), false);
const rollouts = home.entries.filter(entry => !entry.directory && entry.path.startsWith("sessions/") &&
basename(entry.path).endsWith(`-${provider.providerSessionId}.jsonl`));
if (rollouts.length !== 1 || rollouts[0]!.size > 32 * 1024 * 1024) return null;
const rolloutPath = resolve(root, "codex-home", rollouts[0]!.path);
const bytes = readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven");
// A partial final write is not a closed transcript.
if (!bytes.toString("utf8").endsWith("\n")) return null;
const rows = bytes.toString("utf8").trimEnd().split("\n").map(line => JSON.parse(line));
if (!stoppedCodexTurnIsTextOnly({ rows, threadId: provider.providerSessionId, turnId, cwd: execution.workspace.cwd ?? "", completedTaskControlCalls })) return null;
const rolloutSha256 = nativeSha256(bytes.toString("utf8"));
const evidence = { schema: "paperclip.stopped_text_turn.v1", runId: run.id, nativeSessionId: run.nativeSessionId,
runnerInstanceId: run.runnerInstanceId, processPid: stopped.processPid, providerPid: provider.processId,
providerSessionId: provider.providerSessionId, providerTurnId: turnId,
stateFingerprint: snapshot.fingerprint, rolloutSha256,
completedTaskControlCallIds: completedTaskControlCalls.map(call => call.callId) };
return {
evidence,
retire: () => idle() && cleanupProcessAbsent(stopped.processPid) && cleanupProcessAbsent(provider.processId) &&
cleanupStateSnapshot(root).fingerprint === snapshot.fingerprint &&
nativeSha256(readBoundedNativeFile(rolloutPath, 32 * 1024 * 1024, "native_crash_inventory_unproven").toString("utf8")) === rolloutSha256 &&
completeTerminatedLocalNativeSessionCleanup({ companyId: run.companyId, runId: run.id, runnerInstanceId: run.runnerInstanceId! }),
};
} catch { return null; }
}
/** Exact local cleanup only: the accepted result and original quarantine are
* never rewritten. A failed/interrupted maintenance attempt is retained for
* inspection, not retried from an older snapshot with unknown process owners. */
export async function reconcileRetainedNativeSessionCleanup(
db: Db,
input: {
companyId: string;
runId: string;
},
): Promise<{
status: "settled" | "not_eligible" | "operator_required";
runId: string;
}> {
const denied = () => new Error("native_cleanup_maintenance_unproven");
const leaseOwner = `native-cleanup:${randomUUID()}`;
let reservedScope: string | null = null;
const releaseScope = () => {
if (
reservedScope &&
executingRunnerdSessionScopes.get(reservedScope) === leaseOwner
) {
executingRunnerdSessionScopes.delete(reservedScope);
}
};
let claim: {
execution: NativeExecutionInput;
run: typeof heartbeatRuns.$inferSelect;
quarantine: string;
root: string;
emptyRoot: ReturnType<typeof cleanupCanonicalVacancy>;
source: ReturnType<typeof cleanupStateSnapshot>;
providerHome: ReturnType<typeof cleanupProviderHomeSnapshot>;
sourceArchive: {
intent: Record<string, unknown>;
fromCanonical: boolean;
completed: boolean;
} | null;
copySource: {
directory: string;
snapshot: ReturnType<typeof cleanupStateSnapshot>;
requestId: string;
} | null;
providerPid: number;
providerSessionId: string;
identity: {
runnerInstanceId: string;
environmentLeaseId: string;
runId: string;
normalizedSessionId: string;
turnId: string;
itemId: string;
};
history: Array<Record<string, unknown>>;
} | null = null;
try {
claim = await db.transaction(async (tx) => {
const run = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, input.runId),
eq(heartbeatRuns.companyId, input.companyId),
),
)
.for("update", { noWait: true })
.limit(1)
.then((rows) => rows[0]);
if (
!run ||
run.runtimeMode !== "native" ||
!run.nativeIssueId ||
!run.nativeSessionId ||
!run.runnerInstanceId ||
!["succeeded", "failed"].includes(run.status) ||
!run.finishedAt ||
!cleanupProcessAbsent(run.processPid) ||
run.processGroupId !== run.processPid
)
return null;
const execution = parseNativeExecutionInput(
record(run.runnerProfileJson).nativeExecutionInput,
);
const failure = record(record(run.resultJson).recoveredExecutionFailure);
const errorCode = run.errorCode ?? failure.errorCode;
const error = run.error ?? failure.error;
if (
!["adapter_failed", "provider_transport_failed"].includes(String(errorCode)) ||
error !==
"provider_transport_failed: runner did not durably suspend before checkpoint" ||
execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId ||
execution.binding.issueId !== run.nativeIssueId ||
execution.binding.runId !== run.id ||
nativeSessionKey(execution) !== run.nativeSessionId ||
execution.provider.kind !== "codex" ||
execution.session.driverKind !== "codex_app_server" ||
record(run.runnerProfileJson).nativeToolContractFingerprint !==
nativeToolContractFingerprintForTarget("local")
)
return null;
const scope = nativeSessionScopeKey(execution);
if (
activeNativeSessions.has(run.id) ||
executingRunnerdSessionScopes.has(scope) ||
initializingSessionToolAuthorities.has(scope) ||
warmNativeSessions.has(scope)
)
return null;
executingRunnerdSessionScopes.set(scope, leaseOwner);
reservedScope = scope;
const coordinator = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, run.id),
eq(nativeRunFinalizations.companyId, run.companyId),
eq(nativeRunFinalizations.issueId, run.nativeIssueId),
),
)
.for("update", { noWait: true })
.limit(1)
.then((rows) => rows[0]);
if (
!coordinator ||
coordinator.phase !== "committed" ||
!coordinator.resultId ||
!coordinator.assessmentId ||
!coordinator.decisionId ||
coordinator.nextAttemptAt ||
(coordinator.leaseOwner &&
coordinator.leaseExpiresAt &&
coordinator.leaseExpiresAt > new Date()) ||
coordinator.recoveryHistory.some(
(event) => event.kind === "native_cleanup_runner_epoch",
)
)
return null;
const result = await tx
.select()
.from(nativeRunResults)
.where(
and(
eq(nativeRunResults.id, coordinator.resultId),
eq(nativeRunResults.runId, run.id),
eq(nativeRunResults.companyId, run.companyId),
eq(nativeRunResults.issueId, run.nativeIssueId),
),
)
.for("share", { noWait: true })
.limit(1)
.then((rows) => rows[0]);
if (!result || result.schemaStatus !== "accepted") return null;
const environment = await tx
.select({ id: environmentLeases.id })
.from(environmentLeases)
.where(
and(
eq(environmentLeases.companyId, run.companyId),
eq(environmentLeases.heartbeatRunId, run.id),
inArray(environmentLeases.status, ["active", "pending_cleanup"]),
),
)
.limit(1);
if (environment.length) return null;
const root = scopedRunnerdStateRoot(execution);
const maintenanceHistory = coordinator.recoveryHistory.filter(
(event) => event.kind === "native_cleanup_maintenance",
);
const archiveHistory = coordinator.recoveryHistory.filter(
(event) => event.kind === "native_cleanup_source_archive",
);
const parent = resolve(runnerdStateBase(), "quarantine");
if (existsSync(parent) && !isSafeNativeStateDirectory(parent))
return null;
const entries = existsSync(parent) ? readdirSync(parent) : [];
if (entries.length > 4096) return null;
const candidates = entries.filter((name) =>
name.startsWith(`${basename(root)}.identity_indeterminate.`),
);
let emptyRoot: ReturnType<typeof cleanupCanonicalVacancy> = null;
let sourceDirectory: string;
let quarantine: string;
let sourceArchive: NonNullable<typeof claim>["sourceArchive"] = null;
const rootExists = !!lstatSync(root, { throwIfNoEntry: false });
if (rootExists && !isSafeNativeStateDirectory(root)) return null;
if (archiveHistory.length) {
const prepared = archiveHistory[0]!;
const archived = archiveHistory[1];
if (
maintenanceHistory.length ||
archiveHistory.length > 2 ||
prepared.version !== 1 ||
prepared.phase !== "prepared" ||
prepared.companyId !== run.companyId ||
prepared.agentId !== run.agentId ||
prepared.runId !== run.id ||
prepared.nativeSessionId !== run.nativeSessionId ||
prepared.runnerInstanceId !== run.runnerInstanceId ||
prepared.stateKey !== basename(root) ||
typeof prepared.requestId !== "string" ||
!prepared.requestId.startsWith("native-cleanup:") ||
typeof prepared.archiveName !== "string" ||
prepared.archiveName.length > 192 ||
basename(prepared.archiveName) !== prepared.archiveName ||
!prepared.archiveName.startsWith(
`${basename(root)}.identity_indeterminate.cleanup.`,
) ||
typeof prepared.sourceFingerprint !== "string" ||
!/^[0-9a-f]{64}$/.test(prepared.sourceFingerprint) ||
typeof prepared.providerHomeFingerprint !== "string" ||
!/^[0-9a-f]{64}$/.test(prepared.providerHomeFingerprint) ||
(archived &&
(archived.phase !== "archived" ||
Object.entries(prepared).some(
([key, value]) =>
key !== "phase" &&
canonicalJson(archived[key]) !== canonicalJson(value),
)))
)
return null;
quarantine = resolve(parent, prepared.archiveName);
if (rootExists) {
if (archived || candidates.length || existsSync(quarantine))
return null;
sourceDirectory = root;
} else {
if (candidates.length !== 1 || candidates[0] !== prepared.archiveName)
return null;
sourceDirectory = quarantine;
}
sourceArchive = {
intent: prepared,
fromCanonical: rootExists,
completed: !!archived,
};
} else if (rootExists && readdirSync(root).length) {
if (maintenanceHistory.length || candidates.length) return null;
sourceDirectory = root;
quarantine = resolve(
parent,
`${basename(root)}.identity_indeterminate.cleanup.${randomUUID()}`,
);
sourceArchive = {
intent: {},
fromCanonical: true,
completed: false,
};
} else {
// A refused successor may create only the directory before admission
// fails. Inventory that exact empty inode, never replace another owner.
emptyRoot = cleanupCanonicalVacancy(root);
if (candidates.length !== 1) return null;
quarantine = resolve(parent, candidates[0]!);
sourceDirectory = quarantine;
}
const archiveRootIdentity = sourceArchive
? cleanupArchiveRootIdentity(sourceDirectory)
: null;
if (sourceArchive) {
const scopeEntries = readdirSync(runnerdStateBase());
if (
scopeEntries.length > 4096 ||
scopeEntries.some((name) =>
name.startsWith(`${basename(root)}.cleanup-`),
)
)
return null;
}
const source = cleanupStateSnapshot(sourceDirectory);
const identity = record(source.control.identity);
if (
!durableIdentityMatchesExecution(identity, execution) ||
identity.runnerInstanceId !== run.runnerInstanceId ||
![
"runnerInstanceId",
"environmentLeaseId",
"runId",
"normalizedSessionId",
"turnId",
"itemId",
].every(
(key) =>
typeof identity[key] === "string" &&
identity[key] &&
source.runner[key] === identity[key],
) ||
source.runner.lifecycle !== "ready" ||
source.provider.lifecycle !== "turn_active" ||
record(source.provider.config).provider !== "codex" ||
record(source.provider.config).command !== "codex" ||
record(source.provider.config).cwd !== execution.workspace.cwd ||
!Array.isArray(source.control.committedEvents)
)
return null;
const providerEvents = source.control.committedEvents
.map((entry) => record(record(record(entry).envelope).payload))
.filter((event) =>
["session.started", "session.resumed"].includes(
String(event.eventType),
),
);
const providerEvent = providerEvents.at(-1);
const provider = record(providerEvent?.payload);
if (
!providerEvent ||
!validatePrpEvent(providerEvent).ok ||
providerEvent.sourceKind !== "runner" ||
providerEvent.turnId !== identity.turnId ||
providerEvent.itemId !== identity.itemId ||
!cleanupProcessAbsent(provider.processId) ||
typeof provider.providerSessionId !== "string" ||
source.provider.threadId !== provider.providerSessionId ||
providerEvent.runId !== run.id ||
providerEvent.sourceInstanceId !== run.runnerInstanceId ||
providerEvent.normalizedSessionId !== run.nativeSessionId ||
typeof providerEvent.sourceEventId !== "string"
)
return null;
const persisted = await tx
.select()
.from(heartbeatRunEvents)
.where(
and(
eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.companyId, run.companyId),
or(
and(
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId),
inArray(heartbeatRunEvents.eventType, [
"session.started",
"session.resumed",
]),
),
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'sourceKind' = 'control_plane'`,
),
),
)
.limit(20);
if (
persisted.length >= 20 ||
!retainedNativeCleanupJournalMatches({
run,
execution,
accepted: result,
control: source.control,
providerSessionId: provider.providerSessionId,
providerAccountSessionId:
typeof provider.providerAccountSessionId === "string"
? provider.providerAccountSessionId
: null,
persistedEvents: persisted,
})
)
return null;
let copySource: {
directory: string;
snapshot: ReturnType<typeof cleanupStateSnapshot>;
requestId: string;
} | null = null;
const providerHome = cleanupProviderHomeSnapshot(
resolve(sourceDirectory, "codex-home"),
true,
);
if (sourceArchive) {
if (archiveHistory.length) {
if (
sourceArchive.intent.sourceFingerprint !== source.fingerprint ||
sourceArchive.intent.providerHomeFingerprint !==
providerHome.fingerprint ||
canonicalJson(sourceArchive.intent.rootIdentity) !==
canonicalJson(archiveRootIdentity)
)
return null;
} else {
sourceArchive.intent = {
kind: "native_cleanup_source_archive",
version: 1,
phase: "prepared",
requestId: leaseOwner,
companyId: run.companyId,
agentId: run.agentId,
runId: run.id,
nativeSessionId: run.nativeSessionId,
runnerInstanceId: run.runnerInstanceId,
stateKey: basename(root),
archiveName: basename(quarantine),
rootIdentity: archiveRootIdentity,
sourceFingerprint: source.fingerprint,
providerHomeFingerprint: providerHome.fingerprint,
};
}
}
if (maintenanceHistory.length) {
if (
maintenanceHistory.length !== 2 ||
typeof maintenanceHistory[0]?.requestId !== "string"
)
return null;
const priorRequestId = maintenanceHistory[0].requestId;
const scopeEntries = readdirSync(runnerdStateBase());
if (scopeEntries.length > 4096) return null;
const attemptedNames = scopeEntries.filter((name) =>
name.startsWith(`${basename(root)}.cleanup-`),
);
if (attemptedNames.length !== 1) return null;
const directory = resolve(runnerdStateBase(), attemptedNames[0]!);
if (
!retainedRunnerdMaintenanceIsIdle(directory) ||
lstatSync(resolve(directory, CLEANUP_ACTIVATION_FILE), {
throwIfNoEntry: false,
})
)
return null;
const attempted = cleanupStateSnapshot(directory);
const receipts = await tx
.select()
.from(heartbeatRunEvents)
.where(
and(
eq(heartbeatRunEvents.companyId, run.companyId),
eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.eventType, "native.cleanup.event"),
eq(
heartbeatRunEvents.sourceInstanceId,
`${run.runnerInstanceId}:cleanup:${priorRequestId}`,
),
),
)
.limit(513);
if (
receipts.length > 512 ||
!verifyRetainedMaintenanceNoLaunch({
companyId: run.companyId,
agentId: run.agentId,
identity: identity as {
runnerInstanceId: string;
environmentLeaseId: string;
runId: string;
normalizedSessionId: string;
turnId: string;
itemId: string;
},
original: source,
attempted,
requestId: priorRequestId,
requestHistory: coordinator.recoveryHistory,
receipts,
now: new Date(),
})
)
return null;
copySource = {
directory,
snapshot: attempted,
requestId: priorRequestId,
};
}
const history = [
...coordinator.recoveryHistory,
...(sourceArchive
? archiveHistory.length
? []
: [sourceArchive.intent]
: [
{
kind: "native_cleanup_maintenance",
version: 1,
phase: "started",
requestId: leaseOwner,
sourceFingerprint:
copySource?.snapshot.fingerprint ?? source.fingerprint,
...(copySource
? {
originalFingerprint: source.fingerprint,
copiedFromRequestId: copySource.requestId,
copiedFromStagingName: basename(copySource.directory),
}
: {}),
startedAt: new Date().toISOString(),
},
]),
];
await tx
.update(nativeRunFinalizations)
.set({
leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000),
recoveryHistory: history,
updatedAt: new Date(),
})
.where(eq(nativeRunFinalizations.runId, run.id));
return {
execution,
run,
quarantine,
root,
emptyRoot,
source,
providerHome,
sourceArchive,
copySource,
providerPid: provider.processId,
providerSessionId: provider.providerSessionId,
identity: identity as {
runnerInstanceId: string;
environmentLeaseId: string;
runId: string;
normalizedSessionId: string;
turnId: string;
itemId: string;
},
history,
};
});
} catch {
releaseScope();
return { status: "not_eligible", runId: input.runId };
}
if (!claim) {
releaseScope();
return { status: "not_eligible", runId: input.runId };
}
const owned = claim;
let stagingDirectory: string | null = null;
let maintenanceStarted = !owned.sourceArchive;
const archiveReference = owned.sourceArchive
? { sourceArchiveRequestId: owned.sourceArchive.intent.requestId }
: {};
const assertLease = async () => {
const lease = await db
.select({ runId: nativeRunFinalizations.runId })
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.companyId, owned.run.companyId),
eq(nativeRunFinalizations.phase, "committed"),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.limit(1);
if (
!lease.length ||
!reservedScope ||
executingRunnerdSessionScopes.get(reservedScope) !== leaseOwner ||
!cleanupProcessAbsent(owned.run.processPid) ||
!cleanupProcessAbsent(owned.providerPid)
)
throw denied();
};
const authorize = async () => {
await assertLease();
if (
cleanupStateSnapshot(owned.quarantine).fingerprint !==
owned.source.fingerprint ||
cleanupProviderHomeSnapshot(
resolve(owned.quarantine, "codex-home"),
false,
).metadataFingerprint !== owned.providerHome.metadataFingerprint ||
(owned.copySource &&
(!retainedRunnerdMaintenanceIsIdle(owned.copySource.directory) ||
cleanupStateSnapshot(owned.copySource.directory).fingerprint !==
owned.copySource.snapshot.fingerprint ||
lstatSync(
resolve(owned.copySource.directory, CLEANUP_ACTIVATION_FILE),
{ throwIfNoEntry: false },
))) ||
canonicalJson(cleanupCanonicalVacancy(owned.root)) !==
canonicalJson(owned.emptyRoot)
)
throw denied();
};
const appendMaintenanceHistory = async (entry: Record<string, unknown>) => {
const history = await db.transaction(async (tx) => {
const current = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.companyId, owned.run.companyId),
eq(nativeRunFinalizations.phase, "committed"),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
if (!current || current.leaseOwner !== leaseOwner) throw denied();
const next = [
...current.recoveryHistory,
{
...entry,
...(entry.kind === "native_cleanup_source_archive"
? {}
: archiveReference),
},
];
await tx
.update(nativeRunFinalizations)
.set({ recoveryHistory: next, updatedAt: new Date() })
.where(eq(nativeRunFinalizations.runId, owned.run.id));
return next;
});
owned.history = history;
};
try {
if (owned.sourceArchive) {
const archive = owned.sourceArchive;
await assertLease();
const sourceDirectory = archive.fromCanonical
? owned.root
: owned.quarantine;
if (
canonicalJson(cleanupArchiveRootIdentity(sourceDirectory)) !==
canonicalJson(archive.intent.rootIdentity) ||
cleanupStateSnapshot(sourceDirectory).fingerprint !==
owned.source.fingerprint ||
cleanupProviderHomeSnapshot(
resolve(sourceDirectory, "codex-home"),
true,
).fingerprint !== owned.providerHome.fingerprint ||
(archive.fromCanonical
? existsSync(owned.quarantine)
: existsSync(owned.root))
)
throw denied();
if (archive.fromCanonical) {
const parent = dirname(owned.quarantine);
mkdirSync(parent, { recursive: true, mode: 0o700 });
if (!isSafeNativeStateDirectory(parent)) throw denied();
// Preserve the original inode and every byte. In particular, do not
// invoke ordinary quarantine scrubbing on this evidence-only archive.
renameSync(owned.root, owned.quarantine);
archive.fromCanonical = false;
}
const archivedHome = cleanupProviderHomeSnapshot(
resolve(owned.quarantine, "codex-home"),
true,
);
if (
cleanupStateSnapshot(owned.quarantine).fingerprint !==
owned.source.fingerprint ||
archivedHome.fingerprint !== owned.providerHome.fingerprint ||
canonicalJson(cleanupArchiveRootIdentity(owned.quarantine)) !==
canonicalJson(archive.intent.rootIdentity) ||
existsSync(owned.root)
)
throw denied();
owned.providerHome = archivedHome;
if (!archive.completed) {
await appendMaintenanceHistory({
...archive.intent,
phase: "archived",
});
archive.completed = true;
}
await authorize();
await appendMaintenanceHistory({
kind: "native_cleanup_maintenance",
version: 1,
phase: "started",
requestId: leaseOwner,
sourceFingerprint: owned.source.fingerprint,
startedAt: new Date().toISOString(),
});
maintenanceStarted = true;
}
await authorize();
const copy = mkdtempSync(
resolve(runnerdStateBase(), `${basename(owned.root)}.cleanup-`),
);
stagingDirectory = copy;
chmodSync(copy, 0o700);
for (const folder of ["runner", "control-plane"])
mkdirSync(resolve(copy, folder), { mode: 0o700 });
for (const file of CLEANUP_CANONICAL_FILES) {
copyFileSync(
resolve(owned.copySource?.directory ?? owned.quarantine, file),
resolve(copy, file),
constants.COPYFILE_EXCL,
);
chmodSync(resolve(copy, file), 0o600);
}
copyCleanupProviderHome(
resolve(owned.quarantine, "codex-home"),
resolve(copy, "codex-home"),
owned.providerHome,
);
rebaseRetainedNativeCleanupProviderHome(
resolve(copy, "codex-home"),
resolve(owned.root, "codex-home"),
owned.providerSessionId,
"staging",
);
await authorize();
await appendMaintenanceHistory({
kind: "native_cleanup_maintenance",
version: 1,
phase: "staged",
requestId: leaseOwner,
sourceFingerprint:
owned.copySource?.snapshot.fingerprint ?? owned.source.fingerprint,
stagingName: basename(copy),
providerHomeFingerprint: owned.providerHome.fingerprint,
providerHomeBytes: owned.providerHome.bytes,
stagedProviderHomeFingerprint: cleanupProviderHomeSnapshot(
resolve(copy, "codex-home"),
true,
).fingerprint,
});
const proof = await settleRetainedRunnerdSession({
requestId: leaseOwner,
binding: {
companyId: owned.run.companyId,
issueId: owned.run.nativeIssueId!,
agentId: owned.run.agentId,
runId: owned.run.id,
sessionId: owned.run.nativeSessionId!,
},
identity: owned.identity,
backend: { kind: "runner", name: "codex_app_server" },
stateDirectory: copy,
activationDirectory: owned.root,
sourceFingerprint:
owned.copySource?.snapshot.fingerprint ?? owned.source.fingerprint,
providerSessionId: owned.providerSessionId,
originalRunnerPid: owned.run.processPid!,
originalProviderPid: owned.providerPid,
// The control-only provider resume still needs normal host discovery
// and auth-file lookup. Inherit only the existing host allowlist.
environment: buildNativeProviderEnvironment(
{},
process.env,
owned.execution.workspace.cwd ?? undefined,
),
authorize,
recordEpoch: async (receipt) => {
if (receipt.requestId !== leaseOwner || receipt.stateDirectory !== copy)
throw denied();
await appendMaintenanceHistory({
kind: "native_cleanup_runner_epoch",
version: 1,
...receipt,
});
},
appendEvent: async (event) => {
await appendRetainedNativeCleanupEvent(db, {
companyId: owned.run.companyId,
agentId: owned.run.agentId,
runId: owned.run.id,
nativeSessionId: owned.run.nativeSessionId!,
runnerInstanceId: owned.run.runnerInstanceId!,
requestId: leaseOwner,
event,
});
},
});
// Commit intent before the filesystem handoff. A crash can then be
// distinguished from an unattempted quarantine; never replay its source.
if (
cleanupProviderHomeSnapshot(resolve(owned.quarantine, "codex-home"), true)
.fingerprint !== owned.providerHome.fingerprint
)
throw denied();
rebaseRetainedNativeCleanupProviderHome(
resolve(copy, "codex-home"),
resolve(owned.root, "codex-home"),
owned.providerSessionId,
"canonical",
);
const settledHome = cleanupProviderHomeSnapshot(
resolve(copy, "codex-home"),
true,
);
const emptyRootArchive = owned.emptyRoot
? `${basename(owned.root)}.empty-before-cleanup.${leaseOwner}`
: null;
const preparedHistory = [
...owned.history,
{
kind: "native_cleanup_maintenance",
version: 1,
phase: "activation_prepared",
...archiveReference,
requestId: leaseOwner,
sourceFingerprint: proof.sourceFingerprint,
settledFingerprint: proof.settledFingerprint,
settledProviderHomeFingerprint: settledHome.fingerprint,
stagingName: basename(copy),
...(emptyRootArchive
? { emptyRootArchive, emptyRoot: owned.emptyRoot }
: {}),
nativeSessionId: owned.run.nativeSessionId,
runnerInstanceId: owned.run.runnerInstanceId,
providerSessionId: owned.providerSessionId,
},
];
writeFileSync(
resolve(copy, CLEANUP_ACTIVATION_FILE),
JSON.stringify({
schema: "paperclip.native_cleanup_activation.v1",
companyId: owned.run.companyId,
issueId: owned.run.nativeIssueId,
runId: owned.run.id,
requestId: leaseOwner,
sourceFingerprint: proof.sourceFingerprint,
settledFingerprint: proof.settledFingerprint,
settledProviderHomeFingerprint: settledHome.fingerprint,
}),
{ flag: "wx", mode: 0o600 },
);
const prepared = await db
.update(nativeRunFinalizations)
.set({ recoveryHistory: preparedHistory, updatedAt: new Date() })
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.returning({ runId: nativeRunFinalizations.runId });
if (!prepared.length) throw denied();
owned.history = preparedHistory;
await db.transaction(async (tx) => {
const current = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
if (!current || current.phase !== "committed") throw denied();
await authorize();
if (
cleanupProviderHomeSnapshot(resolve(copy, "codex-home"), false)
.metadataFingerprint !== settledHome.metadataFingerprint
)
throw denied();
if (emptyRootArchive) {
const archive = resolve(runnerdStateBase(), emptyRootArchive);
if (lstatSync(archive, { throwIfNoEntry: false })) throw denied();
// Preserve even an empty predecessor directory as evidence. The
// scope reservation and durable prepared intent cover this handoff.
renameSync(owned.root, archive);
}
renameSync(copy, owned.root);
await tx
.update(nativeRunFinalizations)
.set({
leaseOwner: null,
leaseExpiresAt: null,
recoveryHistory: [
...owned.history,
{
kind: "native_cleanup_maintenance",
version: 1,
phase: "settled",
...archiveReference,
requestId: leaseOwner,
sourceFingerprint: proof.sourceFingerprint,
settledFingerprint: proof.settledFingerprint,
settledProviderHomeFingerprint: settledHome.fingerprint,
nativeSessionId: owned.run.nativeSessionId,
runnerInstanceId: owned.run.runnerInstanceId,
providerSessionId: owned.providerSessionId,
settledAt: new Date().toISOString(),
},
],
updatedAt: new Date(),
})
.where(eq(nativeRunFinalizations.runId, owned.run.id));
});
if (
cleanupProviderHomeSnapshot(resolve(owned.root, "codex-home"), true)
.fingerprint !== settledHome.fingerprint
)
throw denied();
completeRetainedNativeSessionCleanup(proof);
rmSync(resolve(owned.root, CLEANUP_ACTIVATION_FILE));
return { status: "settled", runId: input.runId };
} catch {
if (stagingDirectory && existsSync(stagingDirectory)) {
// Keep the failed journal, not transient copied provider credentials.
try {
scrubRunnerdQuarantineLaunchState(stagingDirectory);
} catch {
/* retain fail-closed ownership */
}
}
await db.transaction(async (tx) => {
const current = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
if (!current || current.leaseOwner !== leaseOwner) return;
await tx
.update(nativeRunFinalizations)
.set({
leaseOwner: null,
leaseExpiresAt: null,
// A timed-out epoch callback may have committed before this lock.
// Preserve its evidence rather than replacing it from a stale copy.
recoveryHistory: [
...current.recoveryHistory,
{
kind: maintenanceStarted
? "native_cleanup_maintenance"
: "native_cleanup_source_archive",
version: 1,
phase: "operator_required",
requestId: leaseOwner,
...archiveReference,
code: "native_cleanup_maintenance_unproven",
},
],
updatedAt: new Date(),
})
.where(
and(
eq(nativeRunFinalizations.runId, owned.run.id),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
),
);
});
return { status: "operator_required", runId: input.runId };
} finally {
releaseScope();
}
}
/** Read-only scoped admission fence; never grants provider or recovery authority. */
export async function assertRetainedNativeSourceArchiveSettled(
db: Db,
input: { companyId: string; issueId: string; stateKey: string },
): Promise<void> {
if (!/^[a-f0-9]{64}$/.test(input.stateKey))
throw new NativeSessionCleanupQuarantinedError();
const owners = await db
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.companyId, input.companyId),
eq(nativeRunFinalizations.issueId, input.issueId),
sql`exists (
select 1 from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory}) prepared
where prepared->>'kind' = 'native_cleanup_source_archive'
and prepared->>'phase' = 'prepared'
and jsonb_typeof(prepared->'stateKey') = 'string'
and prepared->>'stateKey' = ${input.stateKey}
and not coalesce(${nativeRunFinalizations.phase} = 'committed'
and (select count(*) from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory}) duplicate
where duplicate->>'kind' = 'native_cleanup_source_archive'
and duplicate->>'phase' = 'prepared'
and duplicate->'stateKey' = prepared->'stateKey') = 1
and prepared->'version' = '1'::jsonb
and jsonb_typeof(prepared->'requestId') = 'string'
and prepared->>'requestId' like 'native-cleanup:%'
and jsonb_typeof(prepared->'sourceFingerprint') = 'string'
and prepared->>'sourceFingerprint' ~ '^[a-f0-9]{64}$'
and (
select settled.value->>'phase' = 'settled'
and jsonb_typeof(settled.value->'sourceArchiveRequestId') = 'string'
and jsonb_typeof(settled.value->'sourceFingerprint') = 'string'
and settled.value->>'sourceFingerprint' = prepared->>'sourceFingerprint'
from jsonb_array_elements(${nativeRunFinalizations.recoveryHistory})
with ordinality as settled(value, position)
where settled.value->>'kind' = 'native_cleanup_maintenance'
and settled.value->>'sourceArchiveRequestId' = prepared->>'requestId'
order by settled.position desc limit 1
), false)
)`,
),
)
.limit(1);
for (const owner of owners) {
const history = Array.isArray(owner.recoveryHistory)
? owner.recoveryHistory
: [];
const intents = history.filter(
(entry) =>
entry.kind === "native_cleanup_source_archive" &&
entry.phase === "prepared" &&
entry.stateKey === input.stateKey,
);
if (!intents.length) continue;
const intent = intents[0]!;
const settlement = history.findLast(
(entry) =>
entry.kind === "native_cleanup_maintenance" &&
entry.sourceArchiveRequestId === intent.requestId,
);
if (
intents.length !== 1 ||
owner.phase !== "committed" ||
intent.version !== 1 ||
typeof intent.requestId !== "string" ||
!intent.requestId.startsWith("native-cleanup:") ||
typeof intent.sourceFingerprint !== "string" ||
!/^[a-f0-9]{64}$/.test(intent.sourceFingerprint) ||
settlement?.phase !== "settled" ||
settlement.sourceFingerprint !== intent.sourceFingerprint
)
throw new NativeSessionCleanupQuarantinedError();
}
}
async function assertCleanupActivationCommitted(
db: Db,
root: string,
execution: NativeExecutionInput,
): Promise<void> {
const path = resolve(root, CLEANUP_ACTIVATION_FILE);
if (!lstatSync(path, { throwIfNoEntry: false })) return;
const marker = record(
JSON.parse(
readBoundedNativeFile(
path,
4096,
"native_cleanup_maintenance_unproven",
).toString("utf8"),
),
);
if (
marker.schema !== "paperclip.native_cleanup_activation.v1" ||
marker.companyId !== execution.binding.companyId ||
marker.issueId !== execution.binding.issueId ||
typeof marker.runId !== "string" ||
typeof marker.requestId !== "string"
) {
throw new NativeSessionCleanupQuarantinedError();
}
const coordinator = await db
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, marker.runId),
eq(nativeRunFinalizations.companyId, execution.binding.companyId),
eq(nativeRunFinalizations.issueId, execution.binding.issueId),
),
)
.limit(1)
.then((rows) => rows[0]);
const receipt = coordinator?.recoveryHistory.findLast(
(entry) =>
entry.kind === "native_cleanup_maintenance" &&
entry.requestId === marker.requestId,
);
const snapshot = cleanupStateSnapshot(root);
if (
marker.settledProviderHomeFingerprint !== undefined ||
receipt?.settledProviderHomeFingerprint !== undefined
) {
if (
typeof marker.settledProviderHomeFingerprint !== "string" ||
!/^[0-9a-f]{64}$/.test(marker.settledProviderHomeFingerprint) ||
receipt?.settledProviderHomeFingerprint !==
marker.settledProviderHomeFingerprint ||
cleanupProviderHomeSnapshot(resolve(root, "codex-home"), true)
.fingerprint !== marker.settledProviderHomeFingerprint
)
throw new NativeSessionCleanupQuarantinedError();
}
if (
coordinator?.phase !== "committed" ||
receipt?.phase !== "settled" ||
receipt.sourceFingerprint !== marker.sourceFingerprint ||
receipt.settledFingerprint !== marker.settledFingerprint ||
snapshot.fingerprint !== marker.settledFingerprint ||
snapshot.runner.runId !== marker.runId ||
snapshot.runner.lifecycle !== "suspended" ||
snapshot.provider.lifecycle !== "prepared" ||
!Array.isArray(snapshot.control.committedEvents)
)
throw new NativeSessionCleanupQuarantinedError();
const owners = snapshot.control.committedEvents
.map((entry) => record(record(record(entry).envelope).payload))
.filter((event) =>
["session.started", "session.resumed"].includes(String(event.eventType)),
)
.map((event) => record(event.payload).processId);
if (!owners.length || !owners.every(cleanupProcessAbsent))
throw new NativeSessionCleanupQuarantinedError();
// A committed receipt survived a crash after rename. Normal admission now
// applies its existing exact old-owner/session fences; no process is started here.
rmSync(path);
}
/** Read-only admission evidence for an explicit retry of a terminal failed
* run. Never migrate/archive state, release an owner, or contact a provider.
* Normal executor admission independently verifies the state again. */
export function nativeFailedRunRetryStateIsSafe(input: {
execution: unknown;
companyId: string;
issueId: string;
agentId: string;
runId: string;
nativeSessionId: string;
runnerInstanceId: string;
providerSessionId: string | null;
providerBackendSessionId: string | null;
processPid: number | null;
processGroupId: number | null;
recoveryMode: "bootstrap_retry" | "exact_checkpoint_resume";
allowVerifiedBackup: boolean;
}): boolean {
try {
const execution = parseNativeExecutionInput(input.execution);
if (
execution.binding.companyId !== input.companyId ||
execution.binding.issueId !== input.issueId ||
execution.binding.agentId !== input.agentId ||
execution.binding.runId !== input.runId ||
nativeSessionKey(execution) !== input.nativeSessionId ||
!input.runnerInstanceId
)
return false;
const scope = nativeSessionScopeKey(execution);
if (
executingRunnerdSessionScopes.has(scope) ||
initializingSessionToolAuthorities.has(scope) ||
warmNativeSessions.has(scope)
)
return false;
for (const [id, group] of [
[input.processPid, false],
[input.processGroupId, true],
] as const) {
if (id === null) continue;
if (!Number.isSafeInteger(id) || id <= 0) return false;
try {
process.kill(group ? -id : id, 0);
return false;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") return false;
}
}
const root = scopedRunnerdStateRoot(execution);
if (!lstatSync(root, { throwIfNoEntry: false })) {
if (
input.recoveryMode !== "bootstrap_retry" ||
lstatSync(legacyRunnerdStateRoot(execution), {
throwIfNoEntry: false,
}) ||
lstatSync(legacyCompanyRunnerdStateRoot(execution), {
throwIfNoEntry: false,
})
)
return false;
// A missing root after quarantine is not proof of a clean bootstrap.
const quarantine = resolve(runnerdStateBase(), "quarantine");
if (lstatSync(quarantine, { throwIfNoEntry: false })) {
if (!isSafeNativeStateDirectory(quarantine)) return false;
const directory = opendirSync(quarantine);
try {
const prefixes = [
root,
legacyRunnerdStateRoot(execution),
legacyCompanyRunnerdStateRoot(execution),
].map((path) => `${basename(path)}.`);
for (let count = 0; ; count++) {
const entry = directory.readSync();
if (!entry) break;
if (
count >= 4096 ||
prefixes.some((prefix) => entry.name.startsWith(prefix))
)
return false;
}
} finally {
directory.closeSync();
}
}
return true;
}
const identity = readRunnerdDurableIdentity(root);
if (
!durableIdentityMatchesSession(identity, execution) ||
!durableIdentityMatchesExecution(identity, execution) ||
identity.runnerInstanceId !== input.runnerInstanceId
)
return false;
let stateRoot = root;
const direct = runnerdAuthorityLifecycle(root, identity);
if (direct === "absent" && input.allowVerifiedBackup) {
const backup = verifyNativeHarnessBackup({
root,
execution,
runnerInstanceId: input.runnerInstanceId,
});
if (!backup) return false;
stateRoot = backup.root;
}
if (runnerdAuthorityLifecycle(stateRoot, identity) !== "suspended")
return false;
const runnerState = record(
JSON.parse(
readBoundedNativeFile(
resolve(stateRoot, "runner", "runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES,
"runner_state_too_large",
).toString("utf8"),
),
);
if (
runnerState.schema !== RUNNERD_STATE_SCHEMA ||
runnerState.lifecycle !== "suspended" ||
runnerState.runId !== identity.runId ||
runnerState.runnerInstanceId !== identity.runnerInstanceId ||
runnerState.normalizedSessionId !== identity.normalizedSessionId ||
runnerState.environmentLeaseId !== identity.environmentLeaseId ||
!Array.isArray(runnerState.outbox) ||
runnerState.outbox.length !== 0
)
return false;
const providerFile = resolve(
stateRoot,
"runner",
runnerProviderStateFilename(execution),
);
if (input.recoveryMode === "bootstrap_retry") return false;
const providerState = record(
JSON.parse(
readBoundedNativeFile(
providerFile,
NATIVE_RUNNER_STATE_MAX_BYTES,
"runner_provider_state_too_large",
).toString("utf8"),
),
);
if (
!Array.isArray(providerState.pendingEvents) ||
providerState.pendingEvents.length !== 0 ||
!Array.isArray(providerState.queuedEvents) ||
providerState.queuedEvents.length !== 0 ||
Object.keys(record(record(providerState.toolBridge).pending)).length !==
0 ||
providerState.activeProviderResultFingerprint != null
)
return false;
const providerIdentity = providerSessionIdentityFromDurableProviderState({
execution,
providerState,
});
return (
!!input.providerSessionId &&
providerIdentity.providerSessionId === input.providerSessionId &&
providerIdentity.providerBackendSessionId ===
input.providerBackendSessionId
);
} catch {
return false;
}
}
/** Physical half of retrying a request rejected before provider admission.
* The caller separately proves the failed receipt/coordinator has no native
* events or result and selects exactly one committed cleanup owner. */
export function nativePreProviderRetryAfterCleanupStateIsSafe(input: {
failedExecution: unknown;
retiredExecution: unknown;
companyId: string;
issueId: string;
agentId: string;
failedRunId: string;
retiredRunId: string;
nativeSessionId: string;
runnerInstanceId: string;
providerSessionId: string;
providerBackendSessionId: string | null;
processPid: number;
processGroupId: number;
receipt: Record<string, unknown>;
}): boolean {
try {
const failed = parseNativeExecutionInput(input.failedExecution);
const retired = parseNativeExecutionInput(input.retiredExecution);
if (
input.failedRunId === input.retiredRunId ||
failed.binding.runId !== input.failedRunId ||
retired.binding.runId !== input.retiredRunId ||
failed.binding.companyId !== input.companyId ||
failed.binding.issueId !== input.issueId ||
failed.binding.agentId !== input.agentId ||
nativeSessionScopeKey(failed) !== nativeSessionScopeKey(retired) ||
input.receipt.kind !== "native_cleanup_maintenance" ||
input.receipt.version !== 1 ||
input.receipt.phase !== "settled" ||
input.receipt.nativeSessionId !== input.nativeSessionId ||
input.receipt.runnerInstanceId !== input.runnerInstanceId ||
input.receipt.providerSessionId !== input.providerSessionId ||
typeof input.receipt.requestId !== "string" ||
input.receipt.requestId.length === 0 ||
typeof input.receipt.sourceFingerprint !== "string" ||
!/^[a-f0-9]{64}$/.test(input.receipt.sourceFingerprint) ||
typeof input.receipt.settledFingerprint !== "string" ||
!/^[a-f0-9]{64}$/.test(input.receipt.settledFingerprint) ||
!nativeFailedRunRetryStateIsSafe({
execution: retired,
companyId: input.companyId,
issueId: input.issueId,
agentId: input.agentId,
runId: input.retiredRunId,
nativeSessionId: input.nativeSessionId,
runnerInstanceId: input.runnerInstanceId,
providerSessionId: input.providerSessionId,
processPid: input.processPid,
processGroupId: input.processGroupId,
providerBackendSessionId: input.providerBackendSessionId,
recoveryMode: "exact_checkpoint_resume",
allowVerifiedBackup: false,
})
)
return false;
const root = scopedRunnerdStateRoot(retired);
const markerPath = resolve(root, CLEANUP_ACTIVATION_FILE);
if (lstatSync(markerPath, { throwIfNoEntry: false })) {
// A crash after receipt commit may leave the activation marker. This
// read-only proof accepts only that exact committed handoff; normal
// executor admission independently reconciles the marker before launch.
const marker = record(
JSON.parse(
readBoundedNativeFile(
markerPath,
4096,
"native_cleanup_maintenance_unproven",
).toString("utf8"),
),
);
if (
marker.schema !== "paperclip.native_cleanup_activation.v1" ||
marker.companyId !== input.companyId ||
marker.issueId !== input.issueId ||
marker.runId !== input.retiredRunId ||
marker.requestId !== input.receipt.requestId ||
marker.sourceFingerprint !== input.receipt.sourceFingerprint ||
marker.settledFingerprint !== input.receipt.settledFingerprint
)
return false;
}
return (
cleanupStateSnapshot(root).fingerprint ===
input.receipt.settledFingerprint
);
} catch {
return false;
}
}
export async function verifyPriorRunnerdStateForSessionScope(input: {
db: Db;
root: string;
identity: RunnerdDurableIdentity;
execution: NativeExecutionInput;
allowVerifiedBackup: boolean;
allowRetainedWarmRunner: boolean;
remoteRunnerState?: boolean;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}): Promise<PriorRunnerdStateVerification> {
let priorRun: {
status: string;
runnerProfileJson: unknown;
} | null;
try {
priorRun = await input.db
.select({
status: heartbeatRuns.status,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, input.identity.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
} catch {
return "unavailable";
}
if (!priorRun) return "authority_indeterminate";
if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(priorRun.status)) {
return "active";
}
const persistedInput = record(
priorRun.runnerProfileJson,
).nativeExecutionInput;
if (persistedInput === undefined) return "authority_indeterminate";
try {
const priorExecution = parseNativeExecutionInput(persistedInput);
const sameScope =
priorExecution.binding.runId === input.identity.runId &&
nativeSessionScopeKey(priorExecution) ===
nativeSessionScopeKey(input.execution);
if (!sameScope) return "scope_mismatch";
const lifecycle = runnerdAuthorityLifecycleWithVerifiedBackup(input);
if (lifecycle === "suspended") return "verified";
const directLifecycle = runnerdAuthorityLifecycle(
input.root,
input.identity,
);
if (
input.allowRetainedWarmRunner &&
(lifecycle === "not_suspended" ||
// A remote runner keeps runner-state.json in the sandbox rather than
// beside the controller's PRP journal. During a genuinely warm handoff
// there is intentionally no suspended failover backup yet. The exact
// idle in-memory session owner is the authority for this one case;
// after a restart that owner is absent and this remains fail-closed.
((input.allowVerifiedBackup || input.remoteRunnerState) && directLifecycle === "absent"))
) {
return "retained_warm_runner";
}
const target = input.runnerExecutionTarget;
if (input.remoteRunnerState && directLifecycle === "absent" &&
target?.kind === "remote" && target.transport === "computer" && target.environmentId) {
const descriptor = record(priorRun.runnerProfileJson).nativeComputerWorkspace;
const priorWorkspace = readNativeComputerWorkspaceReference(descriptor);
if (!priorWorkspace || priorWorkspace.computerOwner.computerId !== target.resourceAuthority.computerId ||
priorWorkspace.remoteCwd !== target.remoteCwd ||
record(descriptor).placementId !== target.fileAuthority.placementId) return "scope_mismatch";
// Persistent runners keep their state remotely. Once the exact prior
// owner has completed retirement, its terminal run and fenced process
// are authoritative even though no local runner-state.json exists.
// This read cannot retire a live owner or accept a newer generation.
try {
// PRP historically calls the immutable execution-workspace coordinate
// environmentLeaseId. Resolve its recorded placement lease separately
// from the latest process owner: workspace identity survives warm turns
// and idle replacement. Explicit physical leases retain their own lineage.
let workspaceRunId: string | null = input.identity.environmentLeaseId ===
priorExecution.binding.executionWorkspaceId ? input.identity.runId : null;
let leaseWorkspace = priorWorkspace;
let physicalLeaseId = workspaceRunId ? priorWorkspace.leaseId : input.identity.environmentLeaseId;
if (!workspaceRunId) {
// Warm attachment advances runId while retaining the process's original
// workspace coordinate. Resolve that recorded origin, never an arbitrary
// run ID, before checking the physical lease and latest retirement.
const origins = await input.db.select({
id: heartbeatRuns.id,
status: heartbeatRuns.status,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
eq(sql<string>`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->>'executionWorkspaceId'`,
input.identity.environmentLeaseId),
)).limit(2);
if (origins.length > 1) return "scope_mismatch";
if (origins.length === 1) {
const origin = origins[0]!;
const originProfile = record(origin.runnerProfileJson);
const originExecution = parseNativeExecutionInput(originProfile.nativeExecutionInput);
const originWorkspace = readNativeComputerWorkspaceReference(originProfile.nativeComputerWorkspace);
if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(origin.status) ||
originExecution.binding.runId !== origin.id ||
originExecution.binding.executionWorkspaceId !== input.identity.environmentLeaseId ||
nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(priorExecution) ||
!originWorkspace || originWorkspace.remoteCwd !== priorWorkspace.remoteCwd ||
record(originProfile.nativeComputerWorkspace).placementId !== record(descriptor).placementId ||
originWorkspace.computerOwner.computerId !== priorWorkspace.computerOwner.computerId ||
(originWorkspace.computerOwner.ownerId === priorWorkspace.computerOwner.ownerId &&
originWorkspace.computerOwner.generation > priorWorkspace.computerOwner.generation)) return "scope_mismatch";
// This coordinate outlives physical runner replacements. Its lease
// proves the recorded workspace origin, not ownership of the latest
// process. The latest exact generation is verified independently below.
workspaceRunId = originExecution.binding.runId;
leaseWorkspace = originWorkspace;
physicalLeaseId = originWorkspace.leaseId;
}
}
const [physicalLease] = await input.db.select({
providerLeaseId: environmentLeases.providerLeaseId,
heartbeatRunId: environmentLeases.heartbeatRunId,
metadata: environmentLeases.metadata,
}).from(environmentLeases).where(and(
eq(environmentLeases.id, physicalLeaseId),
eq(environmentLeases.companyId, input.execution.binding.companyId),
eq(environmentLeases.environmentId, target.environmentId),
)).limit(1);
const leaseMetadata = record(physicalLease?.metadata);
const leaseOwner = record(leaseMetadata.computerOwner);
if ((workspaceRunId && physicalLease?.heartbeatRunId !== workspaceRunId) ||
physicalLease?.providerLeaseId !== leaseWorkspace.computerOwner.ownerId ||
leaseMetadata.agentId !== input.execution.binding.agentId ||
leaseOwner.computerId !== leaseWorkspace.computerOwner.computerId ||
leaseOwner.ownerId !== leaseWorkspace.computerOwner.ownerId ||
!Number.isSafeInteger(leaseOwner.generation) || Number(leaseOwner.generation) < 1 ||
(workspaceRunId
? Number(leaseOwner.generation) !== leaseWorkspace.computerOwner.generation
: Number(leaseOwner.generation) > leaseWorkspace.computerOwner.generation)) return "scope_mismatch";
if (await computerService(input.db).isRetired({
companyId: input.execution.binding.companyId,
environmentId: target.environmentId,
agentId: input.execution.binding.agentId,
runId: input.identity.runId,
owner: priorWorkspace.computerOwner,
})) return "verified";
} catch {
return "unavailable";
}
}
return "terminal_state_indeterminate";
} catch {
return "authority_indeterminate";
}
}
function hasRetainedWarmTransitionEvidence(root: string): boolean {
for (const [directory, filename, maximum] of [
[
"control-plane",
"control-plane-state.json",
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES,
],
["runner", "runner-state.json", NATIVE_RUNNER_STATE_MAX_BYTES],
] as const) {
const path = resolve(root, directory, filename);
if (!lstatSync(path, { throwIfNoEntry: false })) continue;
let bytes: string;
try {
bytes = readBoundedNativeFile(
path,
maximum,
"runner_state_too_large",
).toString("utf8");
} catch {
// The ordinary verifier still owns unreadable legacy state. Inspect the
// other file before deciding whether this is a forward-protocol fence.
continue;
}
try {
const state = record(JSON.parse(bytes));
if (
Object.prototype.hasOwnProperty.call(state, "warmTransition") ||
state.schema ===
"paperclip.runner.durable.control-plane-state.warm-transition.v1" ||
state.schema === "paperclip.runner.durable.state.warm-transition.v1"
)
return true;
} catch {
// This is detection only, never admission. A damaged forward receipt
// must remain available to its owner rather than become legacy state.
if (
bytes.includes("warm-transition.v1") ||
bytes.includes('"warmTransition"')
)
return true;
}
}
return false;
}
type VerifiedWarmTransitionBinding = {
runnerInstanceId: string;
environmentLeaseId: string;
transitionId: string;
stateFingerprint: string;
};
function readWarmTransitionSnapshot(root: string) {
if (
![root, resolve(root, "control-plane"), resolve(root, "runner")].every(
isSafeNativeStateDirectory,
)
) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
const core = readBoundedNativeFile(
resolve(root, "control-plane", "control-plane-state.json"),
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES,
"native_runner_warm_transition_recovery_unproven",
);
const runner = readBoundedNativeFile(
resolve(root, "runner", "runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES,
"native_runner_warm_transition_recovery_unproven",
);
try {
return {
controlPlaneState: JSON.parse(core.toString("utf8")) as unknown,
runnerState: JSON.parse(runner.toString("utf8")) as unknown,
stateFingerprint: nativeSha256([
core.toString("base64"),
runner.toString("base64"),
]),
};
} catch {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
}
/** Forward receipts select a protocol boundary, never a process owner. */
async function verifyWarmTransitionRestart(input: {
db: Db;
execution: NativeExecutionInput;
restartRecovery?: NativeRestartRecoveryClaim;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}): Promise<VerifiedWarmTransitionBinding> {
const deny = (): never => {
throw new Error("native_runner_warm_transition_recovery_unproven");
};
const claim = input.restartRecovery;
// A surviving runner needs its own authenticated reattach path. Do not
// reinterpret that claim, remote evidence, or an incomplete bootstrap as
// permission to launch another process against a retained transition.
if (
claim?.kind !== "resume_dead_runner" ||
input.runnerExecutionTarget?.kind === "remote" ||
input.execution.provider.kind !== "codex" ||
input.execution.session.driverKind !== "codex_app_server" ||
claim.runId !== input.execution.binding.runId
)
return deny();
const root = scopedRunnerdStateRoot(input.execution);
const snapshot = readWarmTransitionSnapshot(root);
const artifact = readRunnerdArtifactBinding(resolvePaperclipRunnerBinary());
const controller = await currentNativeControllerIdentity();
const binding = input.execution.binding;
let finalAuthorityCheck: (() => boolean) | undefined;
const verified = await input.db.transaction(async (tx) => {
const current = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, binding.runId),
eq(heartbeatRuns.companyId, binding.companyId),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
const coordinator = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, binding.runId),
eq(nativeRunFinalizations.companyId, binding.companyId),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
const issue = await tx
.select()
.from(issues)
.where(
and(
eq(issues.id, binding.issueId),
eq(issues.companyId, binding.companyId),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
const now = new Date();
if (
!current ||
!coordinator ||
!issue ||
issue.executionRunId !== binding.runId ||
current.agentId !== binding.agentId ||
current.nativeIssueId !== binding.issueId ||
current.runtimeMode !== "native" ||
current.status !== "running" ||
current.finishedAt !== null ||
current.nativeSessionId !== nativeSessionKey(input.execution) ||
!current.runnerInstanceId ||
isNativeRunnerOwnershipHeld(current) ||
coordinator.issueId !== binding.issueId ||
coordinator.phase !== "observed" ||
coordinator.resultId !== null ||
coordinator.leaseOwner !== claim.leaseOwner ||
!coordinator.leaseExpiresAt ||
coordinator.leaseExpiresAt <= now ||
coordinator.controllerBootId !== controller.bootId ||
coordinator.controllerPid !== controller.pid ||
coordinator.controllerProcessStartedAt?.getTime() !==
controller.processStartedAt.getTime() ||
coordinator.controllerGeneration !== claim.controllerGeneration
)
return deny();
const profile = record(current.runnerProfileJson);
let frozen: NativeExecutionInput;
try {
frozen = parseNativeExecutionInput(profile.nativeExecutionInput);
} catch {
return deny();
}
if (nativeSha256(frozen) !== nativeSha256(input.execution)) return deny();
const cancellation = record(record(current.resultJson).nativeCancellation);
if (
cancellation.scope === "run" &&
["pending", "acknowledged"].includes(String(cancellation.dispatchState))
)
return deny();
let expectedEnvironmentLeaseId = binding.executionWorkspaceId;
if (binding.executionWorkspaceId === binding.runId) {
// Projectless runs retain the first run's lease while their workspace
// placeholder changes. The receipt only SELECTS that origin: its frozen
// DB execution, full scope and terminal ownership independently prove it.
const selectedLease = record(
record(record(snapshot.runnerState).warmTransition).receipt,
).newIdentity;
const originId = record(selectedLease).environmentLeaseId;
if (
typeof originId !== "string" ||
!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
originId,
)
)
return deny();
const origin = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, originId),
eq(heartbeatRuns.companyId, binding.companyId),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
if (
!origin ||
origin.agentId !== binding.agentId ||
origin.nativeIssueId !== binding.issueId ||
origin.runtimeMode !== "native" ||
!["succeeded", "failed", "cancelled", "timed_out"].includes(
origin.status,
) ||
origin.finishedAt === null ||
isNativeRunnerOwnershipHeld(origin) ||
origin.runnerInstanceId !== current.runnerInstanceId ||
origin.nativeSessionId !== current.nativeSessionId
)
return deny();
let originExecution: NativeExecutionInput;
try {
originExecution = parseNativeExecutionInput(
record(origin.runnerProfileJson).nativeExecutionInput,
);
} catch {
return deny();
}
if (
originExecution.binding.runId !== origin.id ||
originExecution.binding.executionWorkspaceId !== origin.id ||
originExecution.binding.companyId !== binding.companyId ||
originExecution.binding.agentId !== binding.agentId ||
originExecution.binding.issueId !== binding.issueId ||
nativeSessionScopeKey(originExecution) !==
nativeSessionScopeKey(input.execution)
)
return deny();
expectedEnvironmentLeaseId = origin.id;
}
const inspectionInput = {
...snapshot,
expectedNewIdentity: {
runnerInstanceId: current.runnerInstanceId,
environmentLeaseId: expectedEnvironmentLeaseId,
runId: binding.runId,
normalizedSessionId: nativeSessionKey(input.execution),
turnId: `turn-${binding.runId}`,
itemId: `item-${binding.runId}`,
},
expectedRunnerVersion: artifact.version,
expectedRunnerDigest: artifact.digest,
};
const proof = inspectWarmRunTransition({
...inspectionInput,
now: Date.now(),
});
if (!proof) return deny();
const prior = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, proof.receipt.oldIdentity.runId),
eq(heartbeatRuns.companyId, binding.companyId),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0]);
if (
!prior ||
prior.agentId !== binding.agentId ||
prior.nativeIssueId !== binding.issueId ||
prior.runtimeMode !== "native" ||
!["succeeded", "failed", "cancelled", "timed_out"].includes(
prior.status,
) ||
prior.finishedAt === null ||
isNativeRunnerOwnershipHeld(prior) ||
prior.runnerInstanceId !== current.runnerInstanceId ||
prior.nativeSessionId !== current.nativeSessionId ||
proof.receipt.oldIdentity.runnerInstanceId !== prior.runnerInstanceId ||
proof.receipt.oldIdentity.normalizedSessionId !== prior.nativeSessionId ||
proof.receipt.oldIdentity.environmentLeaseId !==
expectedEnvironmentLeaseId ||
proof.receipt.oldIdentity.turnId !== `turn-${prior.id}` ||
proof.receipt.oldIdentity.itemId !== `item-${prior.id}`
)
return deny();
let previousExecution: NativeExecutionInput;
try {
previousExecution = parseNativeExecutionInput(
record(prior.runnerProfileJson).nativeExecutionInput,
);
} catch {
return deny();
}
if (
previousExecution.binding.runId !== prior.id ||
previousExecution.binding.companyId !== binding.companyId ||
previousExecution.binding.agentId !== binding.agentId ||
previousExecution.binding.issueId !== binding.issueId ||
nativeSessionScopeKey(previousExecution) !==
nativeSessionScopeKey(input.execution)
)
return deny();
const history = coordinator.recoveryHistory.at(-1);
const checkpoint = record(profile.sessionCheckpoint);
const checkpointIdentity = record(checkpoint.identity);
const processEvidence = record(checkpoint.process);
if (
!history ||
history.disposition !== claim.kind ||
history.controllerBootId !== controller.bootId ||
history.controllerPid !== controller.pid ||
history.controllerGeneration !== claim.controllerGeneration ||
history.recoveryRequestId !== claim.recoveryRequestId ||
history.providerAttempt !== claim.providerAttempt ||
history.stateRootAction !== "reuse_exact_root" ||
history.hasCheckpoint !== true ||
history.checkpointIdentityMatches !== true ||
history.hasProviderEvidence !== true ||
checkpointIdentity.runId !== binding.runId ||
checkpointIdentity.companyId !== binding.companyId ||
checkpointIdentity.issueId !== binding.issueId ||
checkpointIdentity.agentId !== binding.agentId ||
checkpointIdentity.sessionId !== current.nativeSessionId ||
history.processPid !== prior.processPid ||
history.processStartedAt !== prior.processStartedAt?.toISOString() ||
processEvidence.runnerPid !== prior.processPid ||
processEvidence.runnerProcessGroupId !== prior.processGroupId ||
prior.processGroupId !== prior.processPid ||
!cleanupProcessAbsent(prior.processPid)
)
return deny();
// Recheck every independently retained provider owner, including process
// groups. These are DB/checkpoint facts, not claims in a copied journal.
const known = history.knownProviderPids;
if (
!Array.isArray(known) ||
known.length === 0 ||
known.length > 16 ||
![
history.liveProviderPids,
history.ambiguousProviderPids,
history.recycledProviderPids,
].every((pids) => Array.isArray(pids) && pids.length === 0)
)
return deny();
const checkpointPids = new Set<number>();
for (const [pidKey, startedKey] of [
["providerPid", "providerProcessStartedAt"],
["codexPid", "codexProcessStartedAt"],
["sidecarPid", "sidecarProcessStartedAt"],
["agentPid", "agentProcessStartedAt"],
] as const) {
const pid = processEvidence[pidKey];
if (pid === null || pid === undefined) continue;
if (
typeof processEvidence[startedKey] !== "string" ||
!Number.isFinite(Date.parse(processEvidence[startedKey] as string)) ||
!cleanupProcessAbsent(pid)
)
return deny();
checkpointPids.add(pid);
}
if (
checkpointPids.size === 0 ||
new Set(known).size !== checkpointPids.size ||
known.some(
(pid) =>
typeof pid !== "number" ||
!checkpointPids.has(pid) ||
!cleanupProcessAbsent(pid),
)
)
return deny();
// These fences remain relevant if maintenance evidence appears after
// initial construction. Every later registration/launch/auth gate repeats
// them; no settled receipt or pending transition bypasses quarantine.
await assertRetainedNativeSourceArchiveSettled(tx as unknown as Db, {
companyId: binding.companyId,
issueId: binding.issueId,
stateKey: basename(root),
});
await assertCleanupActivationCommitted(
tx as unknown as Db,
root,
input.execution,
);
finalAuthorityCheck = () => {
if (
readWarmTransitionSnapshot(root).stateFingerprint !==
snapshot.stateFingerprint
)
return false;
const selected = readRunnerdArtifactBinding(
resolvePaperclipRunnerBinary(),
);
if (
selected.version !== artifact.version ||
selected.digest !== artifact.digest
)
return false;
const finalProof = inspectWarmRunTransition({
...inspectionInput,
now: Date.now(),
});
return (
coordinator.leaseExpiresAt!.getTime() > Date.now() &&
finalProof?.receipt.transitionId === proof.receipt.transitionId &&
finalProof.receipt.leaseExpiresAtUnixMs > Date.now()
);
};
if (!finalAuthorityCheck()) return deny();
return {
runnerInstanceId: current.runnerInstanceId,
environmentLeaseId: expectedEnvironmentLeaseId,
transitionId: proof.receipt.transitionId,
stateFingerprint: snapshot.stateFingerprint,
};
});
// A lock wait or the transaction completion itself can outlive a lease.
// Never carry a timestamp sampled before those awaits into admission.
if (!finalAuthorityCheck?.()) return deny();
return verified;
}
async function migrateRunnerdStateRootForExecution(input: {
db: Db;
execution: NativeExecutionInput;
allowVerifiedBackup: boolean;
allowRetainedWarmRunner: boolean;
allowLocalRecovery: boolean;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
restartRecovery?: NativeRestartRecoveryClaim;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}): Promise<VerifiedWarmTransitionBinding | undefined> {
const scoped = scopedRunnerdStateRoot(input.execution);
// A crash can leave a source archive intent before/after its rename. Until
// that exact maintenance is settled, absence of a canonical root is not
// permission to bootstrap a replacement provider session.
await assertRetainedNativeSourceArchiveSettled(input.db, {
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
stateKey: basename(scoped),
});
if (
input.allowLocalRecovery &&
!input.allowRetainedWarmRunner &&
!input.restartRecovery
) {
await recoverQuiescentRunnerdState({ ...input, scoped });
}
if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) {
throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing");
}
if (existsSync(scoped)) {
if (!isSafeNativeStateDirectory(scoped)) {
throw new Error("runner_state_directory_unsafe");
}
if (hasRetainedWarmTransitionEvidence(scoped)) {
// A copied receipt is not restart/process authority. Keep both valid
// unsupported and invalid forward evidence out of destructive legacy
// migration until the exact transition admission path proves its owner.
const verified = await verifyWarmTransitionRestart(input);
return verified;
}
await assertCleanupActivationCommitted(input.db, scoped, input.execution);
const identity = readRunnerdDurableIdentity(scoped);
if (!identity) {
if (input.restartRecovery?.kind !== "reattach_existing_runner" &&
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
throw new Error("runner_state_identity_mismatch: durable_identity_unreadable");
}
if (!durableIdentityMatchesSession(identity, input.execution)) {
if (input.restartRecovery?.kind !== "reattach_existing_runner" &&
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_mismatch");
}
throw new Error("runner_state_identity_mismatch: session_scope_mismatch");
}
if (input.restartRecovery?.kind === "bootstrap_incomplete") {
if (runnerdStateProvesIncompleteBootstrap(scoped)) {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
return;
}
// Database evidence alone cannot distinguish a never-connected runner
// from a partially-persisted provider bootstrap. Only the durable PRP
// root can authorize a fresh bootstrap; anything else stays fail-closed.
throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete");
}
if (input.restartRecovery?.kind === "reattach_remote_runner") {
await verifyRemoteRunnerReattachment({
claim: input.restartRecovery, target: input.runnerExecutionTarget, identity,
runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution),
});
return;
}
if (durableIdentityMatchesExecution(identity, input.execution)) {
if (
runnerdAuthorityLifecycleWithVerifiedBackup({
root: scoped,
identity,
execution: input.execution,
allowVerifiedBackup: input.allowVerifiedBackup,
}) === "indeterminate"
) {
if (input.restartRecovery?.kind !== "reattach_existing_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
throw new Error("runner_state_identity_mismatch: authority_indeterminate");
}
} else {
const verification = await verifyPriorRunnerdStateForSessionScope({
db: input.db,
root: scoped,
identity,
execution: input.execution,
allowVerifiedBackup: input.allowVerifiedBackup,
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" &&
input.runnerExecutionTarget.transport === "computer",
runnerExecutionTarget: input.runnerExecutionTarget,
});
if (
verification !== "verified" &&
verification !== "retained_warm_runner"
) {
if (verification !== "active" && verification !== "unavailable") {
quarantineRunnerdStateRoot(
scoped,
verification === "scope_mismatch"
? "identity_mismatch"
: "identity_indeterminate",
);
}
throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`);
}
}
return;
}
for (const legacy of [
legacyCompanyRunnerdStateRoot(input.execution),
legacyRunnerdStateRoot(input.execution),
]) {
if (!existsSync(legacy)) continue;
const identity = readRunnerdDurableIdentity(legacy);
if (
!identity ||
!durableIdentityMatchesSession(identity, input.execution)
) {
// Unlike the full-scope target above, this legacy name can legitimately
// belong to another scope. Leave it in place for its owner and fail the
// attempted migration visibly.
throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch");
}
let verifiedPriorRunId: string | undefined;
if (!durableIdentityMatchesExecution(identity, input.execution)) {
const verification = await verifyPriorRunnerdStateForSessionScope({
db: input.db,
root: legacy,
identity,
execution: input.execution,
allowVerifiedBackup: input.allowVerifiedBackup,
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" &&
input.runnerExecutionTarget.transport === "computer",
runnerExecutionTarget: input.runnerExecutionTarget,
});
if (
verification !== "verified" &&
verification !== "retained_warm_runner"
) {
if (verification === "terminal_state_indeterminate") {
// The database proves this ambiguous legacy path belongs to the same
// full session scope and its owner is terminal, so it is now safe to
// move aside. Scope mismatches and active/unavailable owners remain
// untouched because the legacy name may still belong to them.
quarantineRunnerdStateRoot(legacy, "identity_indeterminate");
}
throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`);
}
verifiedPriorRunId = identity.runId;
}
migrateLegacyRunnerdStateRoot({
legacy,
scoped,
execution: input.execution,
...(verifiedPriorRunId ? { verifiedPriorRunId } : {}),
});
return;
}
}
// A terminal warm run can lose its controller before session.suspend. Older
// controllers quarantined even fully settled state in that case. Recover the
// exact provider thread, never bootstrap a replacement from an ambiguous root.
async function recoverQuiescentRunnerdState(input: {
db: Db;
execution: NativeExecutionInput;
scoped: string;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
}): Promise<void> {
if (input.execution.provider.kind !== "codex") return;
const quarantine = resolve(runnerdStateBase(), "quarantine");
const scopedExists = existsSync(input.scoped);
if (scopedExists && !isSafeNativeStateDirectory(input.scoped)) return;
// Never hide corrupt or contradictory current authority behind an older
// checkpoint. Only an absent/empty root may recover from quarantine.
const currentEmpty = !scopedExists || readdirSync(input.scoped).length === 0;
const candidates = currentEmpty
? isSafeNativeStateDirectory(quarantine)
? readdirSync(quarantine)
.filter((name) =>
name.startsWith(
`${basename(input.scoped)}.identity_indeterminate.`,
),
)
.map((name) => resolve(quarantine, name))
: []
: [input.scoped];
if (
currentEmpty &&
candidates.filter(
(root) =>
isSafeNativeStateDirectory(root) && readdirSync(root).length > 0,
).length > 1
) {
// Do not roll back to an older valid checkpoint when a newer quarantined
// root contains unconfirmed work, even if the newer root is unreadable.
throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous");
}
const verified: Array<{
root: string;
runnerBytes: string;
controlBytes: string;
providerBytes: string;
runner: Record<string, unknown>;
runId: string;
processPid: number;
processGroupId: number;
}> = [];
for (const root of candidates) {
if (!isSafeNativeStateDirectory(root)) continue;
const identity = readRunnerdDurableIdentity(root);
if (!durableIdentityMatchesSession(identity, input.execution)) continue;
if (identity.runId === input.execution.binding.runId) continue;
try {
const readState = (directory: string, name: string) => {
if (!isSafeNativeStateDirectory(resolve(root, directory)))
throw new Error("unsafe_recovery_state");
return readBoundedNativeFile(
resolve(root, directory, name),
directory === "control-plane" && name === "control-plane-state.json"
? NATIVE_CONTROL_PLANE_STATE_MAX_BYTES
: NATIVE_RUNNER_STATE_MAX_BYTES,
"recovery_state_too_large",
).toString("utf8");
};
const runnerBytes = readState("runner", "runner-state.json");
const controlBytes = readState(
"control-plane",
"control-plane-state.json",
);
const providerBytes = readState("runner", "codex-provider-state.json");
const runner = record(JSON.parse(runnerBytes));
const control = record(JSON.parse(controlBytes));
const provider = record(JSON.parse(providerBytes));
const commands = control.commands;
const events = control.committedEvents;
const terminalEnvelope = record(
record(Array.isArray(events) ? events.at(-1) : null).envelope,
);
if (
runner.schema !== RUNNERD_STATE_SCHEMA ||
!["ready", "suspended"].includes(String(runner.lifecycle)) ||
(!currentEmpty && runner.lifecycle === "suspended") ||
[
"runId",
"normalizedSessionId",
"runnerInstanceId",
"environmentLeaseId",
"turnId",
"itemId",
].some((key) => runner[key] !== identity[key]) ||
!Array.isArray(runner.outbox) ||
runner.outbox.length !== 0 ||
runner.pendingTerminalDelivery !== null ||
!Array.isArray(commands) ||
commands.some((command) => record(command).status !== "completed") ||
!Array.isArray(events) ||
record(events.at(-1)).eventType !== "run.terminal" ||
terminalEnvelope.runId !== identity.runId ||
terminalEnvelope.normalizedSessionId !== identity.normalizedSessionId ||
provider.completedTurnAuthoritative !== true
)
continue;
const providerIdentity = providerSessionIdentityFromDurableProviderState({
execution: input.execution,
providerState: provider,
});
if (!providerIdentity.providerSessionId) continue;
const prior = await input.db
.select({
status: heartbeatRuns.status,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRuns.processGroupId,
contextSnapshot: heartbeatRuns.contextSnapshot,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, identity.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!prior || !TERMINAL_HEARTBEAT_RUN_STATUSES.has(prior.status))
continue;
const profile = record(prior.runnerProfileJson);
const previous = parseNativeExecutionInput(profile.nativeExecutionInput);
const checkpoint = record(profile.sessionCheckpoint);
const checkpointIdentity = record(checkpoint.identity);
const current = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, input.execution.binding.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
const latestCheckpoint = record(
record(current?.runnerProfileJson).sessionCheckpoint,
);
const latestIdentity = record(latestCheckpoint.identity);
if (
previous.binding.runId !== identity.runId ||
previous.binding.issueId !== input.execution.binding.issueId ||
nativeSessionScopeKey(previous) !==
nativeSessionScopeKey(input.execution) ||
nativeSessionConfigDigest(previous) !==
nativeSessionConfigDigest(input.execution) ||
record(record(prior.contextSnapshot).paperclipEnvironment).driver !==
"local" ||
checkpointIdentity.runId !== identity.runId ||
checkpointIdentity.sessionId !== identity.normalizedSessionId ||
checkpointIdentity.companyId !== previous.binding.companyId ||
checkpointIdentity.agentId !== previous.binding.agentId ||
checkpointIdentity.issueId !== previous.binding.issueId ||
checkpoint.driverKind !== previous.session.driverKind ||
checkpoint.providerSessionId !== providerIdentity.providerSessionId ||
latestCheckpoint.providerSessionId !==
providerIdentity.providerSessionId ||
latestIdentity.sessionId !== identity.normalizedSessionId ||
latestIdentity.companyId !== previous.binding.companyId ||
latestIdentity.agentId !== previous.binding.agentId ||
latestIdentity.issueId !== previous.binding.issueId ||
checkpoint.activeTurnId !== null ||
!Array.isArray(checkpoint.pendingRuntimeRequests) ||
checkpoint.pendingRuntimeRequests.length !== 0 ||
!localProcessDefinitelyGone(prior.processPid) ||
!localProcessDefinitelyGone(prior.processGroupId, true)
)
continue;
// Provider processes are normally inside the runner group. Check any
// independently recorded child identities too, including detached ones.
const processIds = [
...Object.entries(record(checkpoint.process))
.filter(([key]) => /^(provider|codex|sidecar|agent)Pid$/.test(key))
.map(([, value]) => value),
...events
.map(
(event) => record(record(record(event).envelope).payload).payload,
)
.map((payload) => record(payload).processId)
.filter((pid) => pid !== undefined),
];
if (processIds.some((pid) => !localProcessDefinitelyGone(pid))) continue;
// No mutation if any evidence changed while the database was read.
if (
runnerBytes !== readState("runner", "runner-state.json") ||
controlBytes !==
readState("control-plane", "control-plane-state.json") ||
providerBytes !== readState("runner", "codex-provider-state.json")
)
continue;
verified.push({
root,
runnerBytes,
controlBytes,
providerBytes,
runner,
runId: identity.runId,
processPid: prior.processPid!,
processGroupId: prior.processGroupId!,
});
} catch {
// Unreadable, unsafe, unscoped, or unprovable state stays quarantined.
}
}
if (verified.length !== 1) {
if (
currentEmpty &&
candidates.some(
(root) =>
isSafeNativeStateDirectory(root) && readdirSync(root).length > 0,
)
) {
// Known provider history is not permission to start a replacement when
// recovery cannot prove a unique, settled owner.
throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified");
}
return;
}
const candidate = verified[0]!;
// Candidate enumeration can await other database reads. Revalidate the
// selected evidence and dead owner immediately before the atomic moves.
for (const [relativePath, expected, maxBytes] of [
["runner/runner-state.json", candidate.runnerBytes, NATIVE_RUNNER_STATE_MAX_BYTES],
["runner/codex-provider-state.json", candidate.providerBytes, NATIVE_RUNNER_STATE_MAX_BYTES],
["control-plane/control-plane-state.json", candidate.controlBytes, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES],
] as const) {
if (
readBoundedNativeFile(
resolve(candidate.root, relativePath),
maxBytes,
"recovery_state_too_large",
).toString("utf8") !== expected
) {
throw new Error("runner_state_identity_mismatch: recovery_evidence_changed");
}
}
if (
!localProcessDefinitelyGone(candidate.processPid) ||
!localProcessDefinitelyGone(candidate.processGroupId, true)
) {
throw new Error("runner_state_identity_mismatch: recovery_process_not_gone");
}
if (candidate.root !== input.scoped) {
if (existsSync(input.scoped)) {
if (
!isSafeNativeStateDirectory(input.scoped) ||
readdirSync(input.scoped).length !== 0
)
return;
quarantineRunnerdStateRoot(input.scoped, "identity_indeterminate");
}
renameSync(candidate.root, input.scoped);
}
// The provider and its process group are gone and all work is settled. Seal
// this old authority so the standard epoch rotation archives it before the
// next run, retaining the provider's history and exact thread identity.
const statePath = resolve(input.scoped, "runner", "runner-state.json");
const temporary = `${statePath}.${randomUUID()}.tmp`;
writeFileSync(
temporary,
JSON.stringify({ ...candidate.runner, lifecycle: "suspended" }),
{ encoding: "utf8", mode: 0o600, flag: "wx" },
);
renameSync(temporary, statePath);
await input.onLog?.(
"stdout",
`[paperclip-runner] Automatically recovered settled session from run ${candidate.runId}; preserving the existing provider thread.\n`,
);
}
function localProcessDefinitelyGone(value: unknown, group = false): boolean {
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 1)
return false;
try {
process.kill(group ? -value : value, 0);
return false;
} catch (error) {
return (error as NodeJS.ErrnoException).code === "ESRCH";
}
}
export function runnerdStateProvesIncompleteBootstrap(root: string): boolean {
try {
const statePath = resolve(
root,
"control-plane",
"control-plane-state.json",
);
const state = record(
JSON.parse(
readBoundedNativeFile(
statePath,
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES,
"runner_durable_identity_too_large",
).toString("utf8"),
),
);
const commands = Array.isArray(state.commands)
? state.commands.map(record)
: [];
const committedEvents = Array.isArray(state.committedEvents)
? state.committedEvents
: [];
const onlyUnconsumedBootstrapCommands = commands.every(
(command) =>
command.status === "pending" &&
(command.type === "run.prepare" || command.type === "session.open"),
);
return (
state.schema === RUNNERD_CONTROL_PLANE_STATE_SCHEMA &&
state.connectionCount === 0 &&
committedEvents.length === 0 &&
onlyUnconsumedBootstrapCommands
);
} catch {
return false;
}
}
function isSafeNativeStateDirectory(path: string): boolean {
if (!existsSync(path)) return false;
const stats = lstatSync(path);
return stats.isDirectory() && !stats.isSymbolicLink();
}
type RunnerdDurableIdentity = Record<string, unknown> & {
runId: string;
normalizedSessionId: string;
runnerInstanceId: string;
environmentLeaseId: string;
};
function readRunnerdDurableIdentity(
root: string,
): Record<string, unknown> | null {
if (!isSafeNativeStateDirectory(root)) return null;
const controlPlaneRoot = resolve(root, "control-plane");
if (!isSafeNativeStateDirectory(controlPlaneRoot)) return null;
const statePath = resolve(controlPlaneRoot, "control-plane-state.json");
if (!existsSync(statePath)) return null;
try {
const state = record(
JSON.parse(
readBoundedNativeFile(
statePath,
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES,
"runner_durable_identity_too_large",
).toString("utf8"),
),
);
if (state.schema !== RUNNERD_CONTROL_PLANE_STATE_SCHEMA) return null;
return record(state.identity);
} catch {
return null;
}
}
function durableIdentityMatchesExecution(
identity: Record<string, unknown> | null,
execution: NativeExecutionInput,
): boolean {
return Boolean(
identity &&
identity.runId === execution.binding.runId &&
durableIdentityMatchesSession(identity, execution),
);
}
function durableIdentityMatchesSession(
identity: Record<string, unknown> | null,
execution: NativeExecutionInput,
): identity is RunnerdDurableIdentity {
return Boolean(
identity &&
identity.normalizedSessionId === nativeSessionKey(execution) &&
typeof identity.runId === "string" &&
identity.runId.length > 0 &&
typeof identity.runnerInstanceId === "string" &&
identity.runnerInstanceId.length > 0 &&
typeof identity.environmentLeaseId === "string" &&
identity.environmentLeaseId.length > 0,
);
}
/**
* Pre-v2 state is migrated for an exact active run, or for a suspended prior
* run whose persisted, validated execution input proves the same full native
* session scope. Ambiguous company/session-only state can never be claimed by
* another agent, workspace, or provider profile.
*/
function runnerdStateRoot(execution: NativeExecutionInput): string {
const scoped = scopedRunnerdStateRoot(execution);
if (existsSync(scoped)) {
if (!isSafeNativeStateDirectory(scoped)) {
throw new Error("runner_state_directory_unsafe");
}
return scoped;
}
for (const legacy of [
legacyCompanyRunnerdStateRoot(execution),
legacyRunnerdStateRoot(execution),
]) {
const migrated = migrateLegacyRunnerdStateRoot({
legacy,
scoped,
execution,
});
if (migrated) return migrated;
}
return scoped;
}
function loadRunnerdDurableBinding(
execution: NativeExecutionInput,
transition?: VerifiedWarmTransitionBinding,
): {
runnerInstanceId: string;
environmentLeaseId: string;
} | null {
if (transition) {
if (
readWarmTransitionSnapshot(runnerdStateRoot(execution))
.stateFingerprint !== transition.stateFingerprint
) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
return transition;
}
const identity = readRunnerdDurableIdentity(runnerdStateRoot(execution));
// The run id is intentionally different during a continuation. Reuse only
// the verified runner/lease binding from the same company-scoped durable
// session root; rotateLocalAuthorityEpoch then requires that exact binding
// before it can archive the prior per-run authority.
if (!durableIdentityMatchesSession(identity, execution)) return null;
return {
runnerInstanceId: identity.runnerInstanceId,
environmentLeaseId: identity.environmentLeaseId,
};
}
function nativeSessionConfigDigest(
execution: NativeExecutionInput,
executionTargetKind: "local" | "remote" = "local",
legacyProjectlessRunId?: string,
): string {
const executionLocation = {
executionKind: "local_process",
// Use the same workspace identity as the durable session scope. For a
// projectless task, executionWorkspaceId is a per-run placeholder, not a
// workspace change. Real workspace/provider/policy changes still fence
// retained processes and checkpoints through the rest of this digest.
workspaceId:
execution.binding.executionWorkspaceId === execution.binding.runId
? (legacyProjectlessRunId ?? nativeSessionWorkspaceScope(execution))
: execution.binding.executionWorkspaceId,
cwd: execution.workspace.cwd,
};
return `sha256:${createHash("sha256")
.update(
JSON.stringify({
companyId: execution.binding.companyId,
...(execution.binding.agentKeyId ? { agentKeyId: execution.binding.agentKeyId } : {}),
normalizedSessionId: nativeSessionKey(execution),
executionLocation,
provider: execution.provider,
driverKind: execution.session.driverKind,
lifecyclePolicy: execution.session.lifecyclePolicy,
workMode: execution.task.workMode,
executionMode:
"executionMode" in execution ? execution.executionMode : "default",
runtimeContextDigest:
"runtimeContext" in execution
? execution.runtimeContext.aggregateDigest
: null,
// Provider threads retain their tool declarations across resume. Bump
// this revision whenever the server-authorized native tool surface
// changes so an older thread is rotated instead of falsely resuming
// without newly required tools.
nativeToolContractFingerprint:
nativeToolContractFingerprintForTarget(executionTargetKind),
runtimeContract: nativeRuntimeContractForProvider(execution.provider),
}),
)
.digest("hex")}`;
}
function hasIdleWarmNativeSessionOwner(input: {
execution: NativeExecutionInput;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}): boolean {
if (input.execution.session.lifecyclePolicy.mode !== "warm") return false;
const entry = warmNativeSessions.get(nativeSessionScopeKey(input.execution));
if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return false;
// A verified idle owner proves the checkpoint belongs to this session even
// when its process must later rotate to a new run-scoped broker capability.
// Local environments also have an id. Compare the same environment binding
// stored at acquisition instead of treating every local target as unbound.
const environmentId = input.runnerExecutionTarget?.environmentId ?? null;
return (
entry.companyId === input.execution.binding.companyId &&
entry.environmentId === environmentId &&
entry.configDigest ===
nativeSessionConfigDigest(
input.execution,
input.runnerExecutionTarget?.kind ?? "local",
)
);
}
function nativeHarnessEnvironmentFingerprint(
execution: NativeExecutionInput,
): string {
return `sha256:${createHash("sha256")
.update(
canonicalJson({
companyId: execution.binding.companyId,
...(execution.binding.agentKeyId ? { agentKeyId: execution.binding.agentKeyId } : {}),
agentId: execution.binding.agentId,
issueId: execution.binding.issueId,
normalizedSessionId: nativeSessionKey(execution),
workspace: {
cwd: execution.workspace.cwd,
repoUrl: execution.workspace.repoUrl,
repoRef: execution.workspace.repoRef,
branchName: execution.workspace.branchName,
},
provider: execution.provider,
driverKind: execution.session.driverKind,
}),
)
.digest("hex")}`;
}
type NativeProviderKind = NativeExecutionInput["provider"]["kind"];
type NativeDriverKind = NativeExecutionInput["session"]["driverKind"];
export interface NativeHarnessPersistenceDirectory {
name: "runner" | "codex-home" | "opencode" | "acpx";
location: "runner" | "filesystem";
excludeEntries: readonly string[];
}
export interface NativeHarnessPersistenceProfile {
providerKind: NativeProviderKind;
driverKind: NativeDriverKind;
directories: readonly NativeHarnessPersistenceDirectory[];
}
export interface NativeHarnessBackupManifest {
schema: "paperclip.native-harness-backup.v1";
normalizedSessionId: string;
runnerInstanceId: string;
providerKind: NativeProviderKind;
driverKind: NativeDriverKind;
providerSessionIdentity: unknown;
sourceProviderLeaseId: string;
environmentFingerprint: string;
runnerContractVersion: number;
directories: Array<{
name: string;
sha256: string;
bytes: number;
}>;
completedAt: string;
}
export function resolveNativeHarnessPersistenceProfile(
execution: NativeExecutionInput,
): NativeHarnessPersistenceProfile {
const providerDirectory: NativeHarnessPersistenceDirectory | null =
execution.provider.kind === "codex"
? {
name: "codex-home",
location: "filesystem",
// Codex session history lives below this home, but these files are
// launch-time material: auth.json is copied from the configured
// credential source and config.toml can contain the native MCP
// bearer token. Re-materialize both for a replacement sandbox
// instead of putting credentials into the disaster-recovery copy.
excludeEntries: CODEX_HOME_NON_PERSISTENT_ENTRIES,
}
: execution.provider.kind === "opencode"
? {
name: "opencode",
location: "filesystem",
excludeEntries: [],
}
: execution.provider.kind === "acpx"
? {
name: "acpx",
location: "filesystem",
// ACPX stores each provider beneath a stable session directory.
// Codex creates process-local executable aliases in tmp/arg0;
// they may point outside the runtime tree and are neither safe
// nor necessary to restore. Credentials and launch-time config
// are also re-materialized in the replacement sandbox. Grok
// diagnostics can contain auth fields and are not session state.
excludeEntries:
execution.provider.agent === "grok"
? ["auth.json", "auth-refresh.json", "auth-refresh.json.tmp", "config.toml", "logs"].map((entry) =>
`acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/grok-home/${entry}`)
: execution.provider.agent === "codex"
? CODEX_HOME_NON_PERSISTENT_ENTRIES.map(
(entry) =>
`acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/codex-home/${entry}`,
)
: [],
}
: null;
return {
providerKind: execution.provider.kind,
driverKind: execution.session.driverKind,
directories: [
{ name: "runner", location: "runner", excludeEntries: [] },
...(providerDirectory ? [providerDirectory] : []),
],
};
}
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
if (value && typeof value === "object") {
const entries = Object.entries(value as Record<string, unknown>).sort(
([left], [right]) => left.localeCompare(right),
);
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
}
return JSON.stringify(value) ?? "null";
}
function runnerProviderStateFilename(execution: NativeExecutionInput): string {
switch (execution.provider.kind) {
case "openai_dot":
return "dot-provider-state.json";
case "codex":
case "opencode":
return "codex-provider-state.json";
case "acpx":
return "acpx-provider-state.json";
case "claude_managed":
case "aws_agentcore":
return "managed-provider-state.json";
}
}
/**
* The v2 runner owns PRP identity/lifecycle in runner-state.json and keeps
* provider recovery identity in a sibling provider state file. Never infer a
* resumable provider from the outer PRP journal alone.
*/
export function providerSessionIdentityFromDurableProviderState(input: {
execution: NativeExecutionInput;
providerState: unknown;
}): Record<string, unknown> {
const state = record(input.providerState);
const expectedSessionId = nativeSessionKey(input.execution);
const nonEmptyString = (value: unknown): value is string =>
typeof value === "string" && value.trim().length > 0;
const sha256 = (value: unknown): value is string =>
typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
const emptyIdentity = () => ({
providerSessionId: null,
providerBackendSessionId: null,
providerSessionIdentity: null,
});
switch (input.execution.provider.kind) {
case "openai_dot":
// Dot checkpoints recover the bridge only; there is no native provider thread.
return emptyIdentity();
case "acpx": {
const descriptor = record(state.descriptor);
const identity = record(state.identity);
const expectedModel = input.execution.provider.model;
const requiredIdentityFields = [
"kind",
"normalizedSessionId",
"acpxRecordId",
"backendSessionId",
"agentSessionId",
"profileDigest",
"workspaceDigest",
"requestedModel",
"effectiveModel",
] as const;
if (
state.schema !== ACPX_PROVIDER_STATE_SCHEMA ||
state.lifecycle !== "suspended" ||
state.providerExitUnconfirmed !== false ||
state.activeTurnId !== null ||
descriptor.kind !== "acpx" ||
descriptor.provider !== "acpx" ||
descriptor.driver !== "acpx_runtime" ||
descriptor.agent !== input.execution.provider.agent ||
descriptor.model !== expectedModel ||
descriptor.normalizedSessionId !== expectedSessionId ||
identity.kind !== "acpx" ||
identity.normalizedSessionId !== expectedSessionId ||
requiredIdentityFields.some(
(field) => !nonEmptyString(identity[field]),
) ||
!sha256(identity.profileDigest) ||
!sha256(identity.workspaceDigest) ||
identity.profileDigest !== descriptor.commandDigest ||
identity.requestedModel !== expectedModel ||
identity.effectiveModel !== expectedModel ||
identity.permissionMode !== input.execution.provider.permissionMode ||
!acpxRecoveryModeMatches(input.execution.provider, descriptor.mode, identity.mode) ||
!acpxRecoveryPiThinkingMatches(input.execution.provider, descriptor.piThinkingLevel, identity.piThinkingLevel) ||
!["approve-all", "approve-paperclip", "approve-reads", "deny-all"].includes(
String(identity.permissionMode),
) ||
!Array.isArray(identity.providerLifetimeFenceCandidates) ||
identity.providerLifetimeFenceCandidates.length !== 3 ||
new Set(identity.providerLifetimeFenceCandidates).size !== 3 ||
identity.providerLifetimeFenceCandidates.some(
(port) =>
!Number.isInteger(port) ||
Number(port) < 49_152 ||
Number(port) > 65_535,
)
) {
return emptyIdentity();
}
return {
providerSessionId: identity.acpxRecordId ?? null,
providerBackendSessionId: identity.backendSessionId ?? null,
providerSessionIdentity: structuredClone(identity),
};
}
case "codex":
case "opencode": {
const config = record(state.config);
const expectedDriver =
input.execution.provider.kind === "codex"
? "codex_app_server"
: "opencode_server";
if (
state.schema !== CODEX_PROVIDER_STATE_SCHEMA ||
!["prepared", "session_open", "provider_exited"].includes(
String(state.lifecycle),
) ||
!nonEmptyString(state.threadId) ||
(state.providerSessionId !== null &&
state.providerSessionId !== undefined &&
!nonEmptyString(state.providerSessionId)) ||
state.activeProviderTurnId !== null ||
state.ambiguousTurnStartPending === true ||
config.provider !== input.execution.provider.kind ||
config.driver !== expectedDriver
) {
return emptyIdentity();
}
return {
providerSessionId: state.threadId ?? null,
providerBackendSessionId: state.providerSessionId ?? null,
providerSessionIdentity: null,
};
}
case "claude_managed":
case "aws_agentcore": {
const descriptor = record(state.descriptor);
if (
state.schema !== MANAGED_PROVIDER_STATE_SCHEMA ||
state.lifecycle !== "suspended" ||
state.normalizedSessionId !== expectedSessionId ||
descriptor.kind !== input.execution.provider.kind ||
!nonEmptyString(state.providerSessionId) ||
state.activeTurnId !== null
) {
return emptyIdentity();
}
return {
providerSessionId: state.providerSessionId ?? null,
providerBackendSessionId: state.providerSessionId ?? null,
providerSessionIdentity: null,
};
}
}
}
function providerSessionIdentityIsPresent(value: unknown): boolean {
const identity = record(value);
return (
(identity.providerSessionId !== null &&
identity.providerSessionId !== undefined) ||
(identity.providerBackendSessionId !== null &&
identity.providerBackendSessionId !== undefined) ||
(identity.providerSessionIdentity !== null &&
identity.providerSessionIdentity !== undefined)
);
}
/** Closed diagnostic categories; durable state may contain private provider data. */
export function durableProviderCheckpointFailureReason(execution: NativeExecutionInput, providerState: unknown): string {
if (execution.provider.kind !== "acpx") return "identity_binding";
const state = record(providerState);
if (state.schema !== ACPX_PROVIDER_STATE_SCHEMA) return "state_schema";
if (state.lifecycle !== "suspended") return "provider_not_suspended";
if (state.providerExitUnconfirmed !== false) return "provider_exit_unconfirmed";
if (state.activeTurnId !== null) return "provider_turn_unsettled";
if (!acpxRecoveryModeMatches(execution.provider, record(state.descriptor).mode, record(state.identity).mode)) return "mode_binding";
return "identity_binding";
}
// Recovery consumes observed identities; it must never apply the fresh-config
// default to a missing persisted mode. Provider capability checks belong to admission.
function acpxRecoveryModeMatches(
provider: NativeExecutionInput["provider"],
...observedModes: unknown[]
): boolean {
const expected = record(provider).mode;
return (expected === undefined || isProviderMode(expected))
&& observedModes.every(mode => mode === expected);
}
function acpxRecoveryPiThinkingMatches(provider: NativeExecutionInput["provider"], ...observed: unknown[]): boolean {
const expected = record(provider).piThinkingLevel;
if (provider.kind === "acpx" && provider.agent === "pi") return ["off", "low", "high", "max"].includes(String(expected)) && observed.every(value => value === expected);
return expected === undefined && observed.every(value => value === undefined);
}
export function providerSessionIdentityTransitionIsAllowed(input: {
execution: NativeExecutionInput;
previous: unknown;
current: unknown;
}): boolean {
if (input.execution.provider.kind === "acpx" && !acpxRecoveryModeMatches(
input.execution.provider,
record(record(input.previous).providerSessionIdentity).mode,
record(record(input.current).providerSessionIdentity).mode,
)) return false;
if (!acpxRecoveryPiThinkingMatches(input.execution.provider, record(record(input.previous).providerSessionIdentity).piThinkingLevel, record(record(input.current).providerSessionIdentity).piThinkingLevel)) return false;
if (canonicalJson(input.previous) === canonicalJson(input.current)) {
return true;
}
if (
input.execution.provider.kind !== "acpx" ||
input.execution.interactionResponses.length === 0
) {
return false;
}
const previousOuter = record(input.previous);
const currentOuter = record(input.current);
const previous = record(previousOuter.providerSessionIdentity);
const current = record(currentOuter.providerSessionIdentity);
if (previous.kind !== "acpx" || current.kind !== "acpx") return false;
const stableFields = [
"normalizedSessionId",
"profileDigest",
"workspaceDigest",
"requestedModel",
"effectiveModel",
"permissionMode",
] as const;
if (
current.normalizedSessionId !== nativeSessionKey(input.execution) ||
stableFields.some(
(field) =>
typeof previous[field] !== "string" ||
previous[field] !== current[field],
)
) {
return false;
}
return (
typeof previous.acpxRecordId === "string" &&
typeof previous.backendSessionId === "string" &&
typeof previous.agentSessionId === "string" &&
typeof current.acpxRecordId === "string" &&
typeof current.backendSessionId === "string" &&
typeof current.agentSessionId === "string" &&
previousOuter.providerSessionId === previous.acpxRecordId &&
previousOuter.providerBackendSessionId === previous.backendSessionId &&
currentOuter.providerSessionId === current.acpxRecordId &&
currentOuter.providerBackendSessionId === current.backendSessionId
);
}
function digestBackupDirectory(directory: string): {
sha256: string;
bytes: number;
} {
const hash = createHash("sha256");
let bytes = 0;
const visit = (current: string, relative: string) => {
const entries = readdirSync(current, { withFileTypes: true }).sort(
(left, right) => left.name.localeCompare(right.name),
);
if (entries.length === 0) hash.update(`directory:${relative}\0`);
for (const entry of entries) {
const entryPath = resolve(current, entry.name);
const entryRelative = relative ? `${relative}/${entry.name}` : entry.name;
const stats = lstatSync(entryPath);
if (entry.isDirectory()) {
hash.update(`directory:${entryRelative}:${stats.mode & 0o777}\0`);
visit(entryPath, entryRelative);
} else if (entry.isSymbolicLink()) {
hash.update(`symlink:${entryRelative}:${readlinkSync(entryPath)}\0`);
} else if (entry.isFile()) {
const contents = readFileSync(entryPath);
bytes += contents.byteLength;
hash.update(
`file:${entryRelative}:${stats.mode & 0o777}:${contents.byteLength}\0`,
);
hash.update(contents);
} else {
throw new Error(
`runner_harness_backup_unsupported_entry:${entryRelative}`,
);
}
}
};
visit(directory, "");
return { sha256: `sha256:${hash.digest("hex")}`, bytes };
}
function harnessBackupRoot(root: string): string {
return resolve(root, "failover-backups");
}
function harnessBackupCandidates(root: string): string[] {
const backupRoot = harnessBackupRoot(root);
return [resolve(backupRoot, "current"), resolve(backupRoot, "previous")];
}
type VerifiedHarnessBackup = {
root: string;
manifest: NativeHarnessBackupManifest;
bytes: number;
};
export function shouldRestoreNativeHarnessBackupIntoSandbox(input: {
acquisitionOutcome: "created" | "resumed" | "replacement" | null;
reusableLeaseConfigured: boolean | null | undefined;
backupAvailable: boolean;
}): boolean {
return (
input.acquisitionOutcome === "created" &&
input.reusableLeaseConfigured === false &&
input.backupAvailable
);
}
function compatibleNativeHarnessBackupManifests(input: {
root: string;
execution: NativeExecutionInput;
runnerInstanceId: string;
}): Array<{ root: string; manifest: NativeHarnessBackupManifest }> {
const profile = resolveNativeHarnessPersistenceProfile(input.execution);
const expectedNames = profile.directories
.map((directory) => directory.name)
.sort();
const compatible: Array<{
root: string;
manifest: NativeHarnessBackupManifest;
}> = [];
for (const candidateRoot of harnessBackupCandidates(input.root)) {
const manifestPath = resolve(candidateRoot, "manifest.json");
if (!existsSync(manifestPath)) continue;
let manifest: NativeHarnessBackupManifest;
try {
manifest = JSON.parse(
readFileSync(manifestPath, "utf8"),
) as NativeHarnessBackupManifest;
} catch {
continue;
}
if (
manifest.schema !== "paperclip.native-harness-backup.v1" ||
manifest.normalizedSessionId !== nativeSessionKey(input.execution) ||
manifest.runnerInstanceId !== input.runnerInstanceId ||
manifest.providerKind !== profile.providerKind ||
manifest.driverKind !== profile.driverKind ||
manifest.environmentFingerprint !==
nativeHarnessEnvironmentFingerprint(input.execution) ||
manifest.runnerContractVersion !== RUNNERD_BINARY_CONTRACT_VERSION ||
!providerSessionIdentityIsPresent(manifest.providerSessionIdentity) ||
!Array.isArray(manifest.directories)
)
continue;
const manifestNames = manifest.directories
.map((directory) => directory.name)
.sort();
if (canonicalJson(expectedNames) !== canonicalJson(manifestNames)) continue;
compatible.push({ root: candidateRoot, manifest });
}
return compatible;
}
export function buildNativeHarnessBackupManifest(input: {
backupRoot: string;
execution: NativeExecutionInput;
runnerInstanceId: string;
providerSessionIdentity: unknown;
sourceProviderLeaseId: string;
completedAt?: string;
}): NativeHarnessBackupManifest {
if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) {
throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing");
}
const profile = resolveNativeHarnessPersistenceProfile(input.execution);
const directories = profile.directories.map((directory) => {
const path = resolve(input.backupRoot, directory.name);
if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing");
return { name: directory.name, ...digestBackupDirectory(path) };
});
return {
schema: "paperclip.native-harness-backup.v1",
normalizedSessionId: nativeSessionKey(input.execution),
runnerInstanceId: input.runnerInstanceId,
providerKind: profile.providerKind,
driverKind: profile.driverKind,
providerSessionIdentity: input.providerSessionIdentity,
sourceProviderLeaseId: input.sourceProviderLeaseId,
environmentFingerprint: nativeHarnessEnvironmentFingerprint(
input.execution,
),
runnerContractVersion: RUNNERD_BINARY_CONTRACT_VERSION,
directories,
completedAt: input.completedAt ?? new Date().toISOString(),
};
}
export function verifyNativeHarnessBackup(input: {
root: string;
execution: NativeExecutionInput;
runnerInstanceId: string;
}): VerifiedHarnessBackup | null {
for (const {
root: candidateRoot,
manifest,
} of compatibleNativeHarnessBackupManifests(input)) {
let bytes = 0;
let valid = true;
for (const declared of manifest.directories) {
const directoryPath = resolve(candidateRoot, declared.name);
if (!existsSync(directoryPath)) {
valid = false;
break;
}
try {
const digest = digestBackupDirectory(directoryPath);
if (
digest.sha256 !== declared.sha256 ||
digest.bytes !== declared.bytes
) {
valid = false;
break;
}
bytes += digest.bytes;
} catch {
valid = false;
break;
}
}
if (valid) return { root: candidateRoot, manifest, bytes };
}
return null;
}
function nativeSessionCheckpointDirectory(): string {
const directory = resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"sessions",
);
mkdirSync(directory, { recursive: true, mode: 0o700 });
chmodSync(directory, 0o700);
return directory;
}
function nativeSessionCheckpointPath(execution: NativeExecutionInput): string {
return resolve(
nativeSessionCheckpointDirectory(),
`${createHash("sha256")
.update(nativeSessionScopeKey(execution))
.digest("hex")}.json`,
);
}
function legacyCompanyNativeSessionCheckpointPath(
execution: NativeExecutionInput,
): string {
return resolve(
nativeSessionCheckpointDirectory(),
`${createHash("sha256")
.update(legacyCompanyNativeSessionScopeKey(execution))
.digest("hex")}.json`,
);
}
function legacyNativeSessionCheckpointPath(
execution: NativeExecutionInput,
): string {
return resolve(
nativeSessionCheckpointDirectory(),
`${createHash("sha256")
.update(nativeSessionKey(execution))
.digest("hex")}.json`,
);
}
function persistWarmNativeCheckpoint(
execution: NativeExecutionInput,
configDigest: string,
snapshot: PersistedNativeSession,
): void {
const path = nativeSessionCheckpointPath(execution);
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(
temporary,
JSON.stringify({
schema: "paperclip.native-session-supervisor.v1",
configDigest,
updatedAt: new Date().toISOString(),
snapshot,
}),
{ encoding: "utf8", mode: 0o600 },
);
renameSync(temporary, path);
chmodSync(path, 0o600);
}
function loadWarmNativeCheckpoint(
execution: NativeExecutionInput,
configDigest: string,
executionTargetKind: "local" | "remote" = "local",
): PersistedNativeSession | null {
const scopedPath = nativeSessionCheckpointPath(execution);
const path = [
scopedPath,
legacyCompanyNativeSessionCheckpointPath(execution),
legacyNativeSessionCheckpointPath(execution),
].find((candidate) => existsSync(candidate));
if (!path) return null;
const envelope = JSON.parse(
readBoundedNativeFile(
path,
NATIVE_WARM_CHECKPOINT_MAX_BYTES,
"native_session_supervisor_checkpoint_too_large",
).toString("utf8"),
) as {
schema?: string;
configDigest?: string;
snapshot?: PersistedNativeSession;
};
if (
envelope.schema !== "paperclip.native-session-supervisor.v1" ||
!envelope.snapshot
) {
throw new Error("native_session_supervisor_checkpoint_mismatch");
}
const persistedIdentity = record(envelope.snapshot.identity);
if (
persistedIdentity.sessionId !== nativeSessionKey(execution) ||
persistedIdentity.companyId !== execution.binding.companyId ||
persistedIdentity.agentId !== execution.binding.agentId
) {
throw new Error("native_session_supervisor_checkpoint_mismatch");
}
// Upgrade old projectless checkpoints using their validated prior run id.
// Every provider/model/runtime-context/permission field must still match;
// only the old per-run workspace placeholder is normalized away.
const legacyDigest =
execution.binding.executionWorkspaceId === execution.binding.runId &&
typeof persistedIdentity.runId === "string"
? nativeSessionConfigDigest(
execution,
executionTargetKind,
persistedIdentity.runId,
)
: null;
if (
envelope.configDigest !== configDigest &&
envelope.configDigest !== legacyDigest
) {
return null;
}
const sameRunRecovery =
persistedIdentity.runId === execution.binding.runId &&
persistedIdentity.issueId === execution.binding.issueId;
const resumed = sameRunRecovery
? structuredClone(envelope.snapshot)
: {
...envelope.snapshot,
identity: {
runId: execution.binding.runId,
sessionId: nativeSessionKey(execution),
companyId: execution.binding.companyId,
issueId: execution.binding.issueId,
agentId: execution.binding.agentId,
},
// A warm provider can be rebound only after the previous run settled.
// Its provider identity survives, but run-scoped turn, result, and
// request authority must not cross into the new heartbeat run.
semanticResult: null,
terminal: null,
activeTurnId: null,
terminalTurns: [],
pendingRuntimeRequests: [],
governedWait: undefined,
};
if (path !== scopedPath || envelope.configDigest !== configDigest) {
// Upgrade the validated checkpoint atomically. When moving from a legacy
// path, retain that file so an interrupted migration remains replayable.
persistWarmNativeCheckpoint(execution, configDigest, resumed);
}
return resumed;
}
async function releaseWarmNativeSession(
sessionId: string,
ownerToken: symbol,
idleTimeoutMs: number,
failed: boolean,
): Promise<void> {
const entry = warmNativeSessions.get(sessionId);
// A late completion or cleanup callback from an older execution must never
// release a replacement that has since claimed the same logical session.
if (!entry || entry.ownerToken !== ownerToken) return;
entry.busy = false;
entry.lastActivityAt = new Date().toISOString();
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
if (failed || entry.closeOnReleaseReason !== undefined) {
const requireCleanClose = entry.closeOnReleaseReason !== undefined;
entry.closeOnReleaseReason ??= "warm native session failed";
entry.busy = true;
try {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
} catch (error) {
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
if (requireCleanClose) throw error;
} finally { entry.busy = false; }
return;
}
// Durable shared-computer ownership must be retained before completion is acknowledged.
await entry.computerTarget?.retainWarm(idleTimeoutMs);
entry.idleTimer = setTimeout(() => {
const current = warmNativeSessions.get(sessionId);
if (current !== entry || current.busy) return;
current.busy = true;
current.closeOnReleaseReason = "warm native session idle timeout";
void closeWarmNativeSession(current, current.closeOnReleaseReason)
.then(() => { if (warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); })
.catch((error) => {
if (error instanceof ComputerOwnerSupersededError) {
// Admission owns the next generation. Leave this transport available
// for its imminent warm reservation instead of closing or deleting it.
current.closeOnReleaseReason = undefined;
current.idleTimer = null;
return;
}
if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId);
})
.finally(() => { current.busy = false; });
}, idleTimeoutMs);
entry.idleTimer.unref();
}
/** Classify only a committed provider terminal, never model prose or tool output. */
export function nativeProviderUsageLimitFromEvent(
event: Pick<PrpEvent, "sourceKind" | "eventType" | "payload">,
): boolean {
const payload = record(event.payload);
return (
event.sourceKind === "runner" &&
event.eventType === "turn.failed" &&
payload.status === "failed" &&
record(payload.error).codexErrorInfo === "usageLimitExceeded"
);
}
export function nativeSessionFailureDisposition(
attempt: number,
now = new Date(),
sourceFailureCode?: ReturnType<typeof nativeSessionFailureSourceCode>,
) {
const permanentFailure =
sourceFailureCode === "native_provider_model_rejected" ||
sourceFailureCode === "native_provider_approval_required" ||
sourceFailureCode === "native_event_replay_conflict" ||
sourceFailureCode === "runner_remote_provider_artifact_incompatible" ||
sourceFailureCode === "runner_remote_artifact_platform_mismatch" ||
sourceFailureCode === "native_provider_terminal_failed" ||
sourceFailureCode === "native_current_wake_comments_unread" ||
sourceFailureCode === "native_current_wake_comments_changed_after_read" ||
sourceFailureCode === "native_session_cleanup_quarantined" ||
sourceFailureCode === "native_adopted_runner_authentication_timeout" ||
sourceFailureCode === "native_provider_usage_limit";
const permissionDeclined = sourceFailureCode === "native_permission_declined";
const exhausted = permanentFailure || permissionDeclined || attempt >= 3;
return {
phase: exhausted
? ("terminal_failure" as const)
: ("retryable_failure" as const),
failureCode: permanentFailure || permissionDeclined
? sourceFailureCode!
: exhausted
? ("native_session_retry_exhausted" as const)
: ("native_session_interrupted" as const),
nextAttemptAt: exhausted ? null : new Date(now.getTime() + 30_000),
};
}
export function nativeSessionRecoveryProjection(input: {
phase: "retryable_failure" | "terminal_failure";
failureCode: string;
agentId: string;
}) {
const exhausted = input.phase === "terminal_failure";
return {
exhausted,
issueStatus:
exhausted &&
input.failureCode !== NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT
? ("blocked" as const)
: null,
recoveryOwner: exhausted
? { kind: "board" as const }
: { kind: "agent" as const, agentId: input.agentId },
recoveryActionOwnerType: exhausted
? ("board" as const)
: ("agent" as const),
recoveryActionOwnerAgentId: exhausted ? null : input.agentId,
recoveryActionCause: input.failureCode,
supersedeOnIdentityChange: true as const,
};
}
export function nativeSessionFailureSourceCode(
error: unknown,
):
| "native_permission_declined"
| "native_provider_terminal_failed"
| "native_provider_approval_required"
| "native_provider_usage_limit"
| "native_session_cleanup_quarantined"
| "native_adopted_runner_authentication_timeout"
| "runner_remote_provider_artifact_incompatible"
| "runner_remote_artifact_platform_mismatch"
| "provider_process_exited"
| "provider_stdout_closed"
| "provider_process_output_closed"
| "provider_process_status_failed"
| "provider_initialize_timeout"
| "provider_initialize_protocol_error"
| "provider_request_timeout"
| "provider_request_protocol_error"
| "provider_frame_too_large"
| "provider_transport_failed"
| "native_runner_process_exited"
| "planning_mode_unsupported"
| "native_event_replay_conflict"
| "native_provider_model_rejected"
| "native_current_wake_comments_unread"
| "native_current_wake_comments_changed_after_read"
| "native_session_interrupted" {
if (error instanceof NativePermissionDeclinedError) return "native_permission_declined";
if (error instanceof NativeProviderTerminalFailure) {
if (error.providerCode === "approval_required") return "native_provider_approval_required";
// Failed terminals retain their security meaning across the provider facade.
// A stopped process is insufficient evidence to recover an integrity breach.
if (
/(?:binding_mismatch|start_mismatch|replay_conflict|digest_mismatch|invalid_semantic_result|conflicting_semantic_result|provider_event_type_invalid)/.test(
error.providerCode,
)
)
return "native_event_replay_conflict";
return "native_provider_terminal_failed";
}
if (error instanceof NativeSessionProtocolIntegrityError)
return "native_event_replay_conflict";
if (error instanceof NativeSessionCleanupQuarantinedError)
return "native_session_cleanup_quarantined";
const message = error instanceof Error ? error.message : String(error);
if (/native_provider_model_rejected/i.test(message))
return "native_provider_model_rejected";
if (/native_adopted_runner_authentication_timeout/i.test(message)) {
return "native_adopted_runner_authentication_timeout";
}
if (/^runner_remote_artifact_platform_mismatch(?::|$)/.test(message)) {
return "runner_remote_artifact_platform_mismatch";
}
if (/runner_remote_provider_artifact_incompatible/i.test(message)) {
return "runner_remote_provider_artifact_incompatible";
}
if (/provider_process_exited/i.test(message)) {
return "provider_process_exited";
}
if (/provider_stdout_closed/i.test(message)) {
return "provider_stdout_closed";
}
if (/provider_process_output_closed/i.test(message)) {
return "provider_process_output_closed";
}
if (/provider_process_status_failed/i.test(message)) {
return "provider_process_status_failed";
}
if (/provider_initialize_timeout/i.test(message)) {
return "provider_initialize_timeout";
}
if (/provider_initialize_protocol_error/i.test(message)) {
return "provider_initialize_protocol_error";
}
if (/provider_request_timeout/i.test(message)) {
return "provider_request_timeout";
}
if (/provider_request_protocol_error/i.test(message)) {
return "provider_request_protocol_error";
}
if (/provider_frame_too_large|stdout frame exceeded/i.test(message)) {
return "provider_frame_too_large";
}
if (
/provider_transport_failed|invalid JSON-RPC|provider failed/i.test(message)
) {
return "provider_transport_failed";
}
if (
/native_runner_process_exited|runnerd exited|runner process failed/i.test(
message,
)
) {
return "native_runner_process_exited";
}
if (/planning_mode_unsupported/i.test(message)) {
return "planning_mode_unsupported";
}
if (/native_event_replay_conflict/i.test(message)) {
return "native_event_replay_conflict";
}
if (/native_current_wake_comments_changed_after_read/i.test(message)) {
return "native_current_wake_comments_changed_after_read";
}
if (/native_current_wake_comments_unread/i.test(message)) {
return "native_current_wake_comments_unread";
}
return "native_session_interrupted";
}
const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE =
"Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped.";
const PROVIDER_DURABLE_EVENT_TYPES = new Set([
"harness.ready",
"session.started",
"session.resumed",
"session.updated",
"turn.started",
"provider.event",
"provider.rpc_result",
]);
type NativeRecoveryMode =
"bootstrap_retry" | "exact_checkpoint_resume" | "ambiguous_state";
export async function nativeProviderRecoveryEvidence(input: {
db: Db;
runId: string;
sourceFailureCode: ReturnType<typeof nativeSessionFailureSourceCode>;
}): Promise<{
recoveryMode: NativeRecoveryMode;
providerSessionEstablished: boolean;
providerEventsExist: boolean;
checkpointExists: boolean;
}> {
const run = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
const checkpoint = record(run?.runnerProfileJson).sessionCheckpoint;
const checkpointRecord = record(checkpoint);
const checkpointExists = Object.keys(checkpointRecord).length > 0;
const providerSessionEstablished =
(typeof checkpointRecord.providerSessionId === "string" &&
checkpointRecord.providerSessionId.length > 0) ||
Object.keys(record(checkpointRecord.providerIdentity)).length > 0;
const durableEvents = await input.db
.select({ eventType: heartbeatRunEvents.eventType })
.from(heartbeatRunEvents)
.where(
and(
eq(heartbeatRunEvents.runId, input.runId),
inArray(heartbeatRunEvents.eventType, [
...PROVIDER_DURABLE_EVENT_TYPES,
]),
),
)
.limit(1);
const providerEventsExist = durableEvents.some((event) =>
PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType),
);
if (
checkpointExists &&
providerSessionEstablished &&
record(checkpointRecord.terminal).runTerminalState !== "failed"
) {
return {
recoveryMode: "exact_checkpoint_resume",
providerSessionEstablished: true,
providerEventsExist,
checkpointExists,
};
}
const definitelyPreSession = new Set<
ReturnType<typeof nativeSessionFailureSourceCode>
>([
"runner_remote_provider_artifact_incompatible",
"runner_remote_artifact_platform_mismatch",
"provider_process_exited",
"provider_stdout_closed",
"provider_process_output_closed",
"provider_process_status_failed",
"provider_initialize_timeout",
"provider_initialize_protocol_error",
"provider_request_timeout",
"provider_request_protocol_error",
"native_runner_process_exited",
]).has(input.sourceFailureCode);
if (!checkpointExists && !providerEventsExist && definitelyPreSession) {
return {
recoveryMode: "bootstrap_retry",
providerSessionEstablished: false,
providerEventsExist: false,
checkpointExists: false,
};
}
return {
recoveryMode: "ambiguous_state",
providerSessionEstablished:
providerSessionEstablished || providerEventsExist,
providerEventsExist,
checkpointExists,
};
}
function record(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
type FirstAgentEventKind =
| "reasoning"
| "agentMessage"
| "toolCall"
| "dynamicToolCall";
/** Returns the first provider activity worth measuring after turn.started. */
export function firstMeaningfulAgentEventKind(
event: Pick<PrpEvent, "eventType" | "payload">,
): FirstAgentEventKind | null {
const payload = record(event.payload);
if (event.eventType === "tool.execution.started") {
return typeof payload.executionId === "string" &&
payload.executionId.trim()
? "toolCall"
: null;
}
if (
event.eventType !== "item.started" &&
event.eventType !== "item.delta" &&
event.eventType !== "item.completed"
) return null;
const kind = payload.kind;
if (
kind !== "reasoning" &&
kind !== "agentMessage" &&
kind !== "toolCall" &&
kind !== "dynamicToolCall"
) return null;
if (event.eventType !== "item.delta") return kind;
const text = [payload.text, payload.delta, payload.content].find(
(value): value is string => typeof value === "string" && value.trim().length > 0,
);
return text ? kind : null;
}
export type NativeSessionSteeringState = {
disposition: "available" | "unsupported" | "temporarily_unavailable";
activeTurnId: string | null;
};
export class NativeSessionSteeringError extends Error {
constructor(
readonly code:
| "steering_unsupported"
| "steering_temporarily_unavailable"
| "steering_stale_turn"
| "steering_timeout"
| "steering_rejected",
message: string,
) {
super(message);
this.name = "NativeSessionSteeringError";
}
}
export class NativeRuntimeRequestResolutionError extends Error {
constructor(
readonly code:
| "native_session_not_active"
| "runtime_request_resolution_unsupported"
| "runtime_request_stale_turn"
| "runtime_request_resolution_conflict",
message: string,
) {
super(message);
this.name = "NativeRuntimeRequestResolutionError";
}
}
/** Resolve a provider runtime request on an in-process native backend. */
export async function resolveNativeRuntimeRequest(input: {
runId: string;
requestId: string;
turnId: string;
resolution: HarnessRuntimeRequestResolution;
/**
* Revalidate the caller's durable lifecycle and authorization immediately
* before the provider mutation. Capability and snapshot reads above this
* edge are asynchronous, so route-level checks performed before entering
* this helper are not sufficient to authorize the eventual dispatch.
*/
authorizeBeforeDispatch: () => Promise<void>;
}): Promise<{ commandId: string }> {
const active = activeNativeSessions.get(input.runId);
if (!active) {
throw new NativeRuntimeRequestResolutionError(
"native_session_not_active",
"The active native session is not attached.",
);
}
const capabilities = await active.session.capabilities();
if (
!capabilities.runtimeRequestResolution ||
active.session.resolveRuntimeRequest === undefined
) {
throw new NativeRuntimeRequestResolutionError(
"runtime_request_resolution_unsupported",
"This native session does not resolve runtime requests in-process.",
);
}
const snapshot = await active.session.snapshot();
if (snapshot.activeTurnId !== input.turnId) {
throw new NativeRuntimeRequestResolutionError(
"runtime_request_stale_turn",
"The runtime request belongs to a turn that is no longer active.",
);
}
const key = `${input.runId}:${input.requestId}`;
const fingerprint = JSON.stringify({
turnId: input.turnId,
resolution: input.resolution,
});
const prior = nativeRuntimeRequestResolutions.get(key);
if (prior) {
if (prior.fingerprint !== fingerprint) {
throw new NativeRuntimeRequestResolutionError(
"runtime_request_resolution_conflict",
"A different response was already submitted for this runtime request.",
);
}
await prior.pending;
return { commandId: prior.commandId };
}
const commandId = `native-runtime-response:${randomUUID()}`;
// Reserve the request key before yielding to authorization or provider I/O.
// This makes duplicate retries join one dispatch and makes a conflicting
// response fail closed even while the first authorization check is pending.
const pending = Promise.resolve().then(async () => {
await input.authorizeBeforeDispatch();
if (activeNativeSessions.get(input.runId) !== active) {
throw new NativeRuntimeRequestResolutionError(
"native_session_not_active",
"The active native session changed before the response was dispatched.",
);
}
await active.session.resolveRuntimeRequest!({
requestId: input.requestId,
turnId: input.turnId,
resolution: input.resolution,
});
});
const resolution: NativeRuntimeRequestResolution = {
runId: input.runId,
fingerprint,
commandId,
pending,
completedAt: null,
};
nativeRuntimeRequestResolutions.set(key, resolution);
try {
await pending;
resolution.completedAt = Date.now();
pruneNativeRuntimeRequestResolutionCache();
return { commandId };
} catch (error) {
if (nativeRuntimeRequestResolutions.get(key) === resolution) {
nativeRuntimeRequestResolutions.delete(key);
}
throw error;
}
}
export async function getNativeSessionSteeringState(
runId: string,
): Promise<NativeSessionSteeringState> {
const active = activeNativeSessions.get(runId);
if (!active)
return { disposition: "temporarily_unavailable", activeTurnId: null };
const capabilities = await active.session.capabilities();
if (!capabilities.steering || !active.session.steer) {
return { disposition: "unsupported", activeTurnId: null };
}
const snapshot = await active.session.snapshot();
return {
disposition: snapshot.activeTurnId
? "available"
: "temporarily_unavailable",
activeTurnId: snapshot.activeTurnId ?? null,
};
}
// Receipts live as long as this controller process. Durable identity reservations
// hold credential acquisition after a restart until a provider receipt is known.
const steeringDeliveries = new Map<string, Promise<{ turnId: string }>>();
function clearSteeringDeliveries(runId: string) {
for (const key of steeringDeliveries.keys())
if (key.startsWith(`${runId}:`)) steeringDeliveries.delete(key);
}
/** Dispatches a true same-turn steering message and resolves only after ack. */
export async function steerNativeSession(input: {
runId: string;
message: string;
correlationId: string;
timeoutMs?: number;
onAcknowledged?: () => Promise<void>;
}): Promise<{ turnId: string }> {
const active = activeNativeSessions.get(input.runId);
if (!active) {
throw new NativeSessionSteeringError(
"steering_temporarily_unavailable",
"The active native session is not attached.",
);
}
const capabilities = await active.session.capabilities();
if (!capabilities.steering || !active.session.steer) {
throw new NativeSessionSteeringError(
"steering_unsupported",
"This provider does not support same-turn steering.",
);
}
const snapshot = await active.session.snapshot();
const turnId = snapshot.activeTurnId ?? null;
if (!turnId) {
throw new NativeSessionSteeringError(
"steering_stale_turn",
"The target turn is no longer active.",
);
}
const deliveryKey = `${input.runId}:${input.correlationId}`;
let delivery = steeringDeliveries.get(deliveryKey);
if (!delivery) {
delivery = active.session
.steer({
turnId,
message: { role: "user", text: input.message },
correlationId: input.correlationId,
})
.then(() => ({ turnId }));
steeringDeliveries.set(deliveryKey, delivery);
void delivery.catch(() => {
steeringDeliveries.delete(deliveryKey);
});
}
// The route still serializes queue mutations on the task until acknowledgement.
// Reconciliation can acquire that task lock after success or a timeout; never
// join it from the provider's acknowledgement path.
if (input.onAcknowledged)
void delivery.then(input.onAcknowledged).catch(() => undefined);
let timeout: ReturnType<typeof setTimeout> | null = null;
try {
const acknowledged = await Promise.race([
delivery,
new Promise<never>((_resolve, reject) => {
timeout = setTimeout(
() =>
reject(
new NativeSessionSteeringError(
"steering_timeout",
"The provider did not acknowledge steering in time.",
),
),
input.timeoutMs ?? 10_000,
);
}),
]);
return acknowledged;
} catch (error) {
if (error instanceof NativeSessionSteeringError) throw error;
const message = error instanceof Error ? error.message : String(error);
if (/stale|terminal|active turn/i.test(message)) {
throw new NativeSessionSteeringError(
"steering_stale_turn",
"The target turn is no longer active.",
);
}
if (/unsupported|unavailable|capability/i.test(message)) {
throw new NativeSessionSteeringError(
"steering_unsupported",
"This provider does not support same-turn steering.",
);
}
throw new NativeSessionSteeringError(
"steering_rejected",
"The provider rejected the steering message.",
);
} finally {
if (timeout) clearTimeout(timeout);
}
}
export function cancelNativeSession(
runId: string,
reason: string,
): Promise<boolean>;
export function cancelNativeSession(
runId: string,
reason: string,
options: {
db: Db;
scope?: "turn" | "run" | "issue";
replacementAccepted?: boolean;
cancellationRequestId?: string;
},
): Promise<{
dispatched: boolean;
decision: NativeStatusDecision | null;
decisionId: string | null;
auditId: string | null;
}>;
export async function cancelNativeSession(
runId: string,
reason: string,
options?: {
db: Db;
scope?: "turn" | "run" | "issue";
replacementAccepted?: boolean;
cancellationRequestId?: string;
},
): Promise<
| boolean
| {
dispatched: boolean;
decision: NativeStatusDecision | null;
decisionId: string | null;
auditId: string | null;
}
> {
const runStop = (options?.scope ?? "run") === "run";
let decision: NativeStatusDecision | null = null;
let decisionContext: {
companyId: string;
issueId: string;
assessmentId: string | null;
priorStatus: string;
priorStatusVersion: number;
priorDecisionId: string | null;
coordinatorDecisionId: string | null;
agentId: string;
} | null = null;
if (options) {
const run = await options.db
.select({
agentId: heartbeatRuns.agentId,
companyId: heartbeatRuns.companyId,
nativeIssueId: heartbeatRuns.nativeIssueId,
runtimeMode: heartbeatRuns.runtimeMode,
status: heartbeatRuns.status,
nativePhase: heartbeatRuns.nativePhase,
errorCode: heartbeatRuns.errorCode,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (run?.runtimeMode === "native") {
if (isNativeRunnerOwnershipHeld(run))
throw new NativeRunnerOwnershipUnverifiedError();
const issueId = run.nativeIssueId;
if (!issueId) throw new Error("native_cancellation_binding_missing");
const issue = await options.db
.select({
status: issues.status,
statusVersion: issues.statusVersion,
lastStatusDecisionId: issues.lastStatusDecisionId,
})
.from(issues)
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!issue) throw new Error("native_cancellation_binding_missing");
const coordinator = await options.db
.select({
assessmentId: nativeRunFinalizations.assessmentId,
decisionId: nativeRunFinalizations.decisionId,
})
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, runId),
eq(nativeRunFinalizations.companyId, run.companyId),
eq(nativeRunFinalizations.issueId, issueId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!coordinator)
throw new Error("native_cancellation_coordinator_missing");
decision = resolveNativeCancellationStatus({
scope: options.scope ?? "run",
priorIssueStatus: issue.status as NativeAuthoritativeIssueStatus,
agentId: run.agentId,
replacementAccepted: options.replacementAccepted,
});
decisionContext = {
companyId: run.companyId,
issueId,
assessmentId: coordinator.assessmentId ?? null,
priorStatus: issue.status,
priorStatusVersion: Number(issue.statusVersion),
priorDecisionId: issue.lastStatusDecisionId,
coordinatorDecisionId: coordinator.decisionId ?? null,
agentId: run.agentId,
};
}
}
let decisionId: string | null = null;
let auditId: string | null = null;
let cancellationIntentId: string | null = null;
let recoveringCancellationIntent = false;
let priorCoordinatorDecisionIdAtIntent: string | null = null;
if (options && decision && decisionContext) {
const cancellationDecision = decision;
const cancellationContext = decisionContext;
const effects = cancellationDecision.effects.map((effect) => effect.kind);
let intentPublication: Parameters<typeof publishActivity>[0] | null = null;
const intent = await options.db.transaction(async (tx) => {
const lockedRun = await tx
.select({
agentId: heartbeatRuns.agentId,
companyId: heartbeatRuns.companyId,
nativeIssueId: heartbeatRuns.nativeIssueId,
resultJson: heartbeatRuns.resultJson,
runtimeMode: heartbeatRuns.runtimeMode,
status: heartbeatRuns.status,
nativePhase: heartbeatRuns.nativePhase,
errorCode: heartbeatRuns.errorCode,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (
!lockedRun ||
lockedRun.runtimeMode !== "native" ||
lockedRun.companyId !== cancellationContext.companyId ||
lockedRun.agentId !== cancellationContext.agentId ||
lockedRun.nativeIssueId !== cancellationContext.issueId
) {
throw new Error("native_cancellation_binding_changed");
}
if (isNativeRunnerOwnershipHeld(lockedRun))
throw new NativeRunnerOwnershipUnverifiedError();
const retryCancellation = lockedRun.status === "failed"
|| record(record(lockedRun.resultJson).startupCancellation).retryCancellation === true;
const coordinatorQuery = tx
.select({ runId: nativeRunFinalizations.runId, phase: nativeRunFinalizations.phase, failureCode: nativeRunFinalizations.failureCode })
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, runId),
eq(nativeRunFinalizations.companyId, cancellationContext.companyId),
eq(nativeRunFinalizations.issueId, cancellationContext.issueId),
),
);
// Recheck after reservation, at the same transaction that records intent
// and disables the retry. NOWAIT avoids coordinator -> run lock inversion.
const coordinator = await (retryCancellation
? coordinatorQuery.for("update", { noWait: true }) : coordinatorQuery)
.limit(1).then((rows) => rows[0] ?? null);
if (!coordinator)
throw new Error("native_cancellation_coordinator_missing");
const resultJson = record(lockedRun.resultJson);
// A default Stop joins an already reserved caller intent; it cannot
// replace that intent while the originating request is still dispatching.
const requestedId = options.cancellationRequestId
?? cancellationRequestId(record(resultJson.startupCancellation).cancellationRequestId);
if (requestedId) assertCancellationRequest(resultJson, requestedId);
if (retryCancellation && !nativeRetryCancellationEligible({
runId, companyId: cancellationContext.companyId, issueId: cancellationContext.issueId,
resultJson, requestId: requestedId, scope: options.scope ?? "run",
}, coordinator)) throw conflict("Native retry is no longer cancellable");
const existing = record(resultJson.nativeCancellation);
const existingIntentId =
typeof existing.intentId === "string" && existing.intentId.length > 0
? existing.intentId
: null;
if (existingIntentId) {
const matchingIntent =
existing.schema === "paperclip.native-cancellation.v1" &&
existing.companyId === cancellationContext.companyId &&
existing.runId === runId &&
existing.issueId === cancellationContext.issueId &&
existing.scope === (options.scope ?? "run") &&
existing.reasonCode === cancellationDecision.reasonCode &&
JSON.stringify(existing.effects) === JSON.stringify(effects);
if (!matchingIntent)
throw new Error("native_cancellation_intent_conflict");
const existingAuditId =
typeof existing.intentAuditId === "string" &&
existing.intentAuditId.length > 0
? existing.intentAuditId
: null;
if (!existingAuditId)
throw new Error("native_cancellation_intent_audit_missing");
return {
intentId: existingIntentId,
auditId: existingAuditId,
acknowledged: existing.dispatchState === "acknowledged",
dispatched: existing.dispatched === true,
decisionId:
typeof existing.decisionId === "string"
? existing.decisionId
: null,
priorCoordinatorDecisionId:
typeof existing.priorCoordinatorDecisionId === "string"
? existing.priorCoordinatorDecisionId
: null,
existing: true,
};
}
const intentId = requestedId
? callerCancellationIntentId(requestedId)
: `native-cancellation:${randomUUID()}`;
const activity = await persistActivity(tx as unknown as Db, {
companyId: cancellationContext.companyId,
actorType: "system",
actorId: "native-session-cancellation",
action: "native.cancellation_intent_recorded",
entityType: "heartbeat_run",
entityId: runId,
agentId: cancellationContext.agentId,
runId,
issueId: cancellationContext.issueId,
details: {
intentId,
scope: options.scope ?? "run",
reasonCode: cancellationDecision.reasonCode,
effects,
},
});
const intentAuditId = activity.activity?.id ?? null;
if (!intentAuditId)
throw new Error("native_cancellation_intent_audit_missing");
const written = await tx
.update(heartbeatRuns)
.set({
resultJson: {
...resultJson,
nativeCancellation: {
schema: "paperclip.native-cancellation.v1",
intentId,
intentAuditId,
companyId: cancellationContext.companyId,
runId,
issueId: cancellationContext.issueId,
scope: options.scope ?? "run",
reasonCode: cancellationDecision.reasonCode,
effects,
dispatchState: "pending",
dispatched: false,
decisionId: null,
priorCoordinatorDecisionId:
cancellationContext.coordinatorDecisionId,
recordedAt: new Date().toISOString(),
},
},
updatedAt: new Date(),
})
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, cancellationContext.companyId),
eq(heartbeatRuns.agentId, cancellationContext.agentId),
eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId),
),
)
.returning({ id: heartbeatRuns.id })
.then((rows) => rows[0] ?? null);
if (!written) throw new Error("native_cancellation_binding_changed");
// Cancellation is also an authority fence for a durable retry whose
// preceding provider has already failed. No in-memory session is needed.
await tx
.update(nativeRunFinalizations)
.set({
phase: "terminal_failure",
nextAttemptAt: null,
leaseOwner: null,
leaseExpiresAt: null,
recoveryState: "blocked",
failureCode: "native_retry_cancelled",
updatedAt: new Date(),
})
.where(
and(
eq(nativeRunFinalizations.runId, runId),
eq(nativeRunFinalizations.companyId, cancellationContext.companyId),
eq(nativeRunFinalizations.phase, "retryable_failure"),
),
);
intentPublication = activity.publication;
return {
intentId,
auditId: intentAuditId,
acknowledged: false,
dispatched: false,
decisionId: null,
priorCoordinatorDecisionId: cancellationContext.coordinatorDecisionId,
existing: false,
};
}).catch(rethrowNativeCancellationLockConflict);
if (intentPublication) publishActivity(intentPublication);
cancellationIntentId = intent.intentId;
auditId = intent.auditId;
decisionId = intent.decisionId;
recoveringCancellationIntent = intent.existing;
priorCoordinatorDecisionIdAtIntent = intent.priorCoordinatorDecisionId;
if (intent.acknowledged && (!runStop || intent.dispatched ||
(!nativeSessionStartups.has(runId) && !activeNativeSessions.has(runId)))) {
return {
dispatched: intent.dispatched,
decision,
decisionId,
auditId,
};
}
}
// Startup already owns a provider lifetime even before publishing its handle.
// Do not acknowledge a no-op Stop and let that owner submit a turn afterwards.
const startup = runStop && !activeNativeSessions.has(runId) ? nativeSessionStartups.get(runId) : undefined;
let settleStartupCancellation: (() => void) | undefined;
if (startup) {
startup.stopRequested = true;
startup.cancellationSettled = new Promise<void>(resolve => { settleStartupCancellation = resolve; });
}
try {
const active = activeNativeSessions.get(runId) ??
(startup ? await waitForNativeSessionStartup(runId, () => new NativeCancellationPendingRecoveryError()) : null);
let dispatched = false;
if (active) {
dispatched = true;
if (!active.cancelRequested) {
active.cancelRequested = true;
try {
if (active.session.cancel) {
const cancellationAbort = new AbortController();
const cleanup = active.session.cancel({
reason,
signal: cancellationAbort.signal,
}).cleanup;
let cleanupTimer: ReturnType<typeof setTimeout> | undefined;
const settled = await Promise.race([
cleanup.then(
() => true,
() => true,
),
new Promise<false>((resolve) => {
cleanupTimer = setTimeout(
() => resolve(false),
NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS,
);
}),
]);
if (cleanupTimer) clearTimeout(cleanupTimer);
if (!settled) {
cancellationAbort.abort(
new Error("native session cancellation cleanup timed out"),
);
void cleanup.catch(() => undefined);
}
} else if (active.session.interrupt)
await active.session.interrupt({ reason });
} catch (error) {
active.cancelRequested = false;
throw error;
}
}
}
if (!options) return dispatched;
if (decision && decisionContext) {
const cancellationDecision = decision;
const cancellationContext = decisionContext;
if (!cancellationIntentId || !auditId)
throw new Error("native_cancellation_intent_audit_missing");
if (
recoveringCancellationIntent &&
cancellationContext.coordinatorDecisionId !==
priorCoordinatorDecisionIdAtIntent
) {
decisionId ??= cancellationContext.coordinatorDecisionId;
}
if (
cancellationContext.assessmentId &&
cancellationDecision.reasonCode !== null &&
!decisionId
) {
const committed = await commitNativeStatusDecision({
db: options.db,
companyId: cancellationContext.companyId,
issueId: cancellationContext.issueId,
runId,
assessmentId: cancellationContext.assessmentId,
priorStatus: cancellationContext.priorStatus,
priorStatusVersion: cancellationContext.priorStatusVersion,
priorDecisionId: cancellationContext.priorDecisionId,
decision: cancellationDecision,
});
decisionId = committed.decision.id;
}
const acknowledgement = await options.db.transaction(async (tx) => {
const lockedRun = await tx
.select({
agentId: heartbeatRuns.agentId,
companyId: heartbeatRuns.companyId,
nativeIssueId: heartbeatRuns.nativeIssueId,
resultJson: heartbeatRuns.resultJson,
runtimeMode: heartbeatRuns.runtimeMode,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (
!lockedRun ||
lockedRun.runtimeMode !== "native" ||
lockedRun.companyId !== cancellationContext.companyId ||
lockedRun.agentId !== cancellationContext.agentId ||
lockedRun.nativeIssueId !== cancellationContext.issueId
) {
throw new Error("native_cancellation_binding_changed");
}
const coordinator = await tx
.select({ runId: nativeRunFinalizations.runId })
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, runId),
eq(nativeRunFinalizations.companyId, cancellationContext.companyId),
eq(nativeRunFinalizations.issueId, cancellationContext.issueId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!coordinator)
throw new Error("native_cancellation_coordinator_missing");
const resultJson = record(lockedRun.resultJson);
const intent = record(resultJson.nativeCancellation);
const matchingIntent =
intent.schema === "paperclip.native-cancellation.v1" &&
intent.intentId === cancellationIntentId &&
intent.intentAuditId === auditId &&
intent.companyId === cancellationContext.companyId &&
intent.runId === runId &&
intent.issueId === cancellationContext.issueId;
if (!matchingIntent)
throw new Error("native_cancellation_intent_conflict");
if (intent.dispatchState === "acknowledged" && (intent.dispatched === true || !dispatched)) {
return {
publication: null,
decisionId:
typeof intent.decisionId === "string"
? intent.decisionId
: decisionId,
};
}
if (
options.replacementAccepted &&
cancellationDecision.effects.some(
(effect) => effect.kind === "accept_replacement_turn",
)
) {
await tx
.update(heartbeatRuns)
.set({
status: "running",
continuationAttempt: sql`${heartbeatRuns.continuationAttempt} + 1`,
nextAction: "Accept a replacement native turn on the existing run.",
updatedAt: new Date(),
})
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, cancellationContext.companyId),
eq(heartbeatRuns.agentId, cancellationContext.agentId),
eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId),
),
);
}
const activity = await persistActivity(tx as unknown as Db, {
companyId: cancellationContext.companyId,
actorType: "system",
actorId: "native-session-cancellation",
action: "native.cancellation_dispatch_acknowledged",
entityType: "heartbeat_run",
entityId: runId,
agentId: cancellationContext.agentId,
runId,
issueId: cancellationContext.issueId,
details: {
intentId: cancellationIntentId,
intentAuditId: auditId,
scope: options.scope ?? "run",
reasonCode: cancellationDecision.reasonCode,
effects: cancellationDecision.effects.map((effect) => effect.kind),
dispatched,
decisionId,
},
});
const acknowledgementAuditId = activity.activity?.id ?? null;
if (!acknowledgementAuditId)
throw new Error("native_cancellation_ack_audit_missing");
const cancellationWrite = await tx
.update(heartbeatRuns)
.set({
resultJson: {
...resultJson,
nativeCancellation: {
...intent,
dispatchState: "acknowledged",
dispatched,
decisionId,
acknowledgementAuditId,
acknowledgedAt: new Date().toISOString(),
},
},
updatedAt: new Date(),
})
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, cancellationContext.companyId),
eq(heartbeatRuns.agentId, cancellationContext.agentId),
eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId),
),
)
.returning({ id: heartbeatRuns.id })
.then((rows) => rows[0] ?? null);
if (!cancellationWrite)
throw new Error("native_cancellation_binding_changed");
return { publication: activity.publication, decisionId };
});
decisionId = acknowledgement.decisionId;
if (acknowledgement.publication)
publishActivity(acknowledgement.publication);
}
return { dispatched, decision, decisionId, auditId };
} finally {
settleStartupCancellation?.();
}
}
/** Authenticated cancellation scope projected through the shared arbiter. */
export function resolveNativeCancellationStatus(input: {
scope: "turn" | "run" | "issue";
priorIssueStatus: NativeAuthoritativeIssueStatus;
agentId: string;
replacementAccepted?: boolean;
}): NativeStatusDecision {
if (input.scope === "turn") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: input.replacementAccepted ? null : "cancellation_turn_only",
unblockDescriptor: null,
effects: [{ kind: "accept_replacement_turn" }],
};
}
if (input.scope === "run") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "cancellation_run_only",
unblockDescriptor: null,
effects: [{ kind: "release_run_resources" }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "cancelled",
toStatus: "cancelled",
reasonCode: "cancellation_issue_authorized",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }],
};
}
export async function renewNativeSessionExecutionLease(input: {
db: Db;
runId: string;
companyId: string;
issueId: string;
leaseOwner: string;
attempt: number;
controller?: NativeControllerIdentity;
leaseTtlMs?: number;
}): Promise<void> {
const controller =
input.controller ?? (await currentNativeControllerIdentity());
const leaseTtlMs = input.leaseTtlMs ?? NATIVE_SESSION_EXECUTION_LEASE_TTL_MS;
if (
!Number.isInteger(leaseTtlMs) ||
leaseTtlMs < 1_000 ||
leaseTtlMs > NATIVE_SESSION_EXECUTION_LEASE_TTL_MS
) {
throw new Error("native_session_lease_ttl_invalid");
}
const [updated] = await input.db
.update(nativeRunFinalizations)
.set({
leaseExpiresAt: sql`now() + (${leaseTtlMs} * interval '1 millisecond')`,
updatedAt: sql`now()`,
})
.where(
and(
eq(nativeRunFinalizations.runId, input.runId),
eq(nativeRunFinalizations.companyId, input.companyId),
eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.leaseOwner, input.leaseOwner),
eq(nativeRunFinalizations.attempt, input.attempt),
eq(nativeRunFinalizations.controllerBootId, controller.bootId),
eq(nativeRunFinalizations.controllerPid, controller.pid),
eq(
nativeRunFinalizations.controllerProcessStartedAt,
controller.processStartedAt,
),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.returning({ runId: nativeRunFinalizations.runId });
if (!updated) throw new Error("native_session_lease_lost");
}
/** Computer ACPX cold starts stage the complete qualified Linux provider pack.
* Measured 1.1 GiB transfers plus compression/verification need about 24 minutes;
* this finite admission budget does not extend warm leases or turn execution.
*/
export function nativeSessionBootstrapTimeoutMs(
providerKind: NativeExecutionInput["provider"]["kind"],
target?: Pick<Extract<AdapterExecutionTarget, { kind: "remote" }>, "kind" | "transport"> | { kind: "local" } | null,
): number | undefined {
return providerKind === "acpx" && target?.kind === "remote" && target.transport === "computer"
? 30 * 60 * 1_000
: undefined;
}
function startNativeSessionExecutionLeaseRenewal(input: {
db: Db;
runId: string;
companyId: string;
issueId: string;
leaseOwner: string;
attempt: number;
controller: NativeControllerIdentity;
}): { stop: () => Promise<void> } {
let leaseLost: Error | null = null;
let renewal = Promise.resolve();
const renew = () => {
if (leaseLost) return;
renewal = renewal
.then(() => renewNativeSessionExecutionLease(input))
.then(() => undefined)
.catch(async (error: unknown) => {
leaseLost =
error instanceof Error
? error
: new Error("native_session_lease_lost");
await cancelNativeSession(
input.runId,
"native session execution lease lost",
).catch(() => undefined);
});
};
const timer = setInterval(
renew,
NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS,
);
timer.unref?.();
return {
stop: async () => {
clearInterval(timer);
await renewal;
if (leaseLost) throw leaseLost;
},
};
}
export async function executePaperclipNativeSession(input: {
getFreshSessionHandoff?: () => Promise<string | null>;
/** Retire a retained transport so recovery can replace provider tool declarations. */
refreshTools?: boolean;
db: Db;
execution: NativeExecutionInput;
runnerInstanceId: string;
/** Trusted task identity from the heartbeat orchestration. */
conversationMode?: boolean;
/** Configured total turn bound; zero/unset is unlimited. */
turnTimeoutMs?: number;
leaseOwner?: string;
restartRecovery?: NativeRestartRecoveryClaim;
onSpawn?: (meta: {
pid: number;
processGroupId: number | null;
startedAt: string;
}) => Promise<void>;
/** Test seam at the provider boundary; production uses a qualified package backend. */
backend?: NativeSessionBackend;
useRunnerd?: boolean;
/** Paperclip adapter identity used to scope the durable goal projection. */
adapterType?: string;
/** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
onEvent?: (event: AdapterRuntimeEvent) => Promise<void>;
/** Only this run's registered private instruction entry.
* Probe at terminal; persist only after owned shutdown. */
instructionWorkingCopy?: NativeInstructionWorkingCopy;
/** Persist task-level continuity before a durable goal can outlive this run. */
onUsage?: (receipt: AdapterUsageCheckpoint) => Promise<void>;
onGoalCheckpoint?: (snapshot: PersistedNativeSession) => Promise<void>;
sessionGoalControl?: NativeSessionGoalControl | null;
resumeSessionGoalHeartbeat?: boolean;
preparationSpans?: NativeRunHistoricalSpan[];
/** Use a session-owned GitHub broker, rebound only after run ownership is acquired. */
managedGitHub?: boolean;
/** Resolved adapter env; the runner transport applies a provider allowlist before spawn. */
dotWorkspaceRoot?: string;
runnerEnvironment?: NodeJS.ProcessEnv;
/** Private grant materialization; never a user-configured host path. */
managedAiCredentialHome?: string;
managedAiCredentialIdentity?: string;
billingIdentity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">;
runnerExecutionTarget?: AdapterExecutionTarget | null;
/** Resolved per-run authorization; not an independent instance setting. */
runnerIngressAuthorized?: boolean;
runnerPublicUrl?: string | null;
runnerCaBundlePath?: string | null;
runnerRemoteBinaryPath?: string | null;
runnerRemoteCodexPath?: string | null;
runnerRemoteCodexNpmSpec?: string | null;
runnerRemoteProviderPackPath?: string | null;
stopTaskForReassignment?: (target: { companyId: string; issueId: string; agentId: string; runId: string | null }) => Promise<void>;
syncIssueExternalObjects?: (issueId: string) => Promise<void>;
enqueueWakeup?: (
agentId: string,
options: {
source: "assignment" | "automation";
triggerDetail: "system";
reason: "issue_assigned" | "issue_commented";
payload: Record<string, unknown>;
idempotencyKey: string;
requestedByActorType: "agent";
requestedByActorId: string;
contextSnapshot: Record<string, unknown>;
},
) => Promise<unknown>;
}): Promise<AdapterExecutionResult> {
const agentKeyId = input.runnerEnvironment?.PAPERCLIP_AGENT_KEY_ID;
if (agentKeyId) {
// Recovered pre-upgrade inputs also need the current public identity marker
// before deciding whether an already-running provider can be reused.
input = { ...input, execution: { ...input.execution,
binding: { ...input.execution.binding, agentKeyId },
} };
}
const runId = input.execution.binding.runId;
if (nativeSessionStartups.has(runId)) {
throw new Error("native_session_supervisor_busy");
}
// Register before the first asynchronous operation on either backend path.
// A duplicate execution must not replace the original startup handoff.
let resolveStartup!: (session: ActiveNativeSession | null) => void;
let settle!: () => void;
const startup: NativeSessionStartup = {
settled: new Promise<void>(resolve => { settle = resolve; }),
settle: () => settle(),
promise: new Promise<ActiveNativeSession | null>(resolve => { resolveStartup = resolve; }),
resolve: session => resolveStartup(session),
};
nativeSessionStartups.set(runId, startup);
let preparedInput: typeof input = input;
let cleanupStagedAttachments: () => Promise<void> = async () => undefined;
let sessionScopeId: string | null = null;
let ownsSessionScope = false;
let executionFailure: unknown;
const ownerScope = nativeSessionOwnerScope(
input.execution,
input.runnerExecutionTarget?.environmentId ?? null,
);
const ownerToken = Symbol("native execution owner");
try {
// Reserve across session identities before retiring or staging anything.
if (executingNativeOwnerScopes.has(ownerScope)) {
throw new Error("native_session_supervisor_busy");
}
executingNativeOwnerScopes.set(ownerScope, ownerToken);
await retireSupersededWarmNativeSessions(
ownerScope, nativeSessionScopeKey(input.execution),
);
if (input.execution.provider.kind === "openai_dot" && isCloudManagedInstance()
&& (!input.useRunnerd || input.runnerExecutionTarget?.kind !== "remote"
|| input.runnerExecutionTarget.transport !== "sandbox")) {
throw new Error("dot_cloud_requires_managed_runner");
}
if (!input.useRunnerd) {
return await executePaperclipNativeSessionWithinScope(input);
}
// The session scope is unaffected by appending server-staged attachment
// descriptors. Claim it before any workspace scrub/write so a duplicate
// execution cannot truncate or replace the active turn's staging inode.
sessionScopeId = nativeSessionScopeKey(input.execution);
if (executingRunnerdSessionScopes.has(sessionScopeId)) {
throw new Error("native_session_supervisor_busy");
}
executingRunnerdSessionScopes.set(
sessionScopeId,
input.execution.binding.runId,
);
ownsSessionScope = true;
// The shutdown sweep can have removed an idle owner while its remote
// checkpoint is still being saved. The scope reservation also prevents
// a later sweep from closing an owner this turn is about to acquire.
await closingWarmNativeSessions.get(sessionScopeId);
if (input.execution.provider.kind === "openai_dot") {
return await executePaperclipNativeSessionWithinScope(input);
}
const workspaceRoot = input.execution.workspace.cwd;
if (workspaceRoot === null) throw new Error("native_workspace_required");
const targetKind = input.runnerExecutionTarget?.kind ?? "local";
const chatAttachmentReadScope = new NativeChatAttachmentReadScope({
db: input.db,
binding: input.execution.binding,
workspaceRoot,
executionTargetKind: targetKind,
});
preparedInput = { ...input, chatAttachmentReadScope };
cleanupStagedAttachments = () => chatAttachmentReadScope.close();
const attachmentStage = await stageNativeRunnerWakeAttachments({
db: input.db,
binding: {
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
runId: input.execution.binding.runId,
agentId: input.execution.binding.agentId,
workspaceRoot,
executionTargetKind: targetKind,
},
});
cleanupStagedAttachments = async () => {
const cleanupResults = await Promise.allSettled([
chatAttachmentReadScope.close(),
attachmentStage.cleanup(),
]);
if (cleanupResults.some((result) => result.status === "rejected")) {
throw new Error("paperclip_runner_attachment_staging_cleanup_failed");
}
};
const stagedPrompt = renderNativeRunnerStagedAttachmentPrompt(
attachmentStage.attachments,
);
if (stagedPrompt) {
preparedInput = {
...preparedInput,
execution: parseNativeExecutionInput({
...input.execution,
task: {
...input.execution.task,
prompt: `${input.execution.task.prompt}\n\n${stagedPrompt}`,
},
}),
};
}
return await executePaperclipNativeSessionWithinScope(preparedInput);
} catch (error) {
executionFailure = error;
throw error;
} finally {
if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) {
executingNativeOwnerScopes.delete(ownerScope);
}
startup.resolve(null);
startup.settle();
if (nativeSessionStartups.get(runId) === startup) {
nativeSessionStartups.delete(runId);
}
if (
ownsSessionScope &&
sessionScopeId !== null &&
executingRunnerdSessionScopes.get(sessionScopeId) ===
input.execution.binding.runId
) {
executingRunnerdSessionScopes.delete(sessionScopeId);
}
try {
await cleanupStagedAttachments();
} catch (cleanupError) {
// Cleanup is a confidentiality incident, but it occurs after the native
// provider may already have completed the turn. Reclassifying that turn
// as failed could replay provider side effects. Emit a private runtime
// health event while preserving the provider result/error disposition.
await input
.onEvent?.({
eventType: "native.attachment_staging_cleanup_failed",
level: "error",
message: "Native inbound attachment staging cleanup failed.",
payload: {
runId: input.execution.binding.runId,
issueId: input.execution.binding.issueId,
executionAlreadyFailed: executionFailure !== undefined,
},
})
.catch(() => undefined);
}
}
}
async function executePaperclipNativeSessionWithinScope(
input: Parameters<typeof executePaperclipNativeSession>[0],
): Promise<AdapterExecutionResult> {
const identityRedactor = createAgentIdentityRedactor(input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY);
const redactIdentityText = (text: string) => redactSensitiveText(
identityRedactor.chunk("diagnostic", text) + identityRedactor.finish("diagnostic"),
);
if (
input.execution.provider.kind !== "codex" &&
input.execution.provider.kind !== "opencode" &&
input.execution.provider.kind !== "claude_managed" &&
input.execution.provider.kind !== "aws_agentcore" &&
input.execution.provider.kind !== "acpx" &&
input.execution.provider.kind !== "openai_dot"
) {
throw new Error("paperclip_runner_provider_unsupported");
}
if (
input.execution.provider.kind === "acpx" &&
["copilot"].includes(input.execution.provider.agent) &&
!resolveAcpxQualification(input.execution.provider, process.env)
) {
throw new Error(
"paperclip_runner_provider_unsupported: ACPX candidate requires exact host qualification authorization",
);
}
const preparationSpans = input.preparationSpans ?? [];
const preparationStarts = nativeRunPreparationStarts(
preparationSpans,
Date.now(),
);
const trace = createNativeRunTrace({
runId: input.execution.binding.runId,
startedAtMs: preparationStarts.runStartedAtMs,
onEvent: input.onEvent,
});
const toolTrace = createNativeToolTrace(trace);
const taskPrepareScope = trace.start("task.prepare", {
parentName: "task.run",
startedAtMs: preparationStarts.preparationStartedAtMs,
});
const environmentSpans = preparationSpans.filter(
(span) =>
span.name === "environment.acquire" ||
span.name === "environment.workspace.realize",
);
const environmentStartedAtMs = environmentSpans.reduce(
(earliest, span) => Math.min(earliest, span.startedAtMs),
Date.now(),
);
const environmentEndedAtMs = environmentSpans.reduce(
(latest, span) => Math.max(latest, span.endedAtMs),
environmentStartedAtMs,
);
const environmentScope =
environmentSpans.length > 0
? trace.start("environment.startup", {
parentName: "task.prepare",
startedAtMs: environmentStartedAtMs,
})
: null;
for (const span of preparationSpans) {
const rootMilestone = isNativeRunRootHistoricalSpan(span.name);
await trace.record({
...span,
parentName: rootMilestone
? "task.run"
: environmentSpans.includes(span)
? "environment.startup"
: "task.prepare",
});
}
if (environmentScope) {
await trace.end(environmentScope, { endedAtMs: environmentEndedAtMs });
}
let retainedTransition: VerifiedWarmTransitionBinding | undefined;
if (input.useRunnerd) {
retainedTransition = await migrateRunnerdStateRootForExecution({
db: input.db,
execution: input.execution,
allowVerifiedBackup:
input.runnerExecutionTarget?.kind === "remote" &&
input.runnerExecutionTarget.transport === "sandbox",
// A retained warm runner is deliberately still ready rather than
// suspended. Only the exact idle in-process owner may rotate that
// prior-run authority; after a hard restart the map is empty and the
// durable-state verifier continues to require a suspended runner.
allowRetainedWarmRunner: hasIdleWarmNativeSessionOwner(input),
allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote",
onLog: input.onLog,
restartRecovery: input.restartRecovery,
runnerExecutionTarget: input.runnerExecutionTarget,
});
}
const durableRunnerBinding = input.useRunnerd
? loadRunnerdDurableBinding(input.execution, retainedTransition)
: null;
const effectiveRunnerInstanceId =
durableRunnerBinding?.runnerInstanceId ?? input.runnerInstanceId;
if (effectiveRunnerInstanceId !== input.runnerInstanceId) {
await input.db
.update(heartbeatRuns)
.set({
runnerInstanceId: effectiveRunnerInstanceId,
updatedAt: new Date(),
})
.where(
and(
eq(heartbeatRuns.id, input.execution.binding.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
),
);
}
const leaseOwner =
input.leaseOwner ?? `${effectiveRunnerInstanceId}:${randomUUID()}`;
const controller = await currentNativeControllerIdentity();
const leaseNow = new Date();
const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000);
let attempt: number;
try {
attempt = await trace.measure(
"native.coordinator.claim",
() =>
input.db.transaction(async (tx) => {
const coordinator = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(
nativeRunFinalizations.companyId,
input.execution.binding.companyId,
),
eq(
nativeRunFinalizations.issueId,
input.execution.binding.issueId,
),
),
)
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (!coordinator)
throw new Error("native_finalization_coordinator_missing");
const leaseNow = new Date();
const leaseExpiresAt = new Date(
leaseNow.getTime() + NATIVE_SESSION_EXECUTION_LEASE_TTL_MS,
);
// A durable result means provider execution already completed. The
// recovery/finalization path must reconcile it; never reacquire a
// provider session and execute the turn a second time.
if (coordinator.resultId)
throw new NativeResultPendingFinalizationError();
const boundRun = await tx
.select({
retryOfRunId: heartbeatRuns.retryOfRunId,
scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
agentId: heartbeatRuns.agentId,
companyId: heartbeatRuns.companyId,
nativeIssueId: heartbeatRuns.nativeIssueId,
resultJson: heartbeatRuns.resultJson,
runtimeMode: heartbeatRuns.runtimeMode,
status: heartbeatRuns.status,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.execution.binding.runId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (
!boundRun ||
boundRun.runtimeMode !== "native" ||
boundRun.companyId !== input.execution.binding.companyId ||
boundRun.agentId !== input.execution.binding.agentId ||
boundRun.nativeIssueId !== input.execution.binding.issueId
) {
throw new Error("native_execution_binding_changed");
}
// A cancellation can win after heartbeat dispatch admission but
// before this claim. Never revive a terminal run or a settled startup.
if (boundRun.status !== "running" || boundRun.resultJson?.startupCancellation ||
coordinator.phase === "terminal_failure") {
throw new NativeCancellationPendingRecoveryError();
}
const cancellationIntent = record(
record(boundRun.resultJson).nativeCancellation,
);
if (
cancellationIntent.scope === "run" &&
(cancellationIntent.dispatchState === "pending" ||
cancellationIntent.dispatchState === "acknowledged")
) {
const intentMatchesBinding =
cancellationIntent.schema ===
"paperclip.native-cancellation.v1" &&
cancellationIntent.companyId ===
input.execution.binding.companyId &&
cancellationIntent.runId === input.execution.binding.runId &&
cancellationIntent.issueId === input.execution.binding.issueId;
if (!intentMatchesBinding)
throw new Error("native_cancellation_intent_conflict");
// Run cancellation remains a claim fence after dispatch ack until
// the heartbeat cancellation path terminalizes the run.
throw new NativeCancellationPendingRecoveryError();
}
if (["committed", "applied"].includes(coordinator.phase))
throw new Error("native_run_already_committed");
if (
coordinator.leaseOwner &&
coordinator.leaseOwner !== leaseOwner &&
coordinator.leaseExpiresAt &&
coordinator.leaseExpiresAt > leaseNow
)
throw new Error("native_finalization_lease_busy");
const recovering = input.restartRecovery;
if (
recovering &&
(recovering.runId !== coordinator.runId ||
recovering.leaseOwner !== leaseOwner ||
coordinator.leaseOwner !== leaseOwner ||
coordinator.controllerBootId !== controller.bootId ||
coordinator.controllerPid !== controller.pid ||
coordinator.controllerGeneration !==
recovering.controllerGeneration)
) {
throw new Error("native_restart_recovery_claim_changed");
}
let incidentAttempts = coordinator.attempt;
if (
boundRun.retryOfRunId &&
boundRun.scheduledRetryReason === "native_safe_replacement"
) {
const [predecessor] = await tx
.select()
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, boundRun.retryOfRunId),
eq(
nativeRunFinalizations.companyId,
input.execution.binding.companyId,
),
eq(
nativeRunFinalizations.issueId,
input.execution.binding.issueId,
),
),
);
if (
!predecessor ||
predecessor.failureDetail?.successorRunId !==
input.execution.binding.runId ||
predecessor.phase !== "terminal_failure"
)
throw new Error("native_replacement_lineage_invalid");
incidentAttempts = Math.max(incidentAttempts, predecessor.attempt);
}
const nextAttempt = nextNativeProviderAttempt(
incidentAttempts,
recovering?.kind,
);
if (nextAttempt > 3)
throw new Error("native_session_retry_exhausted");
const nextControllerGeneration = recovering
? recovering.controllerGeneration
: coordinator.controllerBootId === controller.bootId
? Math.max(1, coordinator.controllerGeneration)
: coordinator.controllerGeneration + 1;
const claimed = await tx
.update(nativeRunFinalizations)
.set({
phase: "observed",
attempt: nextAttempt,
leaseOwner,
leaseExpiresAt,
controllerBootId: controller.bootId,
controllerPid: controller.pid,
controllerProcessStartedAt: controller.processStartedAt,
controllerGeneration: nextControllerGeneration,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: leaseNow,
})
.where(
and(
eq(nativeRunFinalizations.runId, coordinator.runId),
eq(
nativeRunFinalizations.companyId,
input.execution.binding.companyId,
),
eq(
nativeRunFinalizations.issueId,
input.execution.binding.issueId,
),
eq(nativeRunFinalizations.attempt, coordinator.attempt),
eq(nativeRunFinalizations.phase, coordinator.phase),
),
)
.returning({ runId: nativeRunFinalizations.runId })
.then((rows) => rows[0] ?? null);
if (!claimed) throw new Error("native_session_lease_lost");
await tx
.update(heartbeatRuns)
.set({
nativePhase: "observed",
nativePhaseUpdatedAt: leaseNow,
updatedAt: leaseNow,
})
.where(eq(heartbeatRuns.id, coordinator.runId));
return nextAttempt;
}),
{ parentName: "task.prepare" },
);
await trace.end(taskPrepareScope);
} catch (error) {
await trace.end(taskPrepareScope, { outcome: "failed" });
await trace.finish("failed");
throw error;
}
const controlPlaneInstanceId = `${effectiveRunnerInstanceId}:control`;
const planSynchronizations: PlanSynchronization[] = [];
const upsertPlanSynchronization = (
synchronization: PlanSynchronization,
): void => {
const existingIndex = planSynchronizations.findIndex(
(candidate) => candidate.eventId === synchronization.eventId,
);
if (existingIndex >= 0) {
planSynchronizations[existingIndex] = synchronization;
return;
}
planSynchronizations.push(synchronization);
};
const recordPlanSynchronization = async (event: {
sourceEventId: string;
turnId?: string;
eventType: string;
payload: Record<string, unknown>;
}) => {
const synchronization = await synchronizeCompletedProviderPlan({
db: input.db,
execution: input.execution,
event,
});
if (!synchronization) return;
upsertPlanSynchronization(synchronization);
const activity = await persistActivity(input.db, {
companyId: input.execution.binding.companyId,
actorType: "agent",
actorId: input.execution.binding.agentId,
agentId: input.execution.binding.agentId,
runId: input.execution.binding.runId,
issueId: input.execution.binding.issueId,
action: "issue.document_updated",
entityType: "issue",
entityId: input.execution.binding.issueId,
details: {
key: "plan",
source: "native_plan_synchronization",
synchronization,
},
});
publishActivity(activity.publication);
if (input.onLog)
await input.onLog(
"stdout",
`${JSON.stringify({ type: "paperclip.plan.synchronization", synchronization })}\n`,
);
};
let nativeSessionExecuteStartedAtMs = Date.now();
let sessionStartedAtMs: number | null = null;
let sessionStartupMode: "bootstrap" | "resume" | null = null;
let turnSubmittedAtMs: number | null = null;
let turnStartedAtMs: number | null = null;
let firstAgentEventRecorded = false;
let providerUsageLimitObserved = false;
const providerFailureObservation = createNativeProviderFailureObservation(input.execution.provider);
let turnCompletedAtMs: number | null = null;
let runnerSessionStartupScope: NativeRunSpanScope | null = null;
let agentTurnScope: NativeRunSpanScope | null = null;
let taskSettleScope: NativeRunSpanScope | null = null;
const governedWaitObservation = createGovernedWaitEventObservation(
resolvePendingGovernedWait,
);
const projectSessionGoalEvent = (event: PrpEvent) =>
applyRunnerGoalPrpEvent(
input.db,
{
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
agentId: input.execution.binding.agentId,
adapterType:
input.adapterType ??
(input.useRunnerd ? "paperclip_runner" : "codex_local"),
},
{
eventType: event.eventType,
sourceInstanceId: event.sourceInstanceId,
sourceRunId: event.runId,
sourceSeq: event.sourceSeq,
payload: event.payload,
},
);
const liveQuestions = createLocalNativeQuestionBridge({
db: input.db,
binding: { ...input.execution.binding, normalizedSessionId: nativeSessionKey(input.execution), runnerSourceInstanceId: effectiveRunnerInstanceId },
resolve: resolveNativeRuntimeRequest,
});
let observedAccountingUsage: Record<string, unknown> | null = null;
let observedAccountingTurn: string | undefined;
let nativeAccountingComplete = false;
// OpenCode and ACPX report their current turn; Codex reports a cumulative run delta.
// Rebuild per-turn state from the durable event log when a controller resumes.
const turnAccounting = ["opencode", "acpx"].includes(input.execution.provider.kind) ? createNativeTurnAccounting(input.execution.provider) : null;
if (turnAccounting) {
const history = await input.db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, input.execution.binding.companyId), eq(heartbeatRunEvents.runId, input.execution.binding.runId),
or(sql`${heartbeatRunEvents.payload}->'prpEvent'->'payload'->>'kind' = 'usage'`,
inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"])),
)).orderBy(heartbeatRunEvents.seq);
for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent);
}
const persistAccountingUsage = async (usage: Record<string, unknown> | null, complete: boolean) => {
const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" });
const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined;
complete = complete && accountingUsage !== undefined;
const costStatus = complete && usage?.accountingCostIncomplete !== true
? input.execution.provider.kind === "aws_agentcore" ? "estimated" as const : undefined
: "unpriced" as const;
const billing = resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity);
const costUsdExact = typeof usage?.accountingCostUsdExact === "string" ? usage.accountingCostUsdExact : undefined;
await input.onUsage?.({ usage: accountingUsage, ...billing, complete, usageBasis: "per_run", costUsdExact,
model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costStatus });
if (!input.onUsage) await input.db.update(heartbeatRuns).set({
costAccountingPending: true,
usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({
...accountingUsage, ...billing, accountingReceiptReady: complete, model: input.execution.provider.model ?? "unknown",
costUsd: accountingCost ?? null, costUsdExact: costUsdExact ?? null, costStatus: costStatus ?? null, usageSource: "per_run",
})}::jsonb`,
}).where(and(eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), isNull(heartbeatRuns.costAccountedAt)));
};
// appendEvent awaits this receipt before the runtime may commit completeRun.
// Replayed events must rebuild it too after a stop between event and receipt.
const observeAccountingEvent = async (event: PrpEvent) => {
if (turnAccounting) {
if (!turnAccounting.observe(event)) return;
const snapshot = turnAccounting.snapshot();
observedAccountingUsage = snapshot.usage;
observedAccountingTurn = snapshot.turnId;
await persistAccountingUsage(snapshot.usage, snapshot.complete);
return;
}
const payload = record(event.payload);
const usage = record(payload.usage);
const delta = record(usage.runDelta);
if (event.eventType === "turn.started") {
observedAccountingTurn = undefined;
if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false);
}
// A warm attachment seeds a zero snapshot before any provider report.
// Only a fresh run delta for this turn can close its accounting. A later
// malformed report invalidates completeness instead of retaining a
// smaller earlier total or falling back to cumulative session usage.
if (payload.kind === "usage") {
observedAccountingTurn = undefined;
if (Object.hasOwn(usage, "runDelta")) {
observedAccountingUsage = { runDelta: delta, runDeltaComplete: usage.runDeltaComplete };
if (normalizeNativeUsage(observedAccountingUsage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })) {
observedAccountingTurn = event.turnId;
}
}
if (observedAccountingUsage) await persistAccountingUsage(observedAccountingUsage, false);
}
if (["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType)
&& observedAccountingUsage && observedAccountingTurn !== undefined && event.turnId === observedAccountingTurn) {
await persistAccountingUsage(observedAccountingUsage, true);
}
};
let completedConversationReply: PrpEvent | null = null;
const assertControllerActive = () => {
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) {
throw new NativeControllerDetachedForRestartError();
}
};
const controlPlane = new PaperclipControlPlanePort(
input.db,
{
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
runId: input.execution.binding.runId,
agentId: input.execution.binding.agentId,
sessionId: nativeSessionKey(input.execution),
completionContractId: input.execution.completionContract.id,
completionContractSha256: input.execution.completionContract.sha256,
sourceInstanceId: effectiveRunnerInstanceId,
controlPlaneSourceInstanceId: controlPlaneInstanceId,
},
{
privateKeyPem: input.runnerEnvironment?.PAPERCLIP_AGENT_PRIVATE_KEY,
assertControllerActive,
onCommittedEvent: async (event) => {
await observeAccountingEvent(event);
await toolTrace.observe(event);
if (event.eventType === "item.completed" &&
record(event.payload).kind === "agentMessage" &&
record(event.payload).channel === "final") {
completedConversationReply = event;
}
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
providerFailureObservation.observe(event);
const eventAtMs = Date.parse(event.emittedAt);
const milestoneAtMs = Number.isFinite(eventAtMs)
? eventAtMs
: Date.now();
if (
event.eventType === "session.started" &&
sessionStartedAtMs === null
) {
sessionStartedAtMs = milestoneAtMs;
sessionStartupMode = "bootstrap";
await trace.record({
name: "runner.session.bootstrap",
parentName: "runner.session.startup",
startedAtMs: nativeSessionExecuteStartedAtMs,
endedAtMs: milestoneAtMs,
});
}
if (
event.eventType === "turn.submitted" &&
turnSubmittedAtMs === null
) {
turnSubmittedAtMs = milestoneAtMs;
// A recovered provider session does not emit session.started again. In
// that case the first durable turn.submitted event is the earliest
// transport-neutral proof that runnerd reattached to the exact
// provider session and is ready for work. Keep that startup time out of
// runner.turn.submit so cold-resume latency is visible as its own span.
if (sessionStartedAtMs === null) {
await trace.record({
name: "runner.session.resume",
parentName: "runner.session.startup",
startedAtMs: nativeSessionExecuteStartedAtMs,
endedAtMs: milestoneAtMs,
attributes: {
provider: input.execution.provider.kind,
strategy: "exact_provider_session",
},
});
sessionStartedAtMs = milestoneAtMs;
sessionStartupMode = "resume";
}
await trace.record({
name: "runner.turn.submit",
parentName: "runner.session.startup",
startedAtMs: sessionStartedAtMs,
endedAtMs: milestoneAtMs,
});
if (runnerSessionStartupScope) {
trace.annotate(runnerSessionStartupScope, {
mode: sessionStartupMode ?? "bootstrap",
});
await trace.end(runnerSessionStartupScope, {
endedAtMs: milestoneAtMs,
});
}
agentTurnScope = trace.start("agent.turn", {
parentName: "native.session.execute",
startedAtMs: milestoneAtMs,
attributes: { provider: input.execution.provider.kind },
});
trace.activate(agentTurnScope);
}
if (event.eventType === "turn.started" && turnStartedAtMs === null) {
turnStartedAtMs = milestoneAtMs;
await trace.record({
name: "provider.turn.queue",
parentName: "agent.turn",
startedAtMs: turnSubmittedAtMs ?? milestoneAtMs,
endedAtMs: milestoneAtMs,
});
}
if (!firstAgentEventRecorded && turnStartedAtMs !== null) {
const kind = firstMeaningfulAgentEventKind(event);
if (kind) {
firstAgentEventRecorded = true;
await trace.record({
name: "provider.time_to_first_agent_event",
parentName: "agent.turn",
startedAtMs: turnStartedAtMs,
endedAtMs: milestoneAtMs,
attributes: { eventKind: kind },
});
}
}
if (
[
"turn.completed",
"turn.failed",
"turn.interrupted",
"turn.cancelled",
].includes(event.eventType) &&
turnCompletedAtMs === null
) {
turnCompletedAtMs = milestoneAtMs;
if (!agentTurnScope) {
agentTurnScope = trace.start("agent.turn", {
parentName: "native.session.execute",
startedAtMs:
turnSubmittedAtMs ??
turnStartedAtMs ??
nativeSessionExecuteStartedAtMs,
attributes: { provider: input.execution.provider.kind },
});
trace.activate(agentTurnScope);
}
const outcome =
event.eventType === "turn.completed" ? "ok" : "failed";
await trace.end(agentTurnScope, {
endedAtMs: milestoneAtMs,
outcome,
});
taskSettleScope = trace.start("task.settle", {
parentName: "task.run",
startedAtMs: milestoneAtMs,
});
trace.activate(taskSettleScope);
}
if (input.onLog)
await input.onLog(
"stdout",
`${JSON.stringify({ type: "paperclip.prp.event", event })}\n`,
);
if (
input.onEvent &&
[
"session.capabilities.updated",
"session.goal.snapshot",
"session.goal.updated",
"session.goal.cleared",
"turn.started",
"turn.completed",
"turn.failed",
"turn.interrupted",
"turn.cancelled",
].includes(event.eventType)
) {
await input.onEvent({
eventType: event.eventType,
stream: "system",
level: event.eventType.includes("failed") ? "error" : "info",
payload: {
...(event.payload as Record<string, unknown>),
prpSourceSeq: event.sourceSeq,
prpSourceInstanceId: event.sourceInstanceId,
},
});
}
const inputMetric = runtimeInputLifecycleMetric(event);
if (inputMetric && input.onLog) {
await input.onLog(
"stdout",
`${JSON.stringify({
type: "paperclip.runtime_input.metric",
...inputMetric,
})}\n`,
);
}
const questionFallback = await materializeRuntimeQuestionFallback({
db: input.db,
binding: input.execution.binding,
event,
});
if (questionFallback) {
if (input.onLog) {
const origin = record(record(record(event.payload).request).origin);
await input.onLog(
"stdout",
`${JSON.stringify({
type: "paperclip.runtime_input.metric",
outcome:
record(event.payload).reason === "durable_handoff"
? "durable_handoff_materialized"
: "provider_loss_materialized",
requestId: record(event.payload).requestId,
interactionId: questionFallback.interaction.id,
adapter:
typeof origin.adapter === "string"
? origin.adapter
: "unknown",
})}\n`,
);
}
}
await governedWaitObservation.observe(
event,
event.eventType === "item.completed" || questionFallback !== null,
);
await recordPlanSynchronization(
event as {
sourceEventId: string;
turnId?: string;
eventType: string;
payload: Record<string, unknown>;
},
);
},
onDuplicateEvent: async (event) => {
await observeAccountingEvent(event);
// A crash can happen after the event commit but before its callback
// finishes. Recover only idempotent durable projections here; activity,
// publication, logging, trace, and metric effects remain committed-only.
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
providerFailureObservation.observe(event);
const questionFallback = await materializeRuntimeQuestionFallback({
db: input.db,
binding: input.execution.binding,
event,
});
await governedWaitObservation.observe(
event,
event.eventType === "item.completed" || questionFallback !== null,
);
const planSynchronization = await synchronizeCompletedProviderPlan({
db: input.db,
execution: input.execution,
event: event as {
sourceEventId: string;
turnId?: string;
eventType: string;
payload: Record<string, unknown>;
},
});
if (planSynchronization) {
upsertPlanSynchronization(planSynchronization);
}
},
},
);
let native: Awaited<ReturnType<typeof executeNativeSession>>;
let releaseGoalController: (() => void) | null = null;
const releaseRegisteredGoalController = () => {
// The controller is assigned from the asynchronous onSession callback.
// Keep release idempotent without relying on TypeScript's synchronous
// control-flow model for that callback assignment.
const release: unknown = releaseGoalController;
if (typeof release === "function") release();
releaseGoalController = null;
};
const lifecyclePolicy = input.execution.session?.lifecyclePolicy ?? {
mode: "per_turn" as const,
idleTimeoutMs: null,
};
const warmSessionId =
lifecyclePolicy.mode === "warm"
? nativeSessionScopeKey(input.execution)
: null;
const warmConfigDigest =
lifecyclePolicy.mode === "warm"
? nativeSessionConfigDigest(
input.execution,
input.runnerExecutionTarget?.kind ?? "local",
)
: null;
const configuredEnvironmentDigest = createHash("sha256")
.update(JSON.stringify(configuredEnvironment(input.runnerEnvironment)))
.digest("hex");
const warmSessionOwnerToken = Symbol(
`native-warm-session:${input.execution.binding.runId}`,
);
let existingWarmSession: NativeSession | undefined;
let managedCredentialSession: NativeSession | undefined;
let githubAccess: NativeGitHubAccess | undefined;
let releaseGitHubRun: (() => void) | undefined;
let persistedWarmSession: PersistedNativeSession | null | undefined;
if (warmSessionId !== null && warmConfigDigest !== null) {
const entry = warmNativeSessions.get(warmSessionId);
if (entry) {
if (entry.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
entry.preparingRunId = undefined;
// Old run-scoped environments still require process replacement. A
// session-owned broker can change run authority without replacing it.
const hasBrokerCapability = Boolean(
!input.managedGitHub && input.runnerEnvironment?.PAPERCLIP_GITHUB_BROKER_TOKEN,
);
const credentialRunChanged =
Boolean(entry.credentialRunId) !== hasBrokerCapability ||
(hasBrokerCapability &&
entry.credentialRunId !== input.execution.binding.runId);
if (
input.refreshTools === true ||
(input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" &&
(entry.computerTarget?.resourceAuthority.computerId !== input.runnerExecutionTarget.resourceAuthority.computerId ||
entry.computerTarget?.resourceAuthority.ownerId !== input.runnerExecutionTarget.resourceAuthority.ownerId)) ||
entry.closeOnReleaseReason !== undefined ||
entry.configDigest !== warmConfigDigest ||
entry.configuredEnvironmentDigest !== configuredEnvironmentDigest ||
entry.instructionRoot !== input.instructionWorkingCopy?.root ||
entry.managedAiCredentialIdentity !== input.managedAiCredentialIdentity ||
credentialRunChanged ||
Boolean(entry.githubAccess) !== Boolean(input.managedGitHub) ||
entry.githubAccess?.ready === false ||
entry.githubAuthenticationMode !==
input.runnerEnvironment?.PAPERCLIP_GITHUB_AUTH_MODE ||
entry.networkAccess !==
(input.runnerEnvironment?.PAPERCLIP_RUNNER_NETWORK_ACCESS ===
"enabled")
) {
const preparedCopy = entry.instructionPreparationRunId === input.execution.binding.runId;
if (entry.busy && !preparedCopy) throw new Error("native_session_supervisor_busy");
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.busy = true;
entry.closeOnReleaseReason = "warm native session configuration changed";
try {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason, input.execution.binding.runId);
warmNativeSessions.delete(warmSessionId);
} finally { entry.busy = false; }
if (preparedCopy) throw new Error("native_instruction_preparation_configuration_changed");
persistedWarmSession = loadWarmNativeCheckpoint(
input.execution,
warmConfigDigest,
input.runnerExecutionTarget?.kind ?? "local",
);
} else {
if (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
if (entry.instructionWorkingCopy?.runId && entry.instructionPreparationRunId !== input.execution.binding.runId) {
throw new Error("native_instruction_materialization_not_claimed");
}
if (entry.instructionPreparationRunId !== undefined && await entry.instructionWorkingCopy!.hasChanges()) {
entry.closeOnReleaseReason = "warm instruction copy changed during preparation";
try {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
warmNativeSessions.delete(warmSessionId);
} finally { entry.busy = false; }
throw new Error("native_instruction_preparation_copy_changed");
}
entry.instructionPreparationRunId = undefined;
entry.instructionWorkingCopy = input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy;
entry.busy = true;
entry.runId = input.execution.binding.runId;
entry.computerTarget = input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined;
entry.ownerToken = warmSessionOwnerToken;
entry.environmentId =
input.runnerExecutionTarget?.environmentId ?? null;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
existingWarmSession = entry.session;
githubAccess = entry.githubAccess;
}
} else {
persistedWarmSession = loadWarmNativeCheckpoint(
input.execution,
warmConfigDigest,
input.runnerExecutionTarget?.kind ?? "local",
);
}
}
async function resolvePendingGovernedWait() {
const continuingInteractionIds = continuingPendingInteractionIds(
input.execution,
);
const interaction = await input.db
.select({
id: issueThreadInteractions.id,
title: issueThreadInteractions.title,
summary: issueThreadInteractions.summary,
})
.from(issueThreadInteractions)
.where(
and(
eq(
issueThreadInteractions.companyId,
input.execution.binding.companyId,
),
eq(issueThreadInteractions.issueId, input.execution.binding.issueId),
or(
eq(
issueThreadInteractions.sourceRunId,
input.execution.binding.runId,
),
and(
eq(issueThreadInteractions.kind, "connection_intent"),
eq(
issueThreadInteractions.createdByAgentId,
input.execution.binding.agentId,
),
),
...(continuingInteractionIds.length > 0
? [inArray(issueThreadInteractions.id, continuingInteractionIds)]
: []),
),
eq(issueThreadInteractions.status, "pending"),
activeIssueInteractionCondition({ runId: input.execution.binding.runId }),
// Live provider questions resume their current turn; only durable
// wake-based cards park it. A timeout creates a separate fallback.
sql`not (${issueThreadInteractions.kind} = 'ask_user_questions' and ${issueThreadInteractions.continuationPolicy} = 'none' and coalesce(${issueThreadInteractions.idempotencyKey}, '') like 'paperclip-runner-question:%')`,
),
)
.orderBy(
desc(issueThreadInteractions.createdAt),
desc(issueThreadInteractions.id),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (interaction)
return nativeGovernedWaitResult({
interaction,
completionContract: input.execution.completionContract.contract,
});
// A ready connection can become installed after the provider snapshot was
// pinned. Its already-durable wake is also a valid reason to end this turn.
const [refresh] = await input.db
.select({
id: agentWakeupRequests.id,
key: agentWakeupRequests.idempotencyKey,
})
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, input.execution.binding.companyId),
eq(agentWakeupRequests.agentId, input.execution.binding.agentId),
like(
agentWakeupRequests.idempotencyKey,
`connection-intent:tools:${input.execution.binding.runId}:%`,
),
notInArray(agentWakeupRequests.status, [
"skipped",
"failed",
"cancelled",
]),
),
)
.limit(1);
return refresh?.key
? nativeToolsRefreshWaitResult({
wakeId: refresh.id,
key: refresh.key,
completionContract: input.execution.completionContract.contract,
})
: null;
}
const runnerExecution: NativeExecutionInput =
input.execution.schema !== "paperclip.native-execution-input.v6" && input.useRunnerd && input.runnerExecutionTarget?.kind === "remote"
? {
...input.execution,
workspace: {
...input.execution.workspace,
cwd: input.runnerExecutionTarget.remoteCwd,
},
}
: input.execution;
let remoteLaunchAttempted = false;
let remoteBackendPreparationStarted = false;
const leaseRenewal = startNativeSessionExecutionLeaseRenewal({
db: input.db,
runId: input.execution.binding.runId,
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
leaseOwner,
attempt,
controller,
});
try {
if (input.managedGitHub && input.execution.provider.kind !== "openai_dot") {
if (!input.execution.workspace.cwd) throw new Error("native_workspace_required");
githubAccess ??= await createNativeGitHubAccess({
scope: input.execution.binding,
target: input.runnerExecutionTarget,
cwd: input.execution.workspace.cwd,
env: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment),
resolveCredentials: (binding) => resolveGitHubOperationCredentials(input.db, binding),
onLog: input.onLog,
});
releaseGitHubRun = githubAccess.activate(input.execution.binding);
input = { ...input, runnerEnvironment: { ...input.runnerEnvironment, ...githubAccess.env } };
}
const expectedCurrentWakeComments = await resolveCurrentWakeCommentsBinding(
input.db,
input.execution.binding,
);
remoteBackendPreparationStarted = input.useRunnerd === true && input.backend === undefined;
const runnerdBackend =
input.useRunnerd && input.backend === undefined
? await createRunnerdBackend({
...input,
// Durable scope and prior-run verification use the controller's
// canonical workspace identity. createRunnerdBackend separately
// projects remoteCwd into the provider execution boundary.
execution: input.execution,
runnerInstanceId: effectiveRunnerInstanceId,
durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId,
onRemoteLaunchAttempt: () => { remoteLaunchAttempted = true; },
trace,
toolTrace,
})
: null;
const remoteCleanupLease = input.runnerExecutionTarget?.kind === "remote" &&
(input.runnerExecutionTarget.transport === "sandbox" || input.runnerExecutionTarget.transport === "computer") &&
input.runnerExecutionTarget.leaseId
? await input.db.select({ provider: environmentLeases.provider, providerLeaseId: environmentLeases.providerLeaseId })
.from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.execution.binding.companyId),
eq(environmentLeases.id, input.runnerExecutionTarget.leaseId),
)).then(rows => rows[0])
: null;
nativeSessionExecuteStartedAtMs = Date.now();
native = await trace.measure(
"native.session.execute",
async () => {
runnerSessionStartupScope = trace.start("runner.session.startup", {
parentName: "native.session.execute",
startedAtMs: nativeSessionExecuteStartedAtMs,
});
trace.activate(runnerSessionStartupScope);
const result = await trace.run(runnerSessionStartupScope, () =>
executeNativeSession({
resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner",
getFreshSessionHandoff: input.getFreshSessionHandoff,
onSessionAdmission: async () => {
// Invalidate prior stop evidence before a backend can spawn.
await appendHeartbeatRunEvent(input.db, {
companyId: input.execution.binding.companyId,
runId: input.execution.binding.runId,
agentId: input.execution.binding.agentId,
eventType: PROCESS_START_REQUESTED,
stream: "system",
level: "info",
message: "Native execution requested; prior local stop evidence no longer applies.",
});
},
input: runnerExecution,
remoteCleanupScope: remoteCleanupLease ? remoteLeaseCleanupScope(remoteCleanupLease) : undefined,
turnTimeoutMs: input.turnTimeoutMs,
bootstrapTimeoutMs: nativeSessionBootstrapTimeoutMs(input.execution.provider.kind, input.runnerExecutionTarget),
backend:
input.backend ??
runnerdBackend ??
createNativeSessionBackend(input.execution, {
runnerInstanceId: input.runnerInstanceId,
onSpawn: input.onSpawn,
opencodeEnvironment: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment),
acpxEnvironment: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment),
opencodeRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"opencode",
),
acpxRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"acpx",
),
}),
controlPlane,
runnerInstanceId: effectiveRunnerInstanceId,
controlPlaneInstanceId,
resolveGovernedWait: ({ event }) => {
assertControllerActive();
const result = governedWaitObservation.consume(event);
if (result) {
const startup = nativeSessionStartups.get(input.execution.binding.runId);
if (startup) startup.governedWait = true;
}
return result;
},
resolveMissingResult: async ({ terminalEvent }) => {
// Governed waits take precedence over an ordinary chat reply.
// Execution tasks still require their normal semantic finish.
if (terminalEvent.eventType !== "turn.completed") return null;
const governedWait = await resolvePendingGovernedWait();
if (governedWait) return governedWait;
if (nativeProviderLifecycle(input.execution.provider)?.planWait) {
const planWait = await readNativePlanWait(input.db, input.execution.binding);
if (planWait?.source.terminalEventId === terminalEvent.sourceEventId) return planWait.result;
}
if (await readCompletedNativePermissionDecline(input.db, input.execution.binding, terminalEvent, input.execution.provider)) {
throw new NativePermissionDeclinedError();
}
const [conversation] = await input.db
.select({ agentId: issues.conversationAgentId })
.from(issues)
.where(and(
eq(issues.id, input.execution.binding.issueId),
eq(issues.companyId, input.execution.binding.companyId),
))
.limit(1);
return nativeConversationReplyResult({
conversation: conversation?.agentId === input.execution.binding.agentId,
terminalEvent,
replyEvent: completedConversationReply,
completionContract: input.execution.completionContract.contract,
});
},
existingSession: existingWarmSession,
persistedSession: persistedWarmSession,
keepSessionOpen: warmSessionId !== null,
sessionGoalControl: input.sessionGoalControl,
resumeSessionGoalHeartbeat: input.resumeSessionGoalHeartbeat,
// Every durable runner must finish its bounded suspension before
// the next run verifies and rotates the saved authority.
requireSessionCloseBeforeReturn: runnerdBackend !== null || input.instructionWorkingCopy !== undefined,
onSessionClosed: input.instructionWorkingCopy?.collectStopped,
onCheckpoint: async (snapshot) => {
if (snapshot.governedWait) {
const startup = nativeSessionStartups.get(input.execution.binding.runId);
if (startup) startup.governedWait = true;
}
snapshot = identityRedactor.redact(snapshot);
if (warmSessionId !== null && warmConfigDigest !== null) {
await persistWarmNativeCheckpoint(
input.execution,
warmConfigDigest,
snapshot,
);
}
if (
snapshot.goal ||
input.sessionGoalControl ||
input.resumeSessionGoalHeartbeat
) {
await input.onGoalCheckpoint?.(snapshot);
}
},
onPostCompletionEnrichmentFailure: async ({ stage, error }) => {
const detail = redactIdentityText(
error instanceof Error ? error.message : String(error),
).slice(-4_096);
await input.onLog?.(
"stderr",
`[paperclip-runner] post-completion ${stage} enrichment failed: ${detail}\n`,
);
},
onSessionQuarantined: async (reason) => {
await input.onLog?.(
"stderr",
`[paperclip-runner] warm native session quarantined: ${redactIdentityText(reason).slice(-1_000)}\n`,
);
},
onContinuityBreak: async (continuity) => {
const atMs = Date.now();
await trace.record({
name: "provider.session.continuity_break",
parentName: "native.session.execute",
startedAtMs: atMs,
endedAtMs: atMs,
outcome: "failed",
attributes: {
reason: continuity.reason,
previousDriverSessionId: continuity.previousDriverSessionId,
previousProviderSessionId:
continuity.previousProviderSessionId ?? "unavailable",
replacementDriverSessionId:
continuity.replacementDriverSessionId,
replacementProviderSessionId:
continuity.replacementProviderSessionId ?? "unavailable",
},
});
await input.onLog?.(
"stderr",
`[paperclip-runner] provider session continuity break: exact resume failed (${continuity.reason}); old driver session=${continuity.previousDriverSessionId}, old provider session=${continuity.previousProviderSessionId ?? "unavailable"}, replacement driver session=${continuity.replacementDriverSessionId}, replacement provider session=${continuity.replacementProviderSessionId ?? "unavailable"}\n`,
);
},
onSession: async (session) => {
if (session && input.managedAiCredentialHome) {
managedCredentialSession = runnerdBackend?.bindManagedSession(session) ?? session;
}
releaseRegisteredGoalController();
liveQuestions.close();
if (session?.goal) {
releaseGoalController = registerLiveRunnerGoalController(
{
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
agentId: input.execution.binding.agentId,
runId: input.execution.binding.runId,
},
{
control: async (control) => {
await applyNativeSessionGoalControl(session, control);
},
},
);
}
if (
session &&
warmSessionId !== null &&
warmConfigDigest !== null
) {
const existing = warmNativeSessions.get(warmSessionId);
if (existing) {
if (existing.ownerToken !== warmSessionOwnerToken) {
throw new Error("native_session_supervisor_busy");
}
existing.session = session;
} else
warmNativeSessions.set(warmSessionId, {
instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy,
instructionRoot: input.instructionWorkingCopy?.root,
agentId: input.execution.binding.agentId,
managedAiCredentialIdentity: input.managedAiCredentialIdentity,
githubAuthenticationMode:
input.runnerEnvironment?.PAPERCLIP_GITHUB_AUTH_MODE,
networkAccess:
input.runnerEnvironment
?.PAPERCLIP_RUNNER_NETWORK_ACCESS === "enabled",
githubAccess,
credentialRunId: !input.managedGitHub && input.runnerEnvironment
?.PAPERCLIP_GITHUB_BROKER_TOKEN
? input.execution.binding.runId
: undefined,
session,
runId: input.execution.binding.runId,
computerTarget: input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined,
ownerToken: warmSessionOwnerToken,
configDigest: warmConfigDigest,
configuredEnvironmentDigest,
ownerScope: nativeSessionOwnerScope(
input.execution, input.runnerExecutionTarget?.environmentId ?? null,
),
companyId: input.execution.binding.companyId,
environmentId:
input.runnerExecutionTarget?.environmentId ?? null,
busy: true,
idleTimer: null,
lastActivityAt: new Date().toISOString(),
});
const owner = warmNativeSessions.get(warmSessionId);
if (owner) {
owner.runId = input.execution.binding.runId;
owner.computerTarget = input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "computer" ? input.runnerExecutionTarget : undefined;
}
if (owner && input.instructionWorkingCopy?.runId && input.instructionWorkingCopy.checkpointWarm) owner.instructionCopy = {
runId: input.instructionWorkingCopy.runId, root: input.instructionWorkingCopy.root, targetIdentity: instructionTargetIdentity(input.runnerExecutionTarget),
collectStopped: input.instructionWorkingCopy.collectStopped,
};
} else if (!session && warmSessionId !== null) {
const existing = warmNativeSessions.get(warmSessionId);
// onSession(null) quarantines a transport that can no longer
// be reused. Remove only this execution's generation so a
// late failure cannot evict a successor session.
if (existing?.ownerToken === warmSessionOwnerToken) {
if (existing.idleTimer !== null) {
clearTimeout(existing.idleTimer);
}
warmNativeSessions.delete(warmSessionId);
}
}
if (session) {
const active: ActiveNativeSession = { session, cancelRequested: false };
activeNativeSessions.set(input.execution.binding.runId, active);
const startup = nativeSessionStartups.get(input.execution.binding.runId);
startup?.resolve(active);
if (startup?.stopRequested) {
// Publication wakes the Stop caller; wait for its durable ACK
// before unwinding. No provider turn may be submitted between
// session startup and the execution-owned cleanup below.
await startup.cancellationSettled;
// If startup exceeded the caller's deadline, its late handle
// must still be cancelled instead of escaping the Stop fence.
await cancelNativeSession(input.execution.binding.runId, "Stop requested during native startup");
throw new Error("native_finalization_missing: session returned no semantic result");
}
// Stop can win after the coordinator claim while the provider
// session is still opening. Publishing the handle before this
// read closes both sides of the race: earlier Stop is durable;
// later Stop can cancel this exact active session.
const [currentRun] = await input.db.select({
status: heartbeatRuns.status,
resultJson: heartbeatRuns.resultJson,
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.execution.binding.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
)).limit(1);
const cancellation = record(currentRun?.resultJson?.nativeCancellation);
if (!currentRun || currentRun.status !== "running" ||
currentRun.resultJson?.startupCancellation ||
(cancellation.scope === "run" &&
["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
await cancelNativeSession(input.execution.binding.runId, "Run stopped during native session startup");
// The execution-owned finally closes the session when this
// callback fails; no provider turn may follow publication.
throw new NativeCancellationPendingRecoveryError();
}
if (session.resolveRuntimeRequest) await liveQuestions.attach();
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) {
if (session.detachControllerForRestart) await detachActiveNativeSessionForRestart(active);
}
if (active.cancelRequested) throw new NativeCancellationPendingRecoveryError();
} else {
liveQuestions.close();
activeNativeSessions.delete(input.execution.binding.runId);
clearSteeringDeliveries(input.execution.binding.runId);
clearNativeRuntimeRequestResolutions(
input.execution.binding.runId,
);
}
},
}),
);
await trace.end(runnerSessionStartupScope, {
outcome:
result.terminal.runTerminalState === "succeeded" ? "ok" : "failed",
});
return result;
},
{ parentName: "task.run" },
);
// A detached consumer can resolve successfully after its stream closes.
// Only the replacement controller may settle the run or certify accounting.
assertControllerActive();
// Persist provider accounting before any workspace/issue finalization. A
// detached controller or failed finalizer must not lose a completed turn.
// session.usage() may be an attachment baseline, a partial report, or a
// cumulative session total. Use the observed run delta, and require its
// turn to match the terminal result before certifying it as complete.
if (turnAccounting) {
const snapshot = turnAccounting.finish(native.turnId);
observedAccountingUsage = snapshot.usage;
nativeAccountingComplete = snapshot.complete && snapshot.turnId === native.turnId;
} else nativeAccountingComplete = observedAccountingUsage !== null
&& observedAccountingTurn !== undefined && observedAccountingTurn === native.turnId;
native = { ...native, usage: observedAccountingUsage };
await persistAccountingUsage(native.usage, nativeAccountingComplete);
try {
await completeManagedNativeCredentialTurn(managedCredentialSession);
} catch {
// A durable result remains successful if optional credential refresh
// fails. Retire this owner so it cannot keep stale auth on a warm turn.
if (warmSessionId !== null && lifecyclePolicy.mode === "warm") {
await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true);
}
await input.onLog?.("stderr", "[paperclip-runner] managed credential refresh failed; provider session retired.\n");
}
if (native.terminal.runTerminalState === "succeeded") {
// A truncated, verified external-chat wake cannot settle from the
// provider's partial inline prompt. The run-scoped reader records a
// durable complete-page receipt; this fence revalidates that exact
// current snapshot before any native finalization can become authoritative.
await assertCurrentWakeCommentsRead(
input.db,
input.execution.binding,
expectedCurrentWakeComments,
);
}
await leaseRenewal.stop();
await trace.record({
name: "native.result.finalize",
parentName: "task.settle",
startedAtMs: turnCompletedAtMs ?? nativeSessionExecuteStartedAtMs,
endedAtMs: Date.now(),
});
liveQuestions.close();
activeNativeSessions.delete(input.execution.binding.runId);
clearSteeringDeliveries(input.execution.binding.runId);
clearNativeRuntimeRequestResolutions(input.execution.binding.runId);
} catch (error) {
// Preserve typed failure semantics while removing credentials before any
// persistence, reporting, logging, or rethrow at this boundary.
if (error instanceof Error) {
const originalMessage = error.message;
error.message = redactIdentityText(originalMessage);
if (error.stack) error.stack = redactIdentityText(originalMessage
? error.stack.split(originalMessage).join(error.message) : error.stack);
} else {
error = identityRedactor.redact(error);
}
if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) {
await leaseRenewal.stop().catch(() => undefined);
liveQuestions.close();
activeNativeSessions.delete(input.execution.binding.runId);
// Disconnecting deliberately ends the old event consumer. It is not a
// provider failure and must not overwrite the shutdown adoption record
// with a retry or release the still-live runner's lease.
throw new NativeControllerDetachedForRestartError();
}
if (remoteBackendPreparationStarted) {
try {
await retireUnlaunchedComputerAttempt(input.runnerExecutionTarget, remoteLaunchAttempted);
} catch {
// Failure to establish the exact tombstone leaves queued commands or
// a lost launch acknowledgement ambiguous. Never retry that owner.
error = new NativeSessionCleanupQuarantinedError();
}
}
const protocolIntegrityFailure =
error instanceof NativeSessionProtocolIntegrityError ? error : null;
const ownershipUnverified =
nativeSessionFailureSourceCode(error) ===
NATIVE_ADOPTED_RUNNER_AUTHENTICATION_TIMEOUT;
const attemptFailureStep = async (operation: () => unknown) => {
try {
await operation();
} catch (secondaryError) {
if (protocolIntegrityFailure === null) throw secondaryError;
}
};
const stoppedLeaseRenewal = leaseRenewal.stop().catch(() => undefined);
try {
await input.db
.update(nativeRunFinalizations)
.set({
controlDeadlineAt: new Date(
Date.now() + EXECUTION_CONTROL_DEADLINE_MS,
),
})
.where(
and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
),
);
const failedAtMs = Date.now();
const executionFailureMessage = redactIdentityText(
error instanceof Error ? error.message : String(error),
).slice(-4_096);
if (!taskSettleScope) {
taskSettleScope = trace.start("task.settle", {
parentName: "task.run",
startedAtMs: failedAtMs,
});
}
trace.activate(taskSettleScope);
liveQuestions.close();
activeNativeSessions.delete(input.execution.binding.runId);
clearSteeringDeliveries(input.execution.binding.runId);
clearNativeRuntimeRequestResolutions(input.execution.binding.runId);
// Stop before paperclip_finish is normal. Bounded provider teardown has
// finished; a missing result must not overwrite cancellation or trigger
// another turn to perform completion bookkeeping.
const stoppedBeforeFirstTurn = error instanceof Error && error.message === "native_session_cancelled";
if (protocolIntegrityFailure === null && error instanceof Error &&
(stoppedBeforeFirstTurn || error instanceof NativePermissionDeclinedError || error.message === "native_finalization_missing: session returned no semantic result")) {
const [stoppedRun] = await input.db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.execution.binding.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
eq(heartbeatRuns.nativeIssueId, input.execution.binding.issueId),
)).limit(1);
const stopIntent = record(stoppedRun?.resultJson?.nativeCancellation);
// The backend can reject the first turn while the Stop API is still
// recording its acknowledgement. Preserve that audited, exactly bound
// cancellation instead of racing it with a generic provider failure.
const pendingStartupStop = stoppedBeforeFirstTurn &&
stopIntent.schema === "paperclip.native-cancellation.v1" &&
stopIntent.companyId === input.execution.binding.companyId &&
stopIntent.runId === input.execution.binding.runId &&
stopIntent.issueId === input.execution.binding.issueId &&
stopIntent.scope === "run" &&
typeof stopIntent.intentAuditId === "string" && stopIntent.intentAuditId.length > 0 &&
["pending", "acknowledged"].includes(String(stopIntent.dispatchState));
if (stoppedRun && (pendingStartupStop || hasAcknowledgedNativeStopIntent(stoppedRun) || hasAcknowledgedNativeReassignmentStopIntent(stoppedRun))) {
const [settled] = await input.db.update(nativeRunFinalizations).set({
phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null,
leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null,
updatedAt: new Date(),
}).where(and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.companyId, input.execution.binding.companyId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
eq(nativeRunFinalizations.attempt, attempt),
isNull(nativeRunFinalizations.resultId),
)).returning({ runId: nativeRunFinalizations.runId });
if (settled) {
await stoppedLeaseRenewal;
if (warmSessionId !== null && lifecyclePolicy.mode === "warm") {
await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true);
}
error = new NativeCancellationPendingRecoveryError();
}
}
}
if (
error instanceof NativeResultPendingFinalizationError ||
error instanceof NativeCancellationPendingRecoveryError
) {
// This is not a provider failure and must not overwrite the durable
// result/coordinator state. The heartbeat boundary will either hand an
// already-materialized result to the finalizer or retain the durable
// cancellation intent for cancellation recovery.
if (taskSettleScope) {
await trace.end(taskSettleScope, { outcome: "ok" });
}
await trace.finish("ok");
throw error;
}
const now = new Date();
const classifiedFailureCode = nativeSessionFailureSourceCode(error);
const sourceFailureCode =
classifiedFailureCode === "native_event_replay_conflict"
? classifiedFailureCode
: providerUsageLimitObserved
? "native_provider_usage_limit"
: classifiedFailureCode;
const recoveryEvidence = await nativeProviderRecoveryEvidence({
db: input.db,
runId: input.execution.binding.runId,
sourceFailureCode,
});
const disposition = nativeSessionFailureDisposition(
attempt,
now,
sourceFailureCode,
);
const phase =
recoveryEvidence.recoveryMode === "ambiguous_state"
? ("terminal_failure" as const)
: disposition.phase;
const failureCode =
recoveryEvidence.recoveryMode === "ambiguous_state"
? sourceFailureCode
: disposition.failureCode;
const nextAttemptAt =
recoveryEvidence.recoveryMode === "ambiguous_state"
? null
: disposition.nextAttemptAt;
const recoveryProjection = nativeSessionRecoveryProjection({
phase,
failureCode,
agentId: input.execution.binding.agentId,
});
const { exhausted } = recoveryProjection;
const integrityFailure =
sourceFailureCode === "native_event_replay_conflict";
const message = redactIdentityText(
error instanceof Error ? error.message : String(error),
).slice(0, 2_000);
const sanitizedStderrTail = redactIdentityText(message).slice(-4_096);
// Set inside the transaction only when the write below genuinely
// transitions the run into "failed". Read after the transaction
// commits, so a rolled-back write never reports a false failure.
let terminalRunToReport: typeof heartbeatRuns.$inferSelect | null = null;
await input.db.transaction(async (tx) => {
await tx.execute(
sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`,
);
// Use the same issue-before-run lock order as admission. A late failure
// can terminalize its own run, but cannot change a reassigned task.
const [failureTask] = await tx
.select()
.from(issues)
.where(
and(
eq(issues.id, input.execution.binding.issueId),
eq(issues.companyId, input.execution.binding.companyId),
),
)
.for("update");
const updated = await tx
.update(nativeRunFinalizations)
.set({
phase,
controlDeadlineAt: null,
leaseOwner: null,
leaseExpiresAt: null,
recoveryState:
phase === "retryable_failure" ? "resuming_session" : "blocked",
failureCode,
failureDetail: identityRedactor.redact({
message,
originalFailureCode:
error instanceof NativeProviderTerminalFailure
? error.providerCode
: sourceFailureCode,
recoverable:
error instanceof NativeProviderTerminalFailure
? error.recoverable
: phase === "retryable_failure",
recoveryMode: recoveryEvidence.recoveryMode,
providerSessionEstablished:
recoveryEvidence.providerSessionEstablished,
providerEventsExist: recoveryEvidence.providerEventsExist,
checkpointExists: recoveryEvidence.checkpointExists,
recoveryOwner: recoveryProjection.recoveryOwner,
nextAction:
sourceFailureCode === "native_permission_declined"
? "Provider permission was declined. Inspect the task and give explicit direction before starting further provider work. Automatic recovery is stopped."
: sourceFailureCode === "native_provider_approval_required"
? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler."
: sourceFailureCode === "native_session_cleanup_quarantined"
? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE
: sourceFailureCode === "native_provider_terminal_failed"
? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation."
: recoveryEvidence.recoveryMode === "ambiguous_state"
? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden."
: integrityFailure
? "Inspect the persisted runner events and checkpoint for a source-sequence integrity conflict; automatic recovery is stopped."
: sourceFailureCode === "native_provider_usage_limit"
? "Restore model provider usage capacity, then explicitly retry the task. Automatic retries cannot resolve an exhausted provider allowance."
: ownershipUnverified
? "Inspect the retained runner's executable and authenticated connection. Do not replace its provider until ownership is safely resolved."
: exhausted
? "Inspect the persisted native session after its bounded resume budget was exhausted."
: recoveryEvidence.recoveryMode ===
"bootstrap_retry"
? "Retry provider bootstrap on this same run; durable evidence proves no provider session or provider event was created."
: "Resume this same run from its exact persisted native provider checkpoint after the retry delay.",
}),
nextAttemptAt,
recoveryHistory: sql`(
select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb)
from jsonb_array_elements(
coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
|| jsonb_build_array(${JSON.stringify({
at: now.toISOString(),
disposition: phase,
reason: sourceFailureCode,
controllerBootId: controller.bootId,
controllerGeneration:
input.restartRecovery?.controllerGeneration ?? null,
providerAttempt: attempt,
stderrTail: sanitizedStderrTail,
providerSessionEstablished:
recoveryEvidence.providerSessionEstablished,
checkpointExists: recoveryEvidence.checkpointExists,
})}::jsonb)
) with ordinality as history(item, ordinal)
where ordinal > greatest(
jsonb_array_length(
coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
|| jsonb_build_array(${JSON.stringify({
at: now.toISOString(),
disposition: phase,
reason: sourceFailureCode,
controllerBootId: controller.bootId,
controllerGeneration:
input.restartRecovery?.controllerGeneration ?? null,
providerAttempt: attempt,
stderrTail: sanitizedStderrTail,
providerSessionEstablished:
recoveryEvidence.providerSessionEstablished,
checkpointExists: recoveryEvidence.checkpointExists,
})}::jsonb)
) - 20,
0
)
)`,
updatedAt: now,
})
.where(
and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(
nativeRunFinalizations.companyId,
input.execution.binding.companyId,
),
eq(
nativeRunFinalizations.issueId,
input.execution.binding.issueId,
),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
eq(nativeRunFinalizations.attempt, attempt),
eq(nativeRunFinalizations.controllerBootId, controller.bootId),
eq(nativeRunFinalizations.controllerPid, controller.pid),
eq(
nativeRunFinalizations.controllerProcessStartedAt,
controller.processStartedAt,
),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.returning({ runId: nativeRunFinalizations.runId })
.then((rows) => rows[0] ?? null);
if (!updated) throw new Error("native_session_lease_lost");
const [runBeforeWrite] = await tx
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.execution.binding.runId))
.for("update");
const [updatedRun] = await tx
.update(heartbeatRuns)
.set({
// An authentication timeout does not prove the retained runner or
// its provider stopped. Preserve physical ownership until verified.
...(!ownershipUnverified
? {
status: "failed",
executionStatusDeliveryId: randomUUID(),
finishedAt: now,
}
: {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
error: message,
errorCode: ownershipUnverified
? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE
: sourceFailureCode,
updatedAt: now,
})
.where(eq(heartbeatRuns.id, input.execution.binding.runId))
.returning();
if (
updatedRun &&
runBeforeWrite &&
updatedRun.status !== runBeforeWrite.status
) {
terminalRunToReport = updatedRun;
}
const stillOwnsTask =
failureTask?.assigneeAgentId === input.execution.binding.agentId &&
["in_progress", "in_review"].includes(failureTask.status) &&
(!failureTask.executionRunId ||
failureTask.executionRunId === input.execution.binding.runId) &&
(!failureTask.checkoutRunId ||
failureTask.checkoutRunId === input.execution.binding.runId);
let failureBlockStatusVersion: number | undefined;
if (stillOwnsTask && recoveryProjection.issueStatus) {
const projected = await issueService(tx as unknown as Db).update(
input.execution.binding.issueId,
{ status: recoveryProjection.issueStatus },
tx,
);
if (projected?.status === "blocked") failureBlockStatusVersion = projected.statusVersion;
}
if (
!ownershipUnverified &&
stillOwnsTask &&
phase === "terminal_failure"
) {
await tx
.update(issues)
.set({ executionRunId: null, checkoutRunId: null, updatedAt: now })
.where(eq(issues.id, input.execution.binding.issueId));
}
if (!stillOwnsTask) return;
await issueRecoveryActionService(
tx as unknown as Db,
).upsertSourceScoped({
companyId: input.execution.binding.companyId,
sourceIssueId: input.execution.binding.issueId,
kind: "active_run_watchdog",
ownerType: recoveryProjection.recoveryActionOwnerType,
ownerAgentId: recoveryProjection.recoveryActionOwnerAgentId,
returnOwnerAgentId: input.execution.binding.agentId,
cause: recoveryProjection.recoveryActionCause,
fingerprint: createHash("sha256")
.update(`${input.execution.binding.runId}:${failureCode}`)
.digest("hex"),
evidence: {
runId: input.execution.binding.runId,
...(failureBlockStatusVersion !== undefined ? { nativeFailureBlock: {
runId: input.execution.binding.runId,
statusVersion: failureBlockStatusVersion,
} } : {}),
coordinatorAttempt: attempt,
sourceFailureCode,
recoveryDisposition: failureCode,
recoveryMode: recoveryEvidence.recoveryMode,
providerSessionEstablished:
recoveryEvidence.providerSessionEstablished,
},
nextAction:
sourceFailureCode === "native_permission_declined"
? "Provider permission was declined. Inspect the task and give explicit direction before starting further provider work. Automatic recovery is stopped."
: sourceFailureCode === "native_provider_approval_required"
? "Approval required. Review the operation and update the agent's permission setting before retrying. This runner has no interactive approval handler."
: sourceFailureCode === "native_session_cleanup_quarantined"
? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE
: sourceFailureCode === "native_provider_terminal_failed"
? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed."
: recoveryEvidence.recoveryMode === "ambiguous_state"
? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session."
: integrityFailure
? "Inspect the persisted runner event collision and explicitly repair or replace the run; automatic retries are disabled."
: sourceFailureCode === "native_provider_usage_limit"
? "Restore model provider usage capacity, then explicitly retry the task; automatic retries are stopped."
: ownershipUnverified
? "Resolve the retained runner's authentication or executable compatibility before an explicit recovery; do not blindly restart, cancel, or replace its provider session."
: exhausted
? "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped."
: recoveryEvidence.recoveryMode === "bootstrap_retry"
? "Retry bootstrap on the same run without manufacturing a provider checkpoint."
: "Resume the exact persisted native session on the same heartbeat run.",
wakePolicy: nextAttemptAt
? {
kind: "resume_native_run",
runId: input.execution.binding.runId,
notBefore: nextAttemptAt.toISOString(),
}
: null,
maxAttempts: 3,
supersedeOnIdentityChange:
recoveryProjection.supersedeOnIdentityChange,
});
});
if (terminalRunToReport) void reportRunFailure(input.db, terminalRunToReport);
await boundedExecutionCleanup(async () => {
await stoppedLeaseRenewal;
await attemptFailureStep(() =>
input.onLog?.(
"stderr",
`[paperclip-runner] native session execution failed: ${executionFailureMessage}\n`,
),
);
if (runnerSessionStartupScope) {
await attemptFailureStep(() =>
trace.end(runnerSessionStartupScope!, {
endedAtMs: failedAtMs,
outcome: "failed",
}),
);
}
if (agentTurnScope) {
await attemptFailureStep(() =>
trace.end(agentTurnScope!, {
endedAtMs: failedAtMs,
outcome: "failed",
}),
);
}
if (
!ownershipUnverified &&
warmSessionId !== null &&
lifecyclePolicy.mode === "warm"
) {
await attemptFailureStep(() =>
releaseWarmNativeSession(
warmSessionId!,
warmSessionOwnerToken,
lifecyclePolicy.idleTimeoutMs,
true,
),
);
}
if (taskSettleScope)
await trace.end(taskSettleScope, { outcome: "failed" });
await trace.finish("failed");
});
throw error;
} finally {
// Even a secondary logging/recovery-write failure cannot authorize
// heartbeat to release the retained process or its task ownership.
if (ownershipUnverified) throw new NativeRunnerOwnershipUnverifiedError();
if (protocolIntegrityFailure !== null) throw protocolIntegrityFailure;
}
} finally {
releaseGitHubRun?.();
// A startup failure or onSession(null) must not leak a transport. A warm
// owner retains only the inactive broker until its normal retirement.
if (githubAccess && (!warmSessionId || warmNativeSessions.get(warmSessionId)?.githubAccess !== githubAccess)) {
await githubAccess.stop();
}
}
if (
planSynchronizations.length === 0 &&
"executionMode" in input.execution &&
input.execution.executionMode === "plan"
) {
const markdown = semanticProviderPlanMarkdown(
native.result as unknown as Record<string, unknown>,
);
if (markdown) {
const digest = createHash("sha256").update(markdown).digest("hex");
await recordPlanSynchronization({
sourceEventId: `semantic-plan:${input.execution.binding.runId}:${digest}`,
...(native.turnId ? { turnId: native.turnId } : {}),
eventType: "plan.updated",
payload: {
schema: "paperclip.plan.updated.v1",
planId: `semantic:${native.turnId ?? input.execution.binding.runId}`,
revision: 1,
complete: true,
markdown,
source: "semantic_result_artifact",
},
});
}
}
const releaseNow = new Date();
const released = await input.db
.update(nativeRunFinalizations)
.set({
leaseOwner: null,
leaseExpiresAt: null,
recoveryState: null,
recoveryRequestId: null,
updatedAt: releaseNow,
})
.where(
and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.companyId, input.execution.binding.companyId),
eq(nativeRunFinalizations.issueId, input.execution.binding.issueId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
eq(nativeRunFinalizations.attempt, attempt),
eq(nativeRunFinalizations.controllerBootId, controller.bootId),
eq(nativeRunFinalizations.controllerPid, controller.pid),
eq(
nativeRunFinalizations.controllerProcessStartedAt,
controller.processStartedAt,
),
gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`),
),
)
.returning({ runId: nativeRunFinalizations.runId })
.then((rows) => rows[0] ?? null);
if (!released) throw new Error("native_session_lease_lost");
const finalization: NativeFinalizationResult = {
schema: "paperclip.native-finalization.v1",
runtimeMode: "native",
runId: input.execution.binding.runId,
issueId: input.execution.binding.issueId,
companyId: input.execution.binding.companyId,
result: native.result as unknown as Record<string, unknown>,
terminal: native.terminal,
turnId: native.turnId,
sourceInstanceId: effectiveRunnerInstanceId,
normalizedSessionId: native.normalizedSessionId,
providerSessionId: native.providerSessionId,
driverKind: native.driverKind,
driverVersion: native.driverVersion,
nativeEventCount: native.nativeEventCount,
highestContiguousSourceSeq: native.highestContiguousSourceSeq,
workspaceFinalizeStatus: "pending",
};
// A following run cannot attach until the prior run's durable finalization
// is committed. Provider completion alone is not an authority boundary.
if (warmSessionId !== null && lifecyclePolicy.mode === "warm") {
// A structured failed/cancelled result completes the protocol without
// throwing. Heartbeat still stops its sandbox, so retire the provider now
// while its transport can collect files and checkpoint the suspended runner.
// Leaving it in the warm map makes the next turn retire a stopped transport.
const retainSession = native.terminal.runTerminalState === "succeeded";
const instructionCopy = input.instructionWorkingCopy;
const ownedSession = warmNativeSessions.get(warmSessionId);
let collectInstructions: boolean;
try {
collectInstructions = Boolean(instructionCopy && ownedSession?.ownerToken === warmSessionOwnerToken &&
(instructionCopy.checkpointWarm ? !await instructionCopy.checkpointWarm() : await instructionCopy.hasChanges()));
} catch (error) {
// A checkpoint or its receipt callback can reject. Retire this owner
// before heartbeat stops the sandbox, preserving the initiating error.
await releaseWarmNativeSession(warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, true)
.catch(() => undefined);
throw error;
}
const collectedByOwner = ownedSession?.instructionCopy?.collectStopped === instructionCopy?.collectStopped
|| ownedSession?.instructionWorkingCopy?.collectStopped === instructionCopy?.collectStopped;
if (collectInstructions && ownedSession) {
// Keep the unchanged warm path intact. A changed private instruction copy
// requires the existing checkpoint-and-close boundary before collection.
ownedSession.closeOnReleaseReason = "registered instruction edits require stopped-provider collection";
}
await releaseWarmNativeSession(
warmSessionId,
warmSessionOwnerToken,
lifecyclePolicy.idleTimeoutMs,
!retainSession,
);
if (collectInstructions && !collectedByOwner) await instructionCopy!.collectStopped();
}
const providerFailure = providerFailureObservation.forTerminal(native.turnId, native.terminal);
const adapterResult: AdapterExecutionResult = {
exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1,
signal: null,
timedOut: false,
errorMessage:
native.terminal.runTerminalState === "succeeded"
? null
: providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`,
...(providerFailure ? { errorCode: providerFailure.errorCode } : {}),
resultJson: {
nativeResult: native.result as unknown as Record<string, unknown>,
nativeTerminal: native.terminal as unknown as Record<string, unknown>,
...(input.execution.provider.kind === "openai_dot" ? { providerAccounting: { usage: null, cost: null, externallyBilled: true }, externalStopConfirmed: false } : {}),
...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}),
...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}),
planSynchronizations,
},
summary: native.result.summary,
sessionId: native.normalizedSessionId,
sessionDisplayId: native.providerSessionId ?? native.normalizedSessionId,
...resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity),
model: input.execution.provider.model,
usage: normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }),
costUsd: input.execution.provider.kind !== "openai_dot" && normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })
? nativeUsageCostUsd(native.usage, input.execution.provider) ?? null : null,
costUsdExact: typeof native.usage?.accountingCostUsdExact === "string" ? native.usage.accountingCostUsdExact : undefined,
costStatus: nativeAccountingComplete && native.usage?.accountingCostIncomplete !== true
? input.execution.provider.kind === "aws_agentcore" ? "estimated" : undefined
: "unpriced",
usageComplete: nativeAccountingComplete,
usageBasis: "per_run",
nativeFinalization: finalization,
};
if (taskSettleScope) {
await trace.end(taskSettleScope, {
outcome:
native.terminal.runTerminalState === "succeeded" ? "ok" : "failed",
});
}
await trace.finish(
native.terminal.runTerminalState === "succeeded" ? "ok" : "failed",
);
return adapterResult;
}
function numericUsageField(
usage: Record<string, unknown> | null,
keys: string[],
): number | undefined {
if (!usage) return undefined;
for (const key of keys) {
const value = usage[key];
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
return value;
}
return undefined;
}
function nativeUsageMeasurement(usage: Record<string, unknown>) {
const nestedUsage = record(usage.usage);
// An explicit partial delta must not fall through to a full session total.
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
const candidates = [
record(usage.runDelta),
record(nestedUsage.runDelta),
record(usage.total),
record(nestedUsage.total),
record(usage.cumulative),
record(nestedUsage.cumulative),
nestedUsage,
usage,
];
return (
candidates.find(
(candidate) =>
numericUsageField(candidate, [
"inputTokens",
"input",
"promptTokens",
"outputTokens",
"output",
"completionTokens",
]) !== undefined,
) ?? usage
);
}
export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
if (provider.kind === "acpx") {
if (provider.agent === "cursor") return "cursor";
if (provider.agent === "copilot") return "github";
if (provider.agent === "pi") return "openrouter";
}
return "openai";
}
function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
const sequences = new Map<string, number>();
let currentTurn: string | undefined;
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
const snapshot = () => {
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
for (const [turnId, turn] of turns) {
const normalized = normalizeNativeUsage(turn.usage);
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
outputTokens += normalized?.outputTokens ?? 0;
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
const price = nativeUsageCostUsd(turn.usage, provider);
if (price === undefined) costIncomplete = true;
else { cost += usdToUnits(price); costKnown = true; }
}
// A free empty attempt contributes no price evidence for later paid work.
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
// still receive an estimate. Preserve positive known spend and true zero totals.
const retainCost = costKnown && (!costIncomplete || cost > 0n);
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
const current = currentTurn ? turns.get(currentTurn) : undefined;
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
};
return {
observe(event: PrpEvent) {
if (!event) return false;
const payload = record(event.payload);
const terminal = terminalTypes.includes(event.eventType);
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
// Historical replay can revisit an earlier report after restoration.
// Re-persist its aggregate, but never rewind or count that report twice.
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
sequences.set(event.sourceInstanceId, event.sourceSeq);
}
const key = event.turnId ?? "unattributed";
const turn = turns.get(key) ?? { usage: null, terminal: false };
if (event.eventType === "turn.started") {
currentTurn = event.turnId;
turn.usage = null;
turn.terminal = false;
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
if (payload.kind === "usage") {
const usage = record(payload.usage);
turn.usage = Object.hasOwn(usage, "runDelta")
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
}
if (terminal) turn.terminal = true;
turns.set(key, turn);
return true;
},
snapshot,
finish(turnId: string | null) {
const turn = turnId ? turns.get(turnId) : undefined;
if (turn) turn.terminal = true;
return snapshot();
},
};
}
export function nativeUsageCostUsd(
usage: Record<string, unknown> | null,
provider?: NativeExecutionInput["provider"],
) {
// The ACP normalization contract fills absent per-turn cost with zero and
// reports actual cost cumulatively. Until it carries an authoritative run
// delta with provenance, neither value is a candidate's billed USD receipt.
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
if (!usage) return undefined;
const measurement = nativeUsageMeasurement(usage);
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
// PRP currently defaults an absent providerCostUsd to zero. That zero is
// not proof of a free request when the runtime observed token consumption.
if (direct !== undefined && direct > 0) return direct;
const normalized = normalizeNativeUsage(usage);
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
const currency =
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
if (currency !== "USD") return undefined;
return numericUsageField(cost, ["amount", "total"]);
}
/** Billing identity follows the selected runtime and resolved credentials.
* Unknown authentication remains unknown; a provider name does not prove billing mode. */
export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">): Pick<AdapterExecutionResult, "provider" | "biller" | "billingType"> {
if (identity) return identity.provider === "openai" && identity.billingType === "metered_api"
? { ...identity, biller: inferOpenAiCompatibleBiller(env, identity.biller ?? "unknown") }
: identity;
if (provider.kind === "claude_managed") return { provider: "anthropic", biller: "anthropic", billingType: "metered_api" };
if (provider.kind === "aws_agentcore") return { provider: "aws_agentcore", biller: "aws_agentcore", billingType: "metered_api" };
if (provider.kind === "acpx" && ["cursor", "copilot"].includes(provider.agent)) {
const biller = nativeUsageBiller(provider);
return { provider: biller, biller, billingType: "unknown" };
}
if (provider.kind === "acpx" && provider.agent === "claude") {
if (env.CLAUDE_CODE_USE_BEDROCK === "1") return { provider: "anthropic", biller: "aws_bedrock", billingType: "metered_api" };
return { provider: "anthropic", biller: "anthropic", billingType: env.ANTHROPIC_API_KEY ? "metered_api" : env.CLAUDE_CODE_OAUTH_TOKEN ? "subscription_included" : "unknown" };
}
if (provider.kind === "opencode" || (provider.kind === "acpx" && provider.agent === "pi")) {
const parts = (provider.model ?? "").split("/");
const biller = parts.length > 1 ? parts[0] : provider.kind === "acpx" ? nativeUsageBiller(provider) : "unknown";
const modelProvider = biller === "openrouter" && parts.length > 2 ? parts[1] : biller;
return { provider: modelProvider, biller, billingType: "unknown" };
}
if (provider.kind !== "codex" && !(provider.kind === "acpx" && provider.agent === "codex"))
return { provider: "unknown", biller: "unknown", billingType: "unknown" };
return { provider: "openai", biller: env.OPENAI_API_KEY ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType: env.OPENAI_API_KEY ? "metered_api" : "unknown" };
}
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
const measurement = nativeUsageMeasurement(usage);
const cache = record(measurement.cache);
const count = (source: Record<string, unknown>, keys: string[]) => {
for (const key of keys) {
if (source[key] === undefined) continue;
const value = source[key];
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
}
return undefined;
};
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
if (input === undefined || outputTokens === undefined
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
return {
inputTokens, outputTokens,
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
};
}
function processEnvironment(
environment: NodeJS.ProcessEnv,
): Record<string, string> {
return Object.fromEntries(
Object.entries(environment).filter(
(entry): entry is [string, string] => typeof entry[1] === "string",
),
);
}
/** Preserve package-manager shims that resolve dependencies relative to argv[0]. */
export function buildRemoteCodexLauncherCommand(
sourcePath: string,
targetPath: string,
): string {
if (sourcePath === targetPath)
throw new Error("runner_remote_preinstalled_source_conflict");
const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'";
const launcher = `#!/bin/sh\nexec ${quote(sourcePath)} "$@"\n`;
// Replace atomically: writing through an existing symlink would corrupt the
// image's shared CLI, and another run may be executing this launcher already.
return (
`umask 077; mkdir -p ${quote(posix.dirname(targetPath))} && ` +
`[ ! -d ${quote(targetPath)} ] && ` +
`paperclip_codex_launcher_tmp=$(mktemp ${quote(targetPath + ".tmp.XXXXXX")}) && ` +
`trap 'rm -f "$paperclip_codex_launcher_tmp"' 0 && ` +
`printf '%s' ${quote(launcher)} > "$paperclip_codex_launcher_tmp" && ` +
`chmod 700 "$paperclip_codex_launcher_tmp" && ` +
`mv -f "$paperclip_codex_launcher_tmp" ${quote(targetPath)}`
);
}
export function mayUsePreinstalledRunnerArtifact(
configuredRemoteBinaryPath: string | null | undefined,
): boolean {
return !configuredRemoteBinaryPath?.trim();
}
const RUNNERD_BUILD_METADATA_SCHEMA =
"paperclip-runner/runnerd-build-metadata/v1";
const RUNNERD_BINARY_CONTRACT_VERSION = 2;
const REMOTE_PROVIDER_PACK_SCHEMA = "paperclip-runner/remote-provider-pack/v1";
const REMOTE_PROVIDER_PACK_PINS = {
nodeMinimum: "24.11.0",
codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion,
opencode: "1.18.34",
acpx: QUALIFIED_ACPX_VERSION,
claudeAcp: QUALIFIED_ACPX_PROFILES.claude.agentServerVersion,
codexAcp: QUALIFIED_ACPX_PROFILES.codex.agentServerVersion,
grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion,
} as const;
const REMOTE_PROVIDER_PACK_PROFILE_DIGESTS = {
grok: QUALIFIED_ACPX_PROFILES.grok.commandDigest,
claude:
QUALIFIED_ACPX_PROFILES.claude.commandDigest,
codex:
QUALIFIED_ACPX_PROFILES.codex.commandDigest,
} as const;
const REMOTE_PROVIDER_PACK_ARTIFACT_PATHS = {
grokLauncher: "dist/providers/grok/launcher.cjs",
nodeCommand: "node_modules/node/bin/node",
productionLock: "pnpm-lock.yaml",
opencodeCommand: "node_modules/.bin/opencode",
opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe",
opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs",
acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs",
} as const;
type RemoteProviderPackManifest = {
schema: typeof REMOTE_PROVIDER_PACK_SCHEMA;
digest: string;
payload: {
pins: typeof REMOTE_PROVIDER_PACK_PINS;
target: { platform: string; architecture: string };
runnerSourceRevision: string;
distDigest: string;
bridgeDigest: string;
acpxProfileDigests: typeof REMOTE_PROVIDER_PACK_PROFILE_DIGESTS;
providers?: Partial<Record<"pi" | "cursor", {
version: string; profileDigest: string; closureDigest: string; qualification: "qualified" | "pending";
path: string; sha256: string;
}>>;
candidateProviders?: Partial<Record<"cursor" | "copilot" | "pi", {
version: string; profileDigest: string; closureDigest: string; qualification: "qualified" | "pending";
path: string; sha256: string;
}>>;
artifacts: {
grokLauncher: { path: string; sha256: string };
nodeCommand: { path: string; sha256: string };
productionLock: { path: string; sha256: string };
opencodeCommand: { path: string; sha256: string };
opencodeExecutable: { path: string; sha256: string };
opencodeProxy: { path: string; sha256: string };
acpxSidecar: { path: string; sha256: string };
};
};
};
function sha256File(path: string): string {
return `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`;
}
export function sha256DirectoryTree(root: string): string {
const hash = createHash("sha256");
const visit = (directory: string, prefix = "") => {
const entries = readdirSync(directory, { withFileTypes: true }).sort(
(left, right) => left.name.localeCompare(right.name),
);
for (const entry of entries) {
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
const absolutePath = join(directory, entry.name);
if (entry.isDirectory()) {
hash.update(`directory\0${relativePath}\n`);
visit(absolutePath, relativePath);
} else if (entry.isFile()) {
hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`);
} else if (entry.isSymbolicLink()) {
hash.update(
`symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`,
);
} else {
throw new Error(
`runner_remote_provider_artifact_incompatible: unsupported dist entry ${relativePath}`,
);
}
}
};
visit(root);
return `sha256:${hash.digest("hex")}`;
}
function providerPackRelativePath(value: unknown, field: string): string {
if (
typeof value !== "string" ||
value.length === 0 ||
value.startsWith("/") ||
value.includes("\\") ||
posix.normalize(value) !== value ||
value.split("/").includes("..")
) {
throw new Error(
`runner_remote_provider_artifact_incompatible: invalid ${field}`,
);
}
return value;
}
export function readRemoteProviderPackManifest(packRoot: string): RemoteProviderPackManifest {
return readRemoteProviderPackIdentity(packRoot, true);
}
export function readBundledRemoteProviderPackManifest(manifestPath = bundledRemoteProviderPackManifestPath()): RemoteProviderPackManifest {
const manifest = readRemoteProviderPackIdentity(dirname(manifestPath), false);
if (manifest.payload.target.platform !== "linux" || manifest.payload.target.architecture !== "x64") {
throw new Error("runner_remote_provider_artifact_incompatible: bundled image pack must target Linux x64");
}
return manifest;
}
function readRemoteProviderPackIdentity(packRoot: string, verifyControllerFiles: boolean): RemoteProviderPackManifest {
let manifest: RemoteProviderPackManifest;
try {
manifest = JSON.parse(
readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"),
) as RemoteProviderPackManifest;
} catch (error) {
// The terminal run report keeps the outer message, not the cause chain.
// Keep a bounded reason there; raw filesystem errors include private paths.
const code = (error as NodeJS.ErrnoException | null)?.code;
const reason = error instanceof SyntaxError ? "invalid_json"
: code === "ENOENT" ? "missing"
: code === "EACCES" || code === "EPERM" ? "permission_denied"
: code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type"
: "io_error";
throw new Error(
`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`,
{ cause: error },
);
}
return validateRemoteProviderPackIdentity(manifest, packRoot, verifyControllerFiles);
}
export function validateRemoteProviderPackIdentity(
manifest: RemoteProviderPackManifest,
packRoot = "",
verifyControllerFiles = false,
retainedAuthority?: { digest: string; agent: string },
): RemoteProviderPackManifest {
const payload = manifest?.payload;
// A retained session may outlive unrelated bundled-provider upgrades. Its
// active provider must still satisfy this release's qualified ACPX contract.
const retainedAgent = retainedAuthority?.agent;
const retainedProfile = retainedAgent === "claude" || retainedAgent === "codex" || retainedAgent === "grok"
? retainedAgent : null;
const activePins = retainedProfile === "claude" ? ["acpx", "claudeAcp"] as const
: retainedProfile === "codex" ? ["acpx", "codex", "codexAcp"] as const
: ["acpx", "grok"] as const;
const pinsMatch = retainedProfile
? canonicalJson(Object.keys(payload?.pins ?? {}).sort()) === canonicalJson(Object.keys(REMOTE_PROVIDER_PACK_PINS).sort())
&& Object.values(payload?.pins ?? {}).every(value => typeof value === "string" && value.length > 0 && value.length <= 120)
&& activePins.every(key => payload?.pins?.[key] === REMOTE_PROVIDER_PACK_PINS[key])
&& canonicalJson(Object.keys(payload?.acpxProfileDigests ?? {}).sort()) === canonicalJson(Object.keys(REMOTE_PROVIDER_PACK_PROFILE_DIGESTS).sort())
&& Object.values(payload?.acpxProfileDigests ?? {}).every(value => /^sha256:[0-9a-f]{64}$/.test(value))
&& payload?.acpxProfileDigests?.[retainedProfile] === REMOTE_PROVIDER_PACK_PROFILE_DIGESTS[retainedProfile]
: canonicalJson(payload?.pins) === canonicalJson(REMOTE_PROVIDER_PACK_PINS)
&& canonicalJson(payload?.acpxProfileDigests) === canonicalJson(REMOTE_PROVIDER_PACK_PROFILE_DIGESTS);
if (
manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
!payload ||
!pinsMatch ||
typeof payload.target?.platform !== "string" ||
typeof payload.target?.architecture !== "string" ||
!/^[0-9a-f]{40}(?:-dirty)?$/.test(payload.runnerSourceRevision) ||
!/^sha256:[0-9a-f]{64}$/.test(payload.distDigest)
) {
throw new Error(
"runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match",
);
}
const digest = `sha256:${createHash("sha256")
.update(canonicalJson(payload))
.digest("hex")}`;
if (manifest.digest !== digest || (retainedAuthority && digest !== retainedAuthority.digest)) {
throw new Error(
"runner_remote_provider_artifact_incompatible: provider pack manifest digest mismatch",
);
}
const artifactEntries = [
["Grok builtin launcher", payload.artifacts?.grokLauncher, REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.grokLauncher],
[
"provider Node",
payload.artifacts?.nodeCommand,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.nodeCommand,
],
[
"production lockfile",
payload.artifacts?.productionLock,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.productionLock,
],
[
"OpenCode command",
payload.artifacts?.opencodeCommand,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeCommand,
],
[
"OpenCode executable",
payload.artifacts?.opencodeExecutable,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeExecutable,
],
[
"OpenCode proxy",
payload.artifacts?.opencodeProxy,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.opencodeProxy,
],
[
"ACPX sidecar",
payload.artifacts?.acpxSidecar,
REMOTE_PROVIDER_PACK_ARTIFACT_PATHS.acpxSidecar,
],
] as const;
for (const [label, artifact, expectedPath] of artifactEntries) {
const artifactPath = providerPackRelativePath(
artifact?.path,
`${label} path`,
);
if (artifactPath !== expectedPath) {
throw new Error(
`runner_remote_provider_artifact_incompatible: ${label} path must be ${expectedPath}`,
);
}
if (
!/^sha256:[a-f0-9]{64}$/.test(artifact?.sha256 ?? "") ||
(verifyControllerFiles && sha256File(resolve(packRoot, artifactPath)) !== artifact.sha256)
) {
throw new Error(
`runner_remote_provider_artifact_incompatible: ${label} digest mismatch`,
);
}
}
if (verifyControllerFiles && sha256DirectoryTree(resolve(packRoot, "dist")) !== payload.distDigest) {
throw new Error(
"runner_remote_provider_artifact_incompatible: provider dist tree digest mismatch",
);
}
for (const [inventory, candidates] of [["providers", payload.providers], ["candidateProviders", payload.candidateProviders]] as const) {
if (candidates === undefined) continue;
if (!candidates || typeof candidates !== "object" || Array.isArray(candidates) || Object.keys(candidates).length > 3) {
throw new Error("runner_remote_provider_artifact_incompatible: invalid candidate inventory");
}
for (const [provider, candidate] of Object.entries(candidates)) {
const expectedPath = `provider-assets/${provider}/${payload.target.platform}-${payload.target.architecture}`;
if (!(inventory === "providers" ? ["pi", "cursor"] : ["cursor", "copilot", "pi"]).includes(provider) || !candidate
|| Object.keys(candidate).some(key => !["version", "profileDigest", "closureDigest", "qualification", "path", "sha256"].includes(key))
|| candidate.qualification !== (["pi", "cursor"].includes(provider) ? "qualified" : "pending")
|| candidate.path !== expectedPath
|| typeof candidate.version !== "string" || !candidate.version || candidate.version.length > 120
|| !/^sha256:[a-f0-9]{64}$/.test(candidate.profileDigest)
|| !/^sha256:[a-f0-9]{64}$/.test(candidate.closureDigest)
|| !/^sha256:[a-f0-9]{64}$/.test(candidate.sha256)) {
throw new Error("runner_remote_provider_artifact_incompatible: invalid candidate identity");
}
const candidatePath = providerPackRelativePath(candidate.path, "candidate assets");
if (provider === "cursor" && !retainedProfile) {
const target = `${payload.target.platform}-${payload.target.architecture}`;
const distribution = Object.hasOwn(CURSOR_DISTRIBUTION_PINS, target)
? CURSOR_DISTRIBUTION_PINS[target as keyof typeof CURSOR_DISTRIBUTION_PINS] : undefined;
if (!distribution || candidate.version !== QUALIFIED_ACPX_PROFILES.cursor.agentServerVersion
|| candidate.profileDigest !== QUALIFIED_ACPX_PROFILES.cursor.commandDigest
|| candidate.closureDigest !== `sha256:${distribution.closureSha256}`) {
throw new Error("runner_remote_provider_artifact_incompatible: Cursor profile or closure does not match this release");
}
}
if (verifyControllerFiles && sha256DirectoryTree(resolve(packRoot, candidatePath)) !== candidate.sha256) {
throw new Error("runner_remote_provider_artifact_incompatible: candidate asset tree digest mismatch");
}
}
}
const bridgeDigest = `sha256:${createHash("sha256")
.update(payload.artifacts.opencodeProxy.sha256)
.update("\n")
.update(payload.artifacts.acpxSidecar.sha256)
.update("\n")
.update(payload.distDigest)
.digest("hex")}`;
if (payload.bridgeDigest !== bridgeDigest) {
throw new Error(
"runner_remote_provider_artifact_incompatible: provider bridge digest mismatch",
);
}
return structuredClone(manifest);
}
export function assertRemoteRunnerBuildMetadata(
value: unknown,
requiredMode: "dial_wss" | "listen_ws",
requiredExternalProvider?: "openai_dot_mcp",
): void {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error("runner_remote_artifact_metadata_invalid");
}
const metadata = value as Record<string, unknown>;
if (
metadata.schema !== RUNNERD_BUILD_METADATA_SCHEMA ||
metadata.binaryName !== "paperclip-runnerd" ||
metadata.packageName !== "@paperclipai/paperclip-runner" ||
metadata.binaryContractVersion !== RUNNERD_BINARY_CONTRACT_VERSION
) {
throw new Error("runner_remote_artifact_contract_incompatible");
}
// Older sandbox images share binary contract v2, but reject the zero
// lifetime now used by the controller and cannot renew connection leases.
// Reject them before launch so preparation stages the bundled runner instead.
if (requiredExternalProvider && (!Array.isArray(metadata.externalProviderCapabilities)
|| !metadata.externalProviderCapabilities.includes(requiredExternalProvider))) {
throw new Error("runner_remote_dot_capability_missing: install the current Paperclip Runner artifact");
}
const sessionCapabilities = Array.isArray(metadata.durableSessionCapabilities)
? metadata.durableSessionCapabilities
: [];
for (const capability of ["unlimited_runtime", "connection_lease_renewal"]) {
if (!sessionCapabilities.includes(capability)) {
throw new Error(`runner_remote_session_capability_missing:${capability}`);
}
}
const modes = Array.isArray(metadata.prpTransportModes)
? metadata.prpTransportModes
: [];
if (!modes.includes(requiredMode)) {
throw new Error(
`runner_remote_transport_capability_missing:${requiredMode}`,
);
}
}
export function nativeComputerToolForExecution(
execution: NativeExecutionInput,
target: AdapterExecutionTarget | null | undefined,
): AdapterComputerExecutionTarget["computerTool"] | undefined {
if (target?.kind !== "remote" || target.transport !== "computer" ||
execution.task.workMode !== "standard" ||
("executionMode" in execution && execution.executionMode === "plan")) return undefined;
return target.computerTool;
}
export async function stageRemoteRunnerFile(input: {
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
runner: CommandManagedRuntimeRunner;
sourcePath: string;
targetPath: string;
mode: number;
}): Promise<void> {
const runner = input.runner;
if (runner.syncIn) {
await runner.syncIn([
{
operationId: `runner-stage-${randomUUID()}`,
files: [
{
sourcePath: input.sourcePath,
targetPath: input.targetPath,
kind: "file",
mode: input.mode,
},
],
},
]);
return;
}
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
const run = async (script: string, stdin?: string) => {
const result = await runner.execute({
command: "sh", args: ["-c", script], stdin,
bypassSession: true, timeoutMs: 180_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error(`runner_remote_staging_failed: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`);
}
};
if (lstatSync(input.sourcePath).size <= 4 * 1024 * 1024) {
await run(`umask 077; mkdir -p ${quote(posix.dirname(input.targetPath))} && ` +
`base64 -d > ${quote(input.targetPath)} && chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`,
readFileSync(input.sourcePath).toString("base64"));
return;
}
const remoteUpload = `${input.targetPath}.upload-${randomUUID()}`;
try {
await uploadRemoteRunnerFileChunks({ sourcePath: input.sourcePath, remotePath: remoteUpload, run });
await run(`chmod ${input.mode.toString(8)} ${quote(remoteUpload)} && mv -f ${quote(remoteUpload)} ${quote(input.targetPath)}`);
} finally {
await run(`rm -f ${quote(remoteUpload)}`).catch(() => undefined);
}
}
const REMOTE_ARTIFACT_UPLOAD_CONCURRENCY = 8;
let activeRemoteArtifactUploads = 0;
const remoteArtifactUploadWaiters: Array<() => void> = [];
async function withRemoteArtifactUploadSlot(operation: () => Promise<void>): Promise<void> {
await new Promise<void>((resolve) => {
if (activeRemoteArtifactUploads < REMOTE_ARTIFACT_UPLOAD_CONCURRENCY) {
activeRemoteArtifactUploads++;
resolve();
} else remoteArtifactUploadWaiters.push(resolve);
});
try { await operation(); }
finally {
const next = remoteArtifactUploadWaiters.shift();
if (next) next();
else activeRemoteArtifactUploads--;
}
}
async function uploadRemoteRunnerFileChunks(input: {
sourcePath: string;
remotePath: string;
run: (script: string, stdin?: string) => Promise<void>;
onProgress?: (completedBytes: number, totalBytes: number) => Promise<void>;
}): Promise<void> {
const { run, remotePath } = input;
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
const pending: Promise<void>[] = [];
let uploadError: unknown;
try {
await run(`umask 077; mkdir -p ${quote(posix.dirname(remotePath))} && : > ${quote(remotePath)}`);
const digest = createHash("sha256");
const totalBytes = lstatSync(input.sourcePath).size;
let completedBytes = 0;
let progressAt = Date.now();
await input.onProgress?.(0, totalBytes);
let chunkIndex = 0;
const stream = createReadStream(input.sourcePath, { highWaterMark: 4 * 1024 * 1024 });
for await (const bytes of stream) {
const chunk = bytes as Buffer;
digest.update(chunk);
completedBytes += chunk.length;
// Disjoint fixed offsets permit bounded parallel upload and idempotent
// writes. The final digest rejects missing, duplicated, or partial bytes.
const offset = chunkIndex++;
pending.push(withRemoteArtifactUploadSlot(() => run(
`base64 -d | dd of=${quote(remotePath)} bs=4194304 seek=${offset} conv=notrunc 2>/dev/null`,
chunk.toString("base64"))).catch((error) => { uploadError ??= error; }));
if (pending.length === REMOTE_ARTIFACT_UPLOAD_CONCURRENCY) {
await Promise.all(pending);
pending.length = 0;
if (uploadError) throw uploadError;
if (Date.now() - progressAt >= 15_000) {
await input.onProgress?.(completedBytes, totalBytes);
progressAt = Date.now();
}
}
}
await Promise.all(pending);
if (uploadError) throw uploadError;
await input.onProgress?.(totalBytes, totalBytes);
const expectedDigest = digest.digest("hex");
await run(`test "$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${quote(remotePath)}; else shasum -a 256 ${quote(remotePath)}; fi | cut -d ' ' -f 1)" = ${quote(expectedDigest)}`);
} finally {
// Do not remove or rename the upload while another offset write is active.
await Promise.all(pending);
}
}
function archiveExcludeArgs(entries: readonly string[]): string[] {
for (const entry of entries) {
const segments = entry.split("/");
if (
entry.length === 0 ||
entry.startsWith("/") ||
segments.some(
(segment) =>
segment === "" ||
segment === "." ||
segment === ".." ||
!/^[A-Za-z0-9._-]+$/.test(segment),
)
) {
throw new Error("runner_remote_checkpoint_exclusion_invalid");
}
}
return entries.map((entry) => `--exclude=./${entry}`);
}
export async function stageRemoteRunnerDirectory(input: {
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
runner: CommandManagedRuntimeRunner;
sourcePath: string;
targetPath: string;
mode: number;
excludeEntries?: readonly string[];
onProgress?: (completedBytes: number, totalBytes: number) => Promise<void>;
}): Promise<void> {
const excludeArgs = archiveExcludeArgs(input.excludeEntries ?? []);
if (input.runner.syncIn) {
let stagingRoot: string | null = null;
let sourcePath = input.sourcePath;
try {
if (excludeArgs.length > 0) {
stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-restore-"));
const archive = execFileSync(
"tar",
[...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."],
{ encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
);
execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], {
input: archive,
maxBuffer: 64 * 1024 * 1024,
});
sourcePath = stagingRoot;
}
await input.runner.syncIn([
{
operationId: `runner-stage-dir-${randomUUID()}`,
files: [
{
sourcePath,
targetPath: input.targetPath,
kind: "directory",
mode: input.mode,
},
],
},
]);
} finally {
if (stagingRoot) rmSync(stagingRoot, { recursive: true, force: true });
}
return;
}
const stagingRoot = mkdtempSync(join(tmpdir(), "paperclip-runner-upload-"));
const archivePath = join(stagingRoot, "payload.tar.gz");
const remoteArchive = `${input.targetPath}.upload-${randomUUID()}.tar.gz`;
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
const run = async (script: string, stdin?: string) => {
const result = await input.runner.execute({
command: "sh", args: ["-c", script], stdin,
bypassSession: true, timeoutMs: 180_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error(`runner_remote_directory_staging_failed: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`);
}
};
try {
// Provider packs can exceed a gigabyte. Keep compression off the event loop
// and transfer from disk in bounded chunks instead of buffering the archive.
await new Promise<void>((resolve, reject) => {
execFile("tar", [...excludeArgs, "-czf", archivePath, "-C", input.sourcePath, "."],
{ maxBuffer: 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } },
(error) => error ? reject(error) : resolve());
});
await uploadRemoteRunnerFileChunks({
sourcePath: archivePath, remotePath: remoteArchive, run, onProgress: input.onProgress,
});
await run(`umask 077 && mkdir -p ${quote(input.targetPath)} && ` +
`tar -xzf ${quote(remoteArchive)} -C ${quote(input.targetPath)} && chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`);
} finally {
rmSync(stagingRoot, { recursive: true, force: true });
await run(`rm -f ${quote(remoteArchive)}`).catch(() => undefined);
}
}
async function remoteRunnerPathExists(input: {
runner: CommandManagedRuntimeRunner;
path: string;
kind: "file" | "directory";
}): Promise<boolean> {
const escapedPath = input.path.replaceAll("'", "'\\''");
const result = await input.runner.execute({
command: "sh",
args: ["-c", `test -${input.kind === "file" ? "f" : "d"} '${escapedPath}'`],
bypassSession: true,
timeoutMs: 10_000,
});
return result.exitCode === 0 && !result.timedOut;
}
function assertSafeRemoteCheckpointArchive(archive: Buffer): void {
if (archive.length > MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES) {
throw new Error("runner_remote_checkpoint_archive_too_large");
}
let names: string[];
let verboseEntries: string[];
try {
names = execFileSync("tar", ["-tzf", "-"], {
input: archive,
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
timeout: 30_000,
})
.split("\n")
.filter((line) => line.length > 0);
verboseEntries = execFileSync("tar", ["-tvzf", "-"], {
input: archive,
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
timeout: 30_000,
})
.split("\n")
.filter((line) => line.length > 0);
} catch {
throw new Error("runner_remote_checkpoint_archive_invalid");
}
if (
names.length === 0 ||
names.length > MAX_REMOTE_CHECKPOINT_ENTRIES ||
verboseEntries.length !== names.length
) {
throw new Error("runner_remote_checkpoint_archive_invalid");
}
for (const name of names) {
const normalized = name.replace(/^\.\//, "");
if (
name.includes("\0") ||
name.startsWith("/") ||
normalized.split("/").some((part) => part === "..")
) {
throw new Error("runner_remote_checkpoint_archive_unsafe_path");
}
}
for (const entry of verboseEntries) {
const type = entry.trimStart()[0];
if (type !== "-" && type !== "d") {
throw new Error("runner_remote_checkpoint_archive_unsafe_entry");
}
}
try {
execFileSync("tar", ["-xOzf", "-"], {
input: archive,
stdio: ["pipe", "pipe", "pipe"],
maxBuffer: MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES,
timeout: 60_000,
});
} catch {
throw new Error("runner_remote_checkpoint_archive_expanded_too_large");
}
}
function assertSafeExtractedCheckpoint(root: string): void {
const pending = [root];
let entries = 0;
let totalBytes = 0;
while (pending.length > 0) {
const directory = pending.pop()!;
for (const entry of readdirSync(directory, { withFileTypes: true })) {
entries += 1;
if (entries > MAX_REMOTE_CHECKPOINT_ENTRIES) {
throw new Error("runner_remote_checkpoint_too_many_entries");
}
const path = join(directory, entry.name);
const metadata = lstatSync(path);
if (
metadata.isSymbolicLink() ||
(!metadata.isDirectory() && !metadata.isFile())
) {
throw new Error("runner_remote_checkpoint_unsafe_entry");
}
if (metadata.isDirectory()) {
pending.push(path);
} else {
totalBytes += metadata.size;
if (totalBytes > MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES) {
throw new Error("runner_remote_checkpoint_expanded_too_large");
}
}
}
}
}
/**
* Copy a remote runner directory into durable local state.
*
* Some provider homes contain process-local scratch trees (for example Codex's
* `tmp/arg0` executable aliases). Those aliases can point outside the directory
* and are neither portable nor required to resume a provider session. When a
* provider supplies a native syncOut implementation, create a sibling snapshot
* first so the live runtime remains untouched and the provider's archive safety
* checks still apply to every persisted entry.
*/
export async function syncRemoteRunnerDirectoryOut(input: {
runner: CommandManagedRuntimeRunner;
sourcePath: string;
targetPath: string;
mode: number;
excludeEntries?: readonly string[];
}): Promise<void> {
if (
!(await remoteRunnerPathExists({
runner: input.runner,
path: input.sourcePath,
kind: "directory",
}))
)
return;
mkdirSync(resolve(input.targetPath, ".."), { recursive: true, mode: 0o700 });
const excluded = input.excludeEntries ?? [];
const excludeArgs = archiveExcludeArgs(excluded)
.map((argument) => `'${argument}'`)
.join(" ");
if (input.runner.syncOut) {
let syncSourcePath = input.sourcePath;
let snapshotPath: string | null = null;
if (excluded.length > 0) {
const checkpointId = randomUUID();
snapshotPath = posix.join(
posix.dirname(input.sourcePath),
`.paperclip-checkpoint-${checkpointId}`,
);
const archivePath = `${snapshotPath}.tar`;
const escapedSource = input.sourcePath.replaceAll("'", "'\\''");
const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''");
const escapedArchive = archivePath.replaceAll("'", "'\\''");
const snapshotResult = await input.runner.execute({
command: "sh",
args: [
"-c",
`set -e; umask 077; mkdir -p '${escapedSnapshot}'; ` +
`trap "rm -f '${escapedArchive}'" EXIT; ` +
`tar ${excludeArgs} -cf '${escapedArchive}' -C '${escapedSource}' .; ` +
`tar -xf '${escapedArchive}' -C '${escapedSnapshot}'`,
],
bypassSession: true,
timeoutMs: 120_000,
});
if (snapshotResult.exitCode !== 0 || snapshotResult.timedOut) {
throw new Error("runner_remote_checkpoint_snapshot_failed");
}
syncSourcePath = snapshotPath;
}
try {
await input.runner.syncOut([
{
operationId: `runner-checkpoint-dir-${randomUUID()}`,
files: [
{
sourcePath: syncSourcePath,
targetPath: input.targetPath,
kind: "directory",
mode: input.mode,
},
],
},
]);
} finally {
if (snapshotPath) {
const escapedSnapshot = snapshotPath.replaceAll("'", "'\\''");
await input.runner
.execute({
command: "sh",
args: ["-c", `rm -rf -- '${escapedSnapshot}'`],
bypassSession: true,
timeoutMs: 30_000,
})
.catch(() => undefined);
}
}
return;
}
const escapedSource = input.sourcePath.replaceAll("'", "'\\''");
const result = await input.runner.execute({
command: "sh",
args: ["-c", `tar ${excludeArgs} -czf - -C '${escapedSource}' . | base64`],
bypassSession: true,
timeoutMs: 120_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error("runner_remote_checkpoint_failed");
}
const archive = Buffer.from(result.stdout.replace(/\s+/g, ""), "base64");
assertSafeRemoteCheckpointArchive(archive);
const parent = resolve(input.targetPath, "..");
const stagingRoot = mkdtempSync(join(parent, ".paperclip-checkpoint-"));
const stagedTarget = join(stagingRoot, "payload");
const previousTarget = join(
parent,
`.paperclip-checkpoint-previous-${randomUUID()}`,
);
let previousMoved = false;
let replacementInstalled = false;
try {
mkdirSync(stagedTarget, { recursive: true, mode: input.mode });
execFileSync(
"tar",
[
"--no-same-owner",
"--no-same-permissions",
"-xzf",
"-",
"-C",
stagedTarget,
],
{ input: archive, maxBuffer: 8 * 1024 * 1024, timeout: 60_000 },
);
assertSafeExtractedCheckpoint(stagedTarget);
chmodSync(stagedTarget, input.mode);
if (existsSync(input.targetPath)) {
renameSync(input.targetPath, previousTarget);
previousMoved = true;
}
renameSync(stagedTarget, input.targetPath);
replacementInstalled = true;
if (previousMoved) {
rmSync(previousTarget, { recursive: true, force: true });
previousMoved = false;
}
} catch (error) {
if (
previousMoved &&
!replacementInstalled &&
!existsSync(input.targetPath)
) {
try {
renameSync(previousTarget, input.targetPath);
previousMoved = false;
} catch {
// Leave the last durable checkpoint at previousTarget. Deleting it in
// finally would turn a failed replacement into irreversible data loss.
}
}
throw error;
} finally {
if (previousMoved && replacementInstalled) {
rmSync(previousTarget, { recursive: true, force: true });
}
rmSync(stagingRoot, { recursive: true, force: true });
}
}
type RetiredComputerCheckpointInput = {
execution: NativeExecutionInput;
priorExecution: NativeExecutionInput;
identity: RunnerdDurableIdentity;
runner: Record<string, unknown>;
control: Record<string, unknown>;
provider: Record<string, unknown>;
checkpoint: Record<string, unknown>;
currentCheckpoint: Record<string, unknown>;
};
type RetiredComputerCheckpointUnsettledReason =
| "provider_kind"
| "runner_schema"
| "runner_lifecycle"
| "runner_identity"
| "controller_identity"
| "runner_outbox"
| "runner_terminal_delivery"
| "controller_commands"
| "controller_terminal_event"
| "controller_terminal_identity"
| "provider_terminal_authority"
| "prior_run_identity"
| "issue_scope"
| "session_scope"
| "session_config"
| "provider_session_identity"
| `${"prior" | "current"}_checkpoint_${"provider_session" | "driver" | "active_turn" | "runtime_requests" | "run_identity" | "session_identity" | "company_identity" | "agent_identity" | "issue_identity"}`;
/** Fixed reason codes are safe to surface in run logs; never include checkpoint
* values, provider configuration, prompts, paths, or credentials in diagnostics. */
export function retiredComputerCheckpointUnsettledReason(
input: RetiredComputerCheckpointInput,
): RetiredComputerCheckpointUnsettledReason | null {
const { execution, priorExecution, identity, runner, control, provider, checkpoint, currentCheckpoint } = input;
if (execution.provider.kind !== "codex" || priorExecution.provider.kind !== "codex") return "provider_kind";
const events = control.committedEvents;
const commands = control.commands;
const terminal = record(record(Array.isArray(events) ? events.at(-1) : null).envelope);
const fields = ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"] as const;
if (runner.schema !== RUNNERD_STATE_SCHEMA) return "runner_schema";
if (runner.lifecycle !== "ready") return "runner_lifecycle";
if (fields.some(key => runner[key] !== identity[key])) return "runner_identity";
if (fields.some(key => record(control.identity)[key] !== identity[key])) return "controller_identity";
if (!Array.isArray(runner.outbox) || runner.outbox.length !== 0) return "runner_outbox";
if (runner.pendingTerminalDelivery !== null) return "runner_terminal_delivery";
if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed")) return "controller_commands";
if (!Array.isArray(events) || record(events.at(-1)).eventType !== "run.terminal") return "controller_terminal_event";
if (terminal.runId !== identity.runId || terminal.normalizedSessionId !== identity.normalizedSessionId) return "controller_terminal_identity";
if (provider.completedTurnAuthoritative !== true) return "provider_terminal_authority";
if (priorExecution.binding.runId !== identity.runId) return "prior_run_identity";
if (priorExecution.binding.issueId !== execution.binding.issueId) return "issue_scope";
if (nativeSessionScopeKey(priorExecution) !== nativeSessionScopeKey(execution)) return "session_scope";
if (nativeSessionConfigDigest(priorExecution) !== nativeSessionConfigDigest(execution)) return "session_config";
const providerIdentity = providerSessionIdentityFromDurableProviderState({ execution: priorExecution, providerState: provider });
if (!providerIdentity.providerSessionId) return "provider_session_identity";
for (const [source, value, expectedRunId] of [
["prior", checkpoint, identity.runId],
["current", currentCheckpoint, execution.binding.runId],
] as const) {
const binding = record(value.identity);
if (value.providerSessionId !== providerIdentity.providerSessionId) return `${source}_checkpoint_provider_session`;
if (value.driverKind !== priorExecution.session.driverKind) return `${source}_checkpoint_driver`;
if (value.activeTurnId !== null) return `${source}_checkpoint_active_turn`;
if (!Array.isArray(value.pendingRuntimeRequests) || value.pendingRuntimeRequests.length !== 0) return `${source}_checkpoint_runtime_requests`;
if (binding.runId !== expectedRunId) return `${source}_checkpoint_run_identity`;
if (binding.sessionId !== identity.normalizedSessionId) return `${source}_checkpoint_session_identity`;
if (binding.companyId !== execution.binding.companyId) return `${source}_checkpoint_company_identity`;
if (binding.agentId !== execution.binding.agentId) return `${source}_checkpoint_agent_identity`;
if (binding.issueId !== execution.binding.issueId) return `${source}_checkpoint_issue_identity`;
}
return null;
}
/** A dead computer runner may retain a ready checkpoint after controller loss.
* Match the local quiescent-recovery proof before sealing it for normal rotation.
*/
export function retiredComputerCheckpointIsSettled(input: RetiredComputerCheckpointInput): boolean {
return retiredComputerCheckpointUnsettledReason(input) === null;
}
async function sealRetiredComputerRunnerState(input: {
db: Db; execution: NativeExecutionInput; root: string;
target: Extract<AdapterExecutionTarget, { transport: "computer" }>;
runner: CommandManagedRuntimeRunner; stateDirectory: string;
}): Promise<void> {
const identity = readRunnerdDurableIdentity(input.root);
if (!identity || identity.runId === input.execution.binding.runId || input.execution.provider.kind !== "codex") return;
if (!durableIdentityMatchesSession(identity, input.execution)) throw new Error("runner_remote_retirement_scope_mismatch");
const controlPath = resolve(input.root, "control-plane", "control-plane-state.json");
const controlBytes = readBoundedNativeFile(controlPath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "recovery_state_too_large");
const runnerPath = posix.join(input.stateDirectory, "runner-state.json");
const providerPath = posix.join(input.stateDirectory, runnerProviderStateFilename(input.execution));
const read = await input.runner.execute({ command: "python3", args: ["-c", String.raw`
import sys,os,stat,json,base64
rows=[]
for p in sys.argv[1:]:
s=os.lstat(p)
if not stat.S_ISREG(s.st_mode) or s.st_size>16777216:raise RuntimeError('unsafe recovery state')
with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:data=f.read(16777217)
if len(data)>16777216:raise RuntimeError('unsafe recovery state')
rows.append(base64.b64encode(data).decode())
print(json.dumps(rows))`, runnerPath, providerPath], bypassSession: true, timeoutMs: 10_000 });
if (read.exitCode !== 0 || read.timedOut) throw new Error("runner_remote_retirement_state_unavailable");
const [runnerBytes, providerBytes] = (JSON.parse(read.stdout) as string[]).map(value => Buffer.from(value, "base64"));
const runner = record(JSON.parse(runnerBytes!.toString("utf8")));
if (runner.lifecycle === "suspended") return;
const runs = await input.db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.execution.binding.companyId),
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
inArray(heartbeatRuns.id, [identity.runId, input.execution.binding.runId])));
const prior = runs.find(run => run.id === identity.runId);
const current = runs.find(run => run.id === input.execution.binding.runId);
if (!prior || !current || !TERMINAL_HEARTBEAT_RUN_STATUSES.has(prior.status))
throw new Error("runner_remote_retirement_authority_unverified");
const profile = record(prior.runnerProfileJson);
const previous = parseNativeExecutionInput(profile.nativeExecutionInput);
const unsettledReason = retiredComputerCheckpointUnsettledReason({ execution: input.execution, priorExecution: previous, identity, runner,
control: record(JSON.parse(controlBytes.toString("utf8"))), provider: record(JSON.parse(providerBytes!.toString("utf8"))),
checkpoint: record(profile.sessionCheckpoint), currentCheckpoint: record(record(current.runnerProfileJson).sessionCheckpoint) });
if (unsettledReason !== null)
throw new Error(`runner_remote_retirement_checkpoint_unsettled: ${unsettledReason}`);
const priorAuthority = await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: input.root, identity, execution: input.execution,
allowVerifiedBackup: false, allowRetainedWarmRunner: false, remoteRunnerState: true, runnerExecutionTarget: input.target });
if (priorAuthority !== "verified")
throw new Error(`runner_remote_retirement_checkpoint_unsettled: prior_authority_${priorAuthority}`);
// Recheck exact durable retirement after all awaited reads; never infer death
// from a missing PID or turn status. The compare-and-swap preserves ambiguity.
const workspace = readNativeComputerWorkspaceReference(profile.nativeComputerWorkspace);
if (!workspace || !input.target.environmentId || !(await computerService(input.db).isRetired({
companyId: input.execution.binding.companyId, environmentId: input.target.environmentId,
owner: workspace.computerOwner, agentId: input.execution.binding.agentId, runId: identity.runId,
})) || !controlBytes.equals(readBoundedNativeFile(controlPath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "recovery_state_too_large")))
throw new Error("runner_remote_retirement_evidence_changed");
await compareAndSealRetiredComputerCheckpoint({ runner: input.runner, runnerPath, providerPath,
runnerBytes: runnerBytes!, providerBytes: providerBytes! });
}
/** Exact retirement is checked by the caller; publication must still compare
* the complete bytes so a changed or interrupted checkpoint is never rotated. */
export async function compareAndSealRetiredComputerCheckpoint(input: {
runner: CommandManagedRuntimeRunner; runnerPath: string; providerPath: string;
runnerBytes: Buffer; providerBytes: Buffer;
}): Promise<void> {
const result = await input.runner.execute({ command: "python3", args: ["-c", String.raw`
import sys,os,stat,json,hashlib,tempfile
runner,provider,expected_runner,expected_provider=sys.argv[1:]
for p,digest in [(runner,expected_runner),(provider,expected_provider)]:
s=os.lstat(p)
if not stat.S_ISREG(s.st_mode) or s.st_size>16777216:raise RuntimeError('unsafe recovery state')
with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:data=f.read(16777217)
if len(data)>16777216:raise RuntimeError('unsafe recovery state')
if hashlib.sha256(data).hexdigest()!=digest:raise RuntimeError('recovery evidence changed')
if p==runner:state=json.loads(data)
state['lifecycle']='suspended'
fd,tmp=tempfile.mkstemp(prefix='.retired-checkpoint-',dir=os.path.dirname(runner))
try:
with os.fdopen(fd,'w') as f:json.dump(state,f,separators=(',',':'));f.flush();os.fsync(f.fileno())
# Recheck immediately before publication, including the provider checkpoint.
for p,digest in [(runner,expected_runner),(provider,expected_provider)]:
with os.fdopen(os.open(p,os.O_RDONLY|os.O_NOFOLLOW),'rb') as f:current=f.read(16777217)
if hashlib.sha256(current).hexdigest()!=digest:raise RuntimeError('recovery evidence changed')
os.replace(tmp,runner)
finally:
if os.path.exists(tmp):os.unlink(tmp)
`, input.runnerPath, input.providerPath, createHash("sha256").update(input.runnerBytes).digest("hex"), createHash("sha256").update(input.providerBytes).digest("hex")],
bypassSession: true, timeoutMs: 10_000 });
if (result.exitCode !== 0 || result.timedOut) throw new Error("runner_remote_retirement_checkpoint_cas_failed");
}
async function readRemoteRunnerState(input: {
runner: CommandManagedRuntimeRunner;
stateDirectory: string;
}): Promise<Record<string, unknown>> {
const statePath = posix.join(input.stateDirectory, "runner-state.json");
const escapedPath = statePath.replaceAll("'", "'\\''");
const result = await input.runner.execute({
command: "sh",
args: ["-c", `test -f '${escapedPath}' && base64 < '${escapedPath}'`],
bypassSession: true,
timeoutMs: 10_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error(
`runner_remote_state_unavailable: exit=${result.exitCode} timedOut=${result.timedOut}${result.stderr.trim() ? ` ${redactSensitiveText(result.stderr).trim().slice(-512)}` : ""}`,
);
}
return record(
JSON.parse(
Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"),
),
);
}
async function readRemoteRunnerProviderState(input: {
runner: CommandManagedRuntimeRunner;
stateDirectory: string;
execution: NativeExecutionInput;
}): Promise<Record<string, unknown>> {
const statePath = posix.join(
input.stateDirectory,
runnerProviderStateFilename(input.execution),
);
const escapedPath = statePath.replaceAll("'", "'\\''");
const result = await input.runner.execute({
command: "sh",
args: ["-c", `test -f '${escapedPath}' && base64 < '${escapedPath}'`],
bypassSession: true,
timeoutMs: 10_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error("runner_remote_provider_state_unavailable");
}
return record(
JSON.parse(
Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"),
),
);
}
const REMOTE_RUNNER_PROCESS_IDENTITY_WAIT_MS = 20_000;
const REMOTE_RUNNER_PROCESS_POLL_MS = 1_000;
// Linux boot identity plus start ticks identifies a process generation across
// PID reuse. exec preserves both the launch shell's PID and its start ticks.
const REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT =
"process_fingerprint=$(node -e 'const fs=require(\"node:fs\");try{const stat=fs.readFileSync(\"/proc/\"+process.argv[1]+\"/stat\",\"utf8\");const end=stat.lastIndexOf(\") \");const ticks=stat.slice(end+2).trim().split(/\\s+/)[19];const boot=fs.readFileSync(\"/proc/sys/kernel/random/boot_id\",\"utf8\").trim();if(end<0||!/^\\d+$/.test(ticks||\"\")||!/^[-a-fA-F0-9]+$/.test(boot))process.exit(4);process.stdout.write(\"linux:\"+boot+\":\"+ticks)}catch{process.exit(4)}' \"$pid\")";
const REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT =
'set -eu; identity_path=$1; expected_nonce=$2; expected_runner_id=$3; expected_pid=$4; test -f "$identity_path" && test ! -L "$identity_path" || exit 3; { IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path"; test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test "$pid" = "$expected_pid" && test -n "$started_at" || exit 4; kill -0 "$pid" 2>/dev/null || exit 3; if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 4; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 4; fi';
export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT =
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT +
'; then :; else process_fingerprint=""; fi; identity_tmp="${identity_path}.tmp.$$"; printf "%s\\n%s\\n%s\\n%s\\n%s\\n" "$identity_nonce" "$$" "$started_at" "$runner_instance_id" "$process_fingerprint" > "$identity_tmp"; chmod 0600 "$identity_tmp"; mv -f -- "$identity_tmp" "$identity_path"; exec "$@"';
const REMOTE_RUNNER_FAILED_IDENTITY_CLEANUP_SCRIPT =
'set -eu; identity_path=$1; expected_nonce=$2; expected_runner_id=$3; marker_wait=0; while { test ! -f "$identity_path" || test -L "$identity_path"; } && test "$marker_wait" -lt 50; do marker_wait=$((marker_wait + 1)); sleep 0.1; done; test -f "$identity_path" && test ! -L "$identity_path" || exit 3; { IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path"; test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test -n "$started_at" || exit 4; case "$pid" in ""|*[!0-9]*) exit 4 ;; esac; test "$pid" -gt 0 || exit 4; if kill -0 "$pid" 2>/dev/null; then if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 4; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 4; fi; signal_target=$pid; if command -v ps >/dev/null 2>&1; then session_id=$(ps -o sid= -p "$pid" 2>/dev/null | tr -d " ") || true; if test "$session_id" = "$pid"; then signal_target="-$pid"; fi; fi; kill -TERM -- "$signal_target" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true; term_wait=0; while kill -0 "$pid" 2>/dev/null && test "$term_wait" -lt 50; do term_wait=$((term_wait + 1)); sleep 0.1; done; if kill -0 "$pid" 2>/dev/null; then kill -KILL -- "$signal_target" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true; kill_wait=0; while kill -0 "$pid" 2>/dev/null && test "$kill_wait" -lt 50; do kill_wait=$((kill_wait + 1)); sleep 0.1; done; fi; kill -0 "$pid" 2>/dev/null && exit 5; fi; test -f "$identity_path" && test ! -L "$identity_path" || exit 4; { IFS= read -r final_nonce; IFS= read -r final_pid; IFS= read -r final_started_at; IFS= read -r final_runner_id; } < "$identity_path"; test "$final_nonce" = "$nonce" && test "$final_pid" = "$pid" && test "$final_started_at" = "$started_at" && test "$final_runner_id" = "$runner_id" || exit 4; rm -f -- "$identity_path"';
export function parseRemoteRunnerProcessIdentity(
value: string,
expected: { nonce: string; runnerInstanceId: string },
): { pid: number; startedAt: string; processStartFingerprint?: string } | null {
const [nonce, rawPid, startedAt, runnerInstanceId, processStartFingerprint, ...remainder] = value
.trim()
.split("\n");
const pid = Number(rawPid);
if (
remainder.length > 0 ||
nonce !== expected.nonce ||
runnerInstanceId !== expected.runnerInstanceId ||
!Number.isSafeInteger(pid) ||
pid <= 0 ||
!startedAt ||
Number.isNaN(new Date(startedAt).getTime()) ||
(processStartFingerprint !== undefined && !/^linux:[0-9a-f-]+:[0-9]+$/.test(processStartFingerprint))
) {
return null;
}
return { pid, startedAt, ...(processStartFingerprint ? { processStartFingerprint } : {}) };
}
/** Verify remote ownership before exposing the transport's authenticated adoption path. */
export async function verifyRemoteRunnerReattachment(input: {
claim: Extract<
NativeRestartRecoveryClaim,
{ kind: "reattach_remote_runner" }
>;
target: AdapterExecutionTarget | null | undefined;
identity: Record<string, unknown>;
runId: string;
normalizedSessionId: string;
}) {
const { claim, target, identity } = input;
if (
!adapterExecutionTargetIsCommandBacked(target) ||
!target.runner ||
claim.runId !== input.runId ||
identity.runId !== input.runId ||
identity.normalizedSessionId !== input.normalizedSessionId ||
claim.remote.providerLeaseId !== (target.transport === "computer" ? target.resourceAuthority.ownerId : target.sandboxLeaseAcquisition?.providerLeaseId) ||
(target.transport === "sandbox" && target.sandboxLeaseAcquisition?.outcome === "replacement") ||
claim.remote.remoteCwd !== target.remoteCwd
) {
throw new Error("native_remote_recovery_lease_mismatch");
}
if (target.transport === "computer") {
const owner = claim.remote.computerOwner;
if (!owner || owner.computerId !== target.resourceAuthority.computerId || owner.ownerId !== target.resourceAuthority.ownerId ||
owner.generation !== target.resourceAuthority.generation || owner.listenerPort !== target.listenerPort ||
!(await target.inspectProcess()).running) throw new Error("computer_process_recovery_mismatch");
}
const runner = target.runner;
const stateDirectory = posix.join(
remoteRunnerStorageRoots(target, input.normalizedSessionId).sessionRoot,
"runner",
);
const runnerState = await readRemoteRunnerState({ runner, stateDirectory });
for (const field of [
"runId",
"normalizedSessionId",
"runnerInstanceId",
"environmentLeaseId",
"turnId",
"itemId",
]) {
if (
typeof identity[field] !== "string" ||
!identity[field] ||
runnerState[field] !== identity[field]
) {
throw new Error("runner_remote_recovery_identity_mismatch");
}
}
const identityPath = posix.join(stateDirectory, "runner-process.identity");
const marker = await runner.execute({
command: "sh",
args: [
"-c",
'test -f "$1" && test ! -L "$1" && cat -- "$1"',
"paperclip-runner-recovery-identity",
identityPath,
],
bypassSession: true,
timeoutMs: 10_000,
});
const nonce = marker.stdout.split("\n")[0] ?? "";
const runnerInstanceId = identity.runnerInstanceId as string;
const processIdentity =
marker.exitCode === 0 && !marker.timedOut && /^[a-zA-Z0-9_-]+$/.test(nonce)
? parseRemoteRunnerProcessIdentity(marker.stdout, {
nonce,
runnerInstanceId,
})
: null;
if (!processIdentity?.processStartFingerprint)
throw new Error("runner_remote_process_identity_unavailable");
const expectedStartFingerprint = processIdentity.processStartFingerprint;
const check = async (signal?: NodeJS.Signals) => {
const result = await runner.execute({
command: "sh",
args: [
"-c",
`${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; test "$process_fingerprint" = "$5" || exit 4` +
(signal ? '; kill -"$6" "$pid"' : ""),
"paperclip-runner-recovery-check",
identityPath,
nonce,
runnerInstanceId,
String(processIdentity.pid),
expectedStartFingerprint,
...(signal ? [signal.slice(3)] : []),
],
bypassSession: true,
timeoutMs: 10_000,
});
return result.exitCode === 0 && !result.timedOut;
};
if (!(await check()))
throw new Error("runner_remote_process_identity_unavailable");
// These checks authorize an attempt to reconnect, not the task itself. The
// transport still requires authentication against the exact durable PRP key.
return {
...processIdentity,
processGroupId: null,
isAlive: () => check(),
signal: check,
};
}
async function waitForRemoteRunnerProcessIdentity(input: {
runner: CommandManagedRuntimeRunner;
identityPath: string;
nonce: string;
runnerInstanceId: string;
requireProcessStartFingerprint?: boolean;
}): Promise<{ pid: number; startedAt: string; processStartFingerprint?: string }> {
const deadline = Date.now() + REMOTE_RUNNER_PROCESS_IDENTITY_WAIT_MS;
while (Date.now() < deadline) {
const result = await input.runner
.execute({
command: "sh",
args: [
"-c",
'test -f "$1" && test ! -L "$1" && cat -- "$1"',
"paperclip-runner-process-identity",
input.identityPath,
],
bypassSession: true,
// An owned remote command includes SSH and supervisor admission before
// reading this tiny file. Its normal roundtrip can exceed two seconds.
// Give the RPC room while preserving the total identity-wait deadline.
timeoutMs: Math.max(1, Math.min(10_000, deadline - Date.now())),
})
.catch(() => null);
const identity =
result && result.exitCode === 0 && !result.timedOut
? parseRemoteRunnerProcessIdentity(result.stdout, input)
: null;
if (identity && (!input.requireProcessStartFingerprint || identity.processStartFingerprint)) return identity;
await new Promise<void>((resolve) => setTimeout(resolve, 100));
}
throw new Error("runner_remote_process_identity_unavailable");
}
async function cleanupRemoteRunnerAfterIdentityFailure(input: {
runner: CommandManagedRuntimeRunner;
identityPath: string;
nonce: string;
runnerInstanceId: string;
}): Promise<boolean> {
const result = await input.runner
.execute({
command: "sh",
args: [
"-c",
REMOTE_RUNNER_FAILED_IDENTITY_CLEANUP_SCRIPT,
"paperclip-runner-identity-failure-cleanup",
input.identityPath,
input.nonce,
input.runnerInstanceId,
],
bypassSession: true,
timeoutMs: 20_000,
})
.catch(() => null);
return result?.exitCode === 0 && result.timedOut === false;
}
export function createRemoteRunnerProcessLauncher(input: {
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
runner: CommandManagedRuntimeRunner;
remoteBinary: string;
processIdentityPath: string;
stateDirectory: string;
diagnosticsDirectory: string;
runnerInstanceId: string;
ensureArtifact?: () => Promise<void>;
requireProcessStartFingerprint?: boolean;
onSpawn?: (meta: {
pid: number;
processGroupId: number | null;
startedAt: string;
}) => Promise<void>;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
trace?: NativeRunTrace;
onRunnerProcessSpawned?: () => void;
onLaunchAttempt?: () => void;
}): (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle {
const runner = input.runner;
return (spec) => {
let launchedIdentity: {
nonce: string;
pid: number;
startedAt: string;
} | null = null;
const child: RunnerProcessHandle["child"] = {
pid: undefined,
exitCode: null,
signalCode: null,
kill: (requestedSignal) => {
const identity = launchedIdentity;
if (!identity) return false;
const signal =
requestedSignal === "SIGKILL" || requestedSignal === 9
? "KILL"
: requestedSignal === "SIGINT" || requestedSignal === 2
? "INT"
: "TERM";
// The durable marker, nonce, exact pid, and runner-id command line are
// revalidated in the sandbox immediately before signalling. A recycled
// pid or replaced marker therefore fails closed instead of killing an
// unrelated process.
void runner.execute({
command: "sh",
args: [
"-c",
`${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; kill -${signal} "$expected_pid"`,
"paperclip-runner-signal",
input.processIdentityPath,
identity.nonce,
input.runnerInstanceId,
String(identity.pid),
],
bypassSession: true,
timeoutMs: 10_000,
}).catch(async () => {
// kill() follows Node's synchronous child-process contract. A deleted
// sandbox or failed signal RPC must not reject outside that boundary
// and crash the controller. This is not a termination receipt: the
// monitor and cleanup verification still decide whether work stopped.
await input.onLog?.(
"stderr",
"Remote runner signal failed; process termination is not confirmed.\n",
);
}).catch(() => undefined);
return true;
},
};
const completion = (async () => {
if (input.ensureArtifact) {
if (input.trace) {
await input.trace.measure(
"runner.runtime.stage",
input.ensureArtifact,
{ parentName: "runner.session.startup" },
);
} else {
await input.ensureArtifact();
}
}
const launchStartedAtMs = Date.now();
// The provider's onSpawn callback is optional and some sandbox command
// runners cannot report a remote pid until after the command has begun
// streaming. Signal as soon as staging is complete and the launch RPC is
// dispatched; this is late enough to avoid preview retries during staging
// and early enough to avoid a callback-dependent deadlock.
input.onRunnerProcessSpawned?.();
await input.trace?.record({
name: "runner.process.dispatch",
parentName: "runner.session.startup",
startedAtMs: launchStartedAtMs,
endedAtMs: Date.now(),
attributes: { target: "remote" },
});
const identityNonce = randomUUID();
const remoteArgs = [...spec.args];
const diagnosticsArgumentIndex = remoteArgs.indexOf(
"--diagnostics-directory",
);
if (diagnosticsArgumentIndex >= 0) {
remoteArgs[diagnosticsArgumentIndex + 1] = input.diagnosticsDirectory;
} else {
remoteArgs.push("--diagnostics-directory", input.diagnosticsDirectory);
}
// Do not keep runnerd as the foreground command of a provider RPC. Some
// sandbox command/session transports impose a provider-side lifetime on
// that RPC even when Paperclip requests a longer timeout. Detach runnerd
// into its own session instead; its own bounded diagnostics directory and
// durable PRP state remain the authorities, and the controller monitors
// the exact persisted process identity below.
// From this exact boundary onward, a lost RPC acknowledgement may mean
// a live process. Asset preparation failures above prove no dispatch.
input.onLaunchAttempt?.();
const launchResult = input.target.transport === "computer"
? await input.target.launch({ command: "sh", args: ["-c", 'mkdir -p -- "$(dirname -- "$1")"; ' + REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
"paperclip-runner-child", input.processIdentityPath, identityNonce, input.runnerInstanceId,
input.diagnosticsDirectory, input.remoteBinary, ...remoteArgs],
cwd: input.target.remoteCwd, env: processEnvironment(spec.environment) })
.then(() => ({ exitCode: 0, timedOut: false }))
: await runner.execute({
command: "sh",
args: [
"-c",
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; child_script=$4; shift 4; umask 077; identity_dir=$(dirname -- "$identity_path"); mkdir -p -- "$identity_dir"; if command -v setsid >/dev/null 2>&1; then nohup setsid sh -c "$child_script" paperclip-runner-child "$identity_path" "$identity_nonce" "$runner_instance_id" "$@" </dev/null >/dev/null 2>&1 & else nohup sh -c "$child_script" paperclip-runner-child "$identity_path" "$identity_nonce" "$runner_instance_id" "$@" </dev/null >/dev/null 2>&1 & fi',
"paperclip-runner-launch",
input.processIdentityPath,
identityNonce,
input.runnerInstanceId,
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
input.diagnosticsDirectory,
input.remoteBinary,
...remoteArgs,
],
cwd: input.target.remoteCwd,
env: processEnvironment(spec.environment),
timeoutMs: 20_000,
bypassSession: true,
onLog: input.onLog,
});
if (launchResult.exitCode !== 0 || launchResult.timedOut) {
throw new Error(
launchResult.timedOut
? "runner_remote_process_launch_timed_out"
: "runner_remote_process_launch_failed",
);
}
let identity: { pid: number; startedAt: string };
try {
identity = await waitForRemoteRunnerProcessIdentity({
runner,
identityPath: input.processIdentityPath,
nonce: identityNonce,
runnerInstanceId: input.runnerInstanceId,
requireProcessStartFingerprint: input.requireProcessStartFingerprint,
});
} catch {
const cleanupStartedAtMs = Date.now();
const cleaned = await cleanupRemoteRunnerAfterIdentityFailure({
runner,
identityPath: input.processIdentityPath,
nonce: identityNonce,
runnerInstanceId: input.runnerInstanceId,
});
await input.trace?.record({
name: "runner.process.identity_failure_cleanup",
parentName: "runner.session.startup",
startedAtMs: cleanupStartedAtMs,
endedAtMs: Date.now(),
attributes: { cleaned },
});
if (!cleaned) {
throw new Error(
"runner_remote_process_identity_unavailable_cleanup_failed",
);
}
throw new Error("runner_remote_process_identity_unavailable");
}
launchedIdentity = { nonce: identityNonce, ...identity };
child.pid = identity.pid;
await input.onSpawn?.({
pid: identity.pid,
processGroupId: null,
startedAt: identity.startedAt,
});
await input.trace?.record({
name: "runner.process.launch",
parentName: "runner.session.startup",
startedAtMs: launchStartedAtMs,
endedAtMs: Date.now(),
attributes: { identitySource: "remote_marker", detached: true },
});
let consecutiveUnknownObservations = 0;
while (true) {
const observed = await runner.execute({
command: "sh",
args: [
"-c",
REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT,
"paperclip-runner-monitor",
input.processIdentityPath,
identityNonce,
input.runnerInstanceId,
String(identity.pid),
],
bypassSession: true,
timeoutMs: 10_000,
}).catch(() => null);
const identityConfirmed = observed?.exitCode === 0 && !observed.timedOut;
const identityLost = observed !== null && !observed.timedOut
&& (observed.exitCode === 3 || observed.exitCode === 4);
if (!identityLost) {
// SSH/provider failures do not prove that runnerd exited. In particular,
// concurrent artifact uploads can delay a monitor RPC while PRP stays
// healthy. Retry unknown observations without allocating another runner.
consecutiveUnknownObservations = identityConfirmed ? 0 : consecutiveUnknownObservations + 1;
if (consecutiveUnknownObservations >= 3) {
throw new Error(`runner_remote_process_monitor_unavailable exitCode=${observed?.exitCode ?? "unknown"} timedOut=${observed?.timedOut ?? "unknown"}`);
}
await new Promise<void>((resolve) =>
setTimeout(resolve, REMOTE_RUNNER_PROCESS_POLL_MS),
);
continue;
}
child.exitCode = null;
const identityMismatch = observed.exitCode === 4;
const diagnostic = await runner
.execute({
command: "sh",
args: [
"-c",
'set -eu; directory=$1; file="$directory/runnerd.stderr.log"; test -d "$directory" && test ! -L "$directory" && test -f "$file" && test ! -L "$file"; tail -c 65536 -- "$file"',
"paperclip-runner-diagnostics",
input.diagnosticsDirectory,
],
bypassSession: true,
timeoutMs: 10_000,
})
.catch(() => null);
const diagnosticTail =
diagnostic && diagnostic.exitCode === 0 && !diagnostic.timedOut
? redactSensitiveText(diagnostic.stdout).slice(-16_384).trim()
: "";
const durableState = await readRemoteRunnerState({
runner,
stateDirectory: input.stateDirectory,
}).catch(() => null);
const lifecycle =
typeof durableState?.lifecycle === "string"
? durableState.lifecycle
: "unavailable";
const recoverableFailure =
typeof durableState?.recoverableFailure === "string"
? durableState.recoverableFailure
: typeof durableState?.recoverable_failure === "string"
? durableState.recoverable_failure
: null;
const stateDiagnostics = Array.isArray(durableState?.diagnostics)
? durableState.diagnostics
.filter((value): value is string => typeof value === "string")
.slice(-4)
.map((value) => redactSensitiveText(value).slice(-1_000))
: [];
const stateSummary = `runner_remote_process_exited lifecycle=${lifecycle}${recoverableFailure ? ` recoverableFailure=${redactSensitiveText(recoverableFailure).slice(-1_000)}` : ""}${stateDiagnostics.length > 0 ? ` diagnostics=${JSON.stringify(stateDiagnostics)}` : ""}`;
return {
code: null,
signal: null,
stdout: "",
stderr: identityMismatch
? "runner_remote_process_identity_mismatch"
: diagnosticTail || stateSummary,
};
}
})();
return { child, completion };
};
}
/**
* Select the remote runner transport before any artifact is staged or provider
* endpoint is acquired. Sandbox ingress is available only to a run already
* authorized by native runtime selection.
*/
export function resolveRemoteRunnerTransportMode(input: {
target: AdapterExecutionTarget;
runnerIngressAuthorized: boolean;
}): "listen_ws" | "dial_wss" {
if (input.target.kind !== "remote") {
throw new Error("runner_transport_ineligible: remote target is required");
}
const requiredMode =
adapterExecutionTargetIsCommandBacked(input.target) &&
input.target.effectiveCapabilities?.runnerWebSocketIngress === true
? "listen_ws"
: "dial_wss";
if (requiredMode === "listen_ws" && !input.runnerIngressAuthorized) {
throw new Error("runner_ingress_unavailable");
}
return requiredMode;
}
export function remoteCheckpointIncompleteFailure(
settlement: "settled" | "unsettled",
incompleteReason: "unavailable" | "not_suspended" | null,
): Error | null {
// A transport that never completed provider bootstrap has no provider state
// to preserve; its original launch failure remains authoritative. Once
// runnerd has proved suspension, however, an unreadable or incomplete
// checkpoint must fail the required close so outer sandbox release is
// withheld. Process containment still happens in the transport finally.
if (settlement === "unsettled") return null;
return new Error(
`runner_remote_checkpoint_incomplete: exact suspended harness state unavailable (${incompleteReason ?? "unknown"})`,
);
}
/** Production runnerd backend seam, exported so provider wiring can be regression tested. */
export async function createRunnerdBackend(input: {
db: Db;
execution: NativeExecutionInput;
runnerInstanceId: string;
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
restartRecovery?: NativeRestartRecoveryClaim;
durableEnvironmentLeaseId?: string;
onRemoteLaunchAttempt?: () => void;
onSpawn?: (meta: {
pid: number;
processGroupId: number | null;
startedAt: string;
}) => Promise<void>;
dotWorkspaceRoot?: string;
runnerEnvironment?: NodeJS.ProcessEnv;
/** Private grant materialization; never a user-configured host path. */
managedAiCredentialHome?: string;
runnerExecutionTarget?: AdapterExecutionTarget | null;
/** Resolved per-run authorization; not an independent instance setting. */
runnerIngressAuthorized?: boolean;
runnerPublicUrl?: string | null;
runnerCaBundlePath?: string | null;
runnerRemoteBinaryPath?: string | null;
runnerRemoteCodexPath?: string | null;
runnerRemoteCodexNpmSpec?: string | null;
runnerRemoteProviderPackPath?: string | null;
trace?: NativeRunTrace;
toolTrace?: NativeToolTrace;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
stopTaskForReassignment?: (target: { companyId: string; issueId: string; agentId: string; runId: string | null }) => Promise<void>;
syncIssueExternalObjects?: (issueId: string) => Promise<void>;
enqueueWakeup?: (
agentId: string,
options: {
source: "assignment" | "automation";
triggerDetail: "system";
reason: "issue_assigned" | "issue_commented";
payload: Record<string, unknown>;
idempotencyKey: string;
requestedByActorType: "agent";
requestedByActorId: string;
contextSnapshot: Record<string, unknown>;
},
) => Promise<unknown>;
}): Promise<NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession }> {
const sessionScopeId = nativeSessionScopeKey(input.execution);
const scopeOwner = executingRunnerdSessionScopes.get(sessionScopeId);
if (scopeOwner && scopeOwner !== input.execution.binding.runId) {
throw new Error("native_session_supervisor_busy");
}
if (initializingSessionToolAuthorities.has(sessionScopeId)) {
throw new Error("native_session_supervisor_busy");
}
initializingSessionToolAuthorities.add(sessionScopeId);
try {
let retainedTransition: VerifiedWarmTransitionBinding | undefined;
// executePaperclipNativeSession holds the full session-scope claim and
// verifies/migrates the durable root before it acquires the coordinator
// lease. Avoid reclassifying the same root after that path has marked its
// retained warm owner busy for this run. Direct backend construction still
// performs the complete fail-closed verification here.
if (
executingRunnerdSessionScopes.get(sessionScopeId) !==
input.execution.binding.runId ||
hasRetainedWarmTransitionEvidence(scopedRunnerdStateRoot(input.execution))
) {
retainedTransition = await migrateRunnerdStateRootForExecution({
db: input.db,
execution: input.execution,
allowVerifiedBackup:
input.runnerExecutionTarget?.kind === "remote" &&
input.runnerExecutionTarget.transport === "sandbox",
allowRetainedWarmRunner: false,
allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote",
onLog: input.onLog,
restartRecovery: input.restartRecovery,
runnerExecutionTarget: input.runnerExecutionTarget,
});
}
return await createRunnerdBackendWithinSessionClaim(
input,
sessionScopeId,
retainedTransition,
);
} finally {
initializingSessionToolAuthorities.delete(sessionScopeId);
}
}
/** Dot uses the same managed process launcher and authenticated PRP transports.
* Its execution contract still exposes no workspace, shell, or provider secrets. */
async function prepareRemoteDotRunner({ input, target, runner, root, identity }: {
input: Parameters<typeof createRunnerdBackend>[0];
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
runner: CommandManagedRuntimeRunner;
root: string;
identity: NonNullable<NonNullable<Parameters<typeof createNativeSessionBackend>[1]>["dotRunnerOptions"]>["identity"];
}) {
const runnerBinary = input.runnerRemoteBinaryPath?.trim() || resolvePaperclipRunnerBinary();
const storage = remoteRunnerStorageRoots(target, identity.normalizedSessionId);
const remoteRoot = storage.runtimeRoot;
const remoteBinary = posix.join(remoteRoot, "bin", "paperclip-runnerd");
const runnerStateDirectory = posix.join(storage.sessionRoot, "runner");
const requiredMode = resolveRemoteRunnerTransportMode({ target, runnerIngressAuthorized: input.runnerIngressAuthorized === true });
const artifact = readRunnerdArtifactBinding(runnerBinary);
const prepare = async () => {
if (input.restartRecovery?.kind !== "reattach_remote_runner") {
if (!input.runnerRemoteBinaryPath?.trim()) {
const platform = await runner.execute({ command: "sh", args: ["-c", "uname -s; uname -m"], bypassSession: true, timeoutMs: 10000 });
const [os, arch] = platform.stdout.trim().split(/\r?\n/);
if (platform.exitCode !== 0 || platform.timedOut || os !== (process.platform === "darwin" ? "Darwin" : "Linux")
|| ![process.arch === "x64" ? "x86_64" : "aarch64", process.arch].includes(arch)) {
throw new Error("runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the sandbox platform");
}
}
await stageRemoteRunnerFile({ target, runner, sourcePath: runnerBinary, targetPath: remoteBinary, mode: 0o700 });
}
const metadata = await runner.execute({ command: remoteBinary, args: ["--build-metadata"], bypassSession: true, timeoutMs: 30000 });
if (metadata.exitCode !== 0 || metadata.timedOut) throw new Error("runner_remote_artifact_verification_failed");
assertRemoteRunnerBuildMetadata(JSON.parse(metadata.stdout), requiredMode, "openai_dot_mcp");
const checksum = await runner.execute({ command: "sh", args: ["-c",
'if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi', "paperclip-dot-artifact", remoteBinary],
bypassSession: true, timeoutMs: 10000 });
if (checksum.exitCode !== 0 || checksum.timedOut || `sha256:${checksum.stdout.trim().split(/\s+/)[0]}` !== artifact.digest) {
throw new Error("runner_remote_artifact_verification_failed");
}
};
const readProviderState = async () => {
const result = await runner.execute({ command: "sh", args: ["-c",
'set -eu; if test ! -e "$1" && test ! -L "$1"; then exit 3; fi; test -f "$1" && test ! -L "$1"; test "$(wc -c < "$1")" -le 33554432; base64 < "$1"',
"paperclip-dot-checkpoint", posix.join(runnerStateDirectory, "dot-provider-state.json")], bypassSession: true, timeoutMs: 10000 });
if (result.exitCode === 3 && !result.timedOut) return null;
if (result.exitCode !== 0 || result.timedOut) throw new Error("dot_provider_checkpoint_unavailable");
const state = record(JSON.parse(Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8")));
if (state.schema !== "paperclip.runner.dot-provider-state.v1" || state.runId !== identity.runId
|| state.sessionId !== identity.normalizedSessionId || state.turnId !== identity.turnId) throw new Error("dot_runner_checkpoint_authority_mismatch");
// Retain inspectable recovery evidence on the controller without changing
// which execution target is authoritative for replay.
mkdirSync(join(root, "runner"), { recursive: true, mode: 0o700 });
const saved = join(root, "runner", "dot-provider-state.json");
const temporary = `${saved}.${randomUUID()}.tmp`;
writeFileSync(temporary, JSON.stringify(state), { mode: 0o600 });
renameSync(temporary, saved);
return state;
};
let spawned: () => void = () => {};
const processStarted = new Promise<void>(resolve => { spawned = resolve; });
// A transport handshake alone does not prove a recoverable process owner.
// Do not attach the Dot broker until the exact sandbox marker is validated.
let identityReady: () => void = () => {};
let identityFailed: (error: unknown) => void = () => {};
const processIdentityReady = new Promise<void>((resolve, reject) => { identityReady = resolve; identityFailed = reject; });
void processIdentityReady.catch(() => {});
const adoptExistingRunner = input.restartRecovery?.kind === "reattach_remote_runner"
? await verifyRemoteRunnerReattachment({ claim: input.restartRecovery, target, identity: { ...identity },
runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution) }) : undefined;
if (adoptExistingRunner) { spawned(); identityReady(); }
const launch = createRemoteRunnerProcessLauncher({ target, runner, remoteBinary,
processIdentityPath: posix.join(runnerStateDirectory, "runner-process.identity"), stateDirectory: runnerStateDirectory,
diagnosticsDirectory: posix.join(remoteRoot, "diagnostics"), runnerInstanceId: input.runnerInstanceId,
requireProcessStartFingerprint: true,
onSpawn: async meta => { await input.onSpawn?.(meta); identityReady(); },
onLog: input.onLog, trace: input.trace, onRunnerProcessSpawned: () => spawned() });
const runnerProcessLauncher: typeof launch = spec => {
const handle = launch(spec);
void handle.completion.then(() => identityFailed(new Error("runner_remote_process_identity_unavailable")), identityFailed);
return handle;
};
const controlPlaneRegistration: NonNullable<NonNullable<NonNullable<Parameters<typeof createNativeSessionBackend>[1]>["dotRunnerOptions"]>["controlPlaneRegistration"]> = async authority => {
let transport: PaperclipRunnerTransport;
// Validate outbound URL before staging; provider ingress needs the staged
// listener binary before requesting the provider's private endpoint.
if (requiredMode === "dial_wss") {
transport = await resolvePaperclipRunnerTransport({ target, runId: input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath, runnerIngressAuthorized: input.runnerIngressAuthorized === true });
await prepare();
} else {
await prepare();
transport = await resolvePaperclipRunnerTransport({ target, runId: input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused", runnerIngressAuthorized: input.runnerIngressAuthorized === true });
}
await input.onLog?.("stderr", `[paperclip-runner] Dot controller target=remote transport=${transport.mode}\n`);
if (transport.mode !== "provider_ingress") {
const registration = await registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId,
runId: input.execution.binding.runId, authority });
let caBundlePath = transport.mode === "direct_outbound" ? transport.caBundlePath : undefined;
if (caBundlePath) {
const remoteCaBundlePath = posix.join(remoteRoot, "bin", "runner-ca-bundle.pem");
await stageRemoteRunnerFile({ target, runner, sourcePath: caBundlePath, targetPath: remoteCaBundlePath, mode: 0o600 });
caBundlePath = remoteCaBundlePath;
}
return { ...registration, ready: async () => { await processIdentityReady; }, connection: { mode: "connect" as const, connectUrl: transport.connectUrl,
...(caBundlePath ? { caBundlePath } : {}) } };
}
let outbound: ReturnType<typeof connectRunnerPrpIngress> | null = null;
let activation: Promise<void> | null = null;
return { connection: { mode: "listen" as const, listenAddress: transport.listenAddress,
listenPort: transport.listenPort, listenPath: transport.listenPath },
activate: () => { activation = (async () => { await processStarted;
outbound = connectRunnerPrpIngress({ authority, endpoint: transport.ingress }); })();
void activation.catch(() => {}); },
ready: async () => { await activation; if (!outbound) throw new Error("runner_ingress_unavailable"); await outbound.ready; await processIdentityReady; },
get failure() { return outbound?.failure; },
startupFailureCode: "runner_ingress_unavailable" as const,
release: async () => { if (outbound) await outbound.close(); else await transport.ingress.close(); },
};
};
return { runnerBinary, runnerStateDirectory, runnerProcessLauncher, readProviderState, controlPlaneRegistration, adoptExistingRunner };
}
async function createRunnerdBackendWithinSessionClaim(
input: Parameters<typeof createRunnerdBackend>[0],
sessionScopeId: string,
retainedTransition?: VerifiedWarmTransitionBinding,
): Promise<NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession }> {
let recoveryPending = retainedTransition !== undefined;
const target = input.runnerExecutionTarget ?? { kind: "local" as const };
const remoteTarget = target.kind === "remote" ? target : null;
const remoteCommandRunner = remoteTarget
? remoteTarget.transport === "ssh"
? createNativeSshCommandRunner({
spec: remoteTarget.spec,
defaultCwd: remoteTarget.remoteCwd,
})
: remoteTarget.transport === "computer" ? remoteTarget.processRunner : remoteTarget.runner
: null;
if (remoteTarget && !remoteCommandRunner) {
throw new Error(
"runner_transport_ineligible: remote process runner is unavailable",
);
}
const currentWakeComments = await resolveCurrentWakeCommentsBinding(
input.db,
input.execution.binding,
);
const pinnedSkills = new Set("runtimeContext" in input.execution ? input.execution.runtimeContext.skills.map((skill) => skill.key) : []);
const connectorAssignments = [...pinnedSkills].some(isConnectorSkill)
? await resolveConnectorAssignments(input.db, input.execution.binding) : [];
const reviewRun = await input.db.select({ contextSnapshot: heartbeatRuns.contextSnapshot })
.from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.execution.binding.runId),
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
)).limit(1).then((rows) => rows[0]);
const nativeReview = readNativeReviewAssignmentContext(reviewRun?.contextSnapshot);
if (nativeReview && !await getNativeReviewAssignment(input.db, {
...input.execution.binding, contextSnapshot: nativeReview,
})) throw new Error("native_review_assignment_no_longer_available");
// Remote Codex already sends dynamic tool calls over authenticated PRP. Keep
// the assigned gateway on the control plane instead of asking the sandbox to
// reach the host's HTTP origin (which may be private or loopback-only).
const relayAssignedMcp = (remoteTarget !== null && input.execution.provider.kind === "codex") || input.execution.provider.kind === "openai_dot";
const assignedMcpUrl = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_URL;
const assignedMcpToken = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_TOKEN;
const assignedMcpName = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_NAME;
const hasAssignedMcp = Boolean(assignedMcpName || assignedMcpUrl || assignedMcpToken);
if (relayAssignedMcp && hasAssignedMcp && (!assignedMcpName?.trim() || !assignedMcpUrl?.trim() || !assignedMcpToken?.trim())) {
throw new Error("assigned native MCP launch binding is incomplete");
}
const assignedGatewayPublicId = relayAssignedMcp && assignedMcpUrl
? new URL(assignedMcpUrl).pathname.match(/^\/mcp\/gateways\/([a-zA-Z0-9_-]+)$/)?.[1]
: undefined;
if (relayAssignedMcp && hasAssignedMcp && !assignedGatewayPublicId) {
throw new Error("assigned native MCP gateway path is invalid");
}
const assignedMcpTools = relayAssignedMcp && !nativeReview && assignedMcpUrl && assignedMcpToken
? await createAssignedMcpTools({
gateway: getAssignedMcpGateway(input.db),
gatewayPublicId: assignedGatewayPublicId!,
bearerToken: assignedMcpToken,
workMode: input.execution.task.workMode,
})
: undefined;
const dotAttachmentActor = input.execution.provider.kind === "openai_dot"
? await input.db.select({ config: agents.adapterConfig }).from(agents).where(and(
eq(agents.id, input.execution.binding.agentId), eq(agents.companyId, input.execution.binding.companyId),
)).limit(1).then(rows => rows[0]) : undefined;
const authority = new PaperclipRunnerToolAuthority(input.db, {
...(nativeReview ? { nativeReview } : {}),
connectorAssignments: connectorAssignments.filter((assignment) => pinnedSkills.has(assignment.skillKey)),
assignedMcpTools,
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
runId: input.execution.binding.runId,
agentId: input.execution.binding.agentId,
normalizedSessionId: nativeSessionKey(input.execution),
pinnedMcpDigest:
"runtimeContext" in input.execution
? input.execution.runtimeContext.mcp.digest
: undefined,
workMode: input.execution.task.workMode,
runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : undefined,
workspaceBridge: input.execution.provider.kind === "openai_dot" && !!input.dotWorkspaceRoot,
taskAttachmentRead: dotAttachmentActor?.config.dotAttachmentAccess === true,
dotRuntime: input.execution.provider.kind === "openai_dot",
assertBridgeAuthority: input.execution.provider.kind === "openai_dot" ? () => dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6) : undefined,
workspaceRoot: input.dotWorkspaceRoot ?? remoteTarget?.remoteCwd ?? input.execution.workspace.cwd ?? undefined,
executionTargetKind: target.kind,
readRemoteWorkspaceFile: remoteTarget && remoteCommandRunner
? (file) => readVerifiedRemoteWorkspaceFile({ runner: remoteCommandRunner, workspaceRoot: remoteTarget.remoteCwd, ...file })
: undefined,
currentWakeComments: currentWakeComments ?? undefined,
chatAttachmentReadScope: input.chatAttachmentReadScope,
stopTaskForReassignment: input.stopTaskForReassignment,
syncIssueExternalObjects: input.syncIssueExternalObjects,
enqueueWakeup: input.enqueueWakeup,
});
const authorityEpoch = new SessionToolAuthorityEpoch(
input.execution.binding.runId,
authority,
input.toolTrace,
);
let dynamicTools: Awaited<
ReturnType<SessionToolAuthorityEpoch["definitions"]>
>;
try {
dynamicTools = await authorityEpoch.definitions();
} catch (error) {
authorityEpoch.revoke();
throw error;
}
const root = runnerdStateRoot(input.execution);
const durableIdentity = readRunnerdDurableIdentity(root);
const durableBinding = retainedTransition
? loadRunnerdDurableBinding(input.execution, retainedTransition)
: durableIdentityMatchesSession(durableIdentity, input.execution)
? durableIdentity
: null;
const effectiveRunnerInstanceId =
durableBinding?.runnerInstanceId ?? input.runnerInstanceId;
const effectiveEnvironmentLeaseId =
durableBinding?.environmentLeaseId ??
input.durableEnvironmentLeaseId ??
input.execution.binding.executionWorkspaceId;
mkdirSync(root, { recursive: true, mode: 0o700 });
if (input.execution.schema === "paperclip.native-execution-input.v6") {
if (target.kind === "remote" && input.dotWorkspaceRoot) throw new Error("dot_remote_workspace_access_unavailable");
const remoteDot = target.kind === "remote"
? await prepareRemoteDotRunner({ input, target, runner: remoteCommandRunner!, root,
identity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId,
runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution),
turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` } })
: null;
const recoveredProcess = input.restartRecovery?.kind === "reattach_existing_runner" ? input.restartRecovery.process : null;
const backend = createNativeSessionBackend(input.execution, {
onSpawn: input.onSpawn, dynamicTools,
dynamicToolHandler: async call => { await dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6); return authorityEpoch.execute(call); },
completionFeedback: async result => { await dotRunnerBroker(input.db).assertRunAuthority(input.execution as import("../../vendor/paperclip-runner/index.js").NativeExecutionInputV6); await authorityEpoch.definitions(); return nativeCompletionFeedback(input.db, input.execution.binding.runId, result); },
dotRunnerOptions: {
stateDirectory: root, runnerBinary: remoteDot?.runnerBinary ?? resolvePaperclipRunnerBinary(),
...(remoteDot ?? {}),
identity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId,
runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution),
turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` },
port: dotRunnerBroker(input.db).port(input.execution),
controlPlaneRegistration: remoteDot?.controlPlaneRegistration ?? (async authority => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId, agentId: input.execution.binding.agentId,
runId: input.execution.binding.runId, authority })),
adoptExistingRunner: remoteDot?.adoptExistingRunner ?? (recoveredProcess ? { ...recoveredProcess,
isAlive: () => verifiedRecoveryProcessIsAlive(recoveredProcess),
signal: signal => signalVerifiedRecoveryProcess(recoveredProcess, signal) } : undefined),
},
});
const prior = sessionToolAuthorityEpochs.get(sessionScopeId); if (prior) prior.revoke();
sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch);
return { ...backend, descriptor: () => backend.descriptor(), openSession: input => backend.openSession(input),
recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options), bindManagedSession: session => session };
}
const remoteStorage = remoteTarget
? remoteRunnerStorageRoots(remoteTarget, nativeSessionKey(input.execution))
: null;
const remoteRuntimeRoot = remoteStorage?.runtimeRoot ?? null;
const requiresRemoteProviderPack =
remoteTarget !== null &&
(input.execution.provider.kind === "opencode" ||
input.execution.provider.kind === "acpx");
// Pi's explicit operator import supplies the target-platform authority. Never
// substitute a Mac controller daemon or trust an image's self-reported hashes.
// Existing explicit overrides and every other provider keep their old path.
const remotePiCompanion = await selectRemotePiCompanion({
provider: input.execution.provider, remote: remoteTarget !== null,
binaryOverride: input.runnerRemoteBinaryPath, packOverride: input.runnerRemoteProviderPackPath,
workspaceRoot: input.execution.workspace.cwd,
});
const configuredProviderPackRoot =
input.runnerRemoteProviderPackPath?.trim() || remotePiCompanion?.providerPack || null;
let expectedProviderPackManifest: RemoteProviderPackManifest | null = null;
const useBundledRemoteImageAssets = requiresRemoteProviderPack && !configuredProviderPackRoot &&
(input.execution.provider.kind === "opencode" ||
(input.execution.provider.kind === "acpx" && input.execution.provider.agent === "cursor"));
const retainedComputerPack = remoteTarget?.transport === "computer" &&
input.execution.provider.kind === "acpx" && durableBinding && durableIdentity && remoteCommandRunner && remoteStorage
? pinnedComputerProviderPack({
agentHome: remoteTarget.fileAuthority.agentHome, identity: durableIdentity,
provider: input.execution.provider,
control: JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"),
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")),
}) : null;
if (retainedComputerPack && remoteCommandRunner && remoteStorage && input.execution.provider.kind === "acpx") {
// The local attachment template chooses the digest. Remote content cannot
// authorize a newer pack or rewrite the existing provider's launch identity.
const [manifest, state] = await readPinnedComputerProviderMetadata({ runner: remoteCommandRunner,
packRoot: retainedComputerPack.root, stateDirectory: posix.join(remoteStorage.sessionRoot, "runner") });
expectedProviderPackManifest = validateRemoteProviderPackIdentity(manifest as RemoteProviderPackManifest, "", false,
{ digest: retainedComputerPack.digest, agent: input.execution.provider.agent });
if (expectedProviderPackManifest.digest !== retainedComputerPack.digest ||
expectedProviderPackManifest.payload.target.platform !== "linux" ||
expectedProviderPackManifest.payload.target.architecture !== "x64")
throw new Error("runner_remote_provider_pack_provenance_mismatch");
const launchDigest = computerAcpxLaunchProfileDigest({ authorityDigest: retainedComputerPack.digest,
command: retainedComputerPack.command, sidecar: retainedComputerPack.sidecar,
commandSha256: expectedProviderPackManifest.payload.artifacts.nodeCommand.sha256,
sidecarSha256: expectedProviderPackManifest.payload.artifacts.acpxSidecar.sha256 });
assertPinnedComputerProviderState({ state, launchProfileDigest: launchDigest,
normalizedSessionId: nativeSessionKey(input.execution), command: retainedComputerPack.command, sidecar: retainedComputerPack.sidecar });
} else if (useBundledRemoteImageAssets) {
expectedProviderPackManifest = readBundledRemoteProviderPackManifest();
} else if (requiresRemoteProviderPack) {
if (
!configuredProviderPackRoot ||
!existsSync(configuredProviderPackRoot) ||
!lstatSync(configuredProviderPackRoot).isDirectory()
) {
throw new Error(
"runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH with the build-owned provider pack",
);
}
expectedProviderPackManifest = readRemoteProviderPackManifest(
configuredProviderPackRoot,
);
}
const stagedRemoteProviderPackRoot = remoteTarget?.transport === "computer" && expectedProviderPackManifest
? computerProviderPackCachePath(remoteTarget.fileAuthority.agentHome, expectedProviderPackManifest.digest)
: remoteRuntimeRoot ? posix.join(remoteRuntimeRoot, "provider-pack") : null;
let activeRemoteProviderPackRoot: string | null = null;
const remoteBinary = remoteRuntimeRoot
? posix.join(remoteRuntimeRoot, "bin", "paperclip-runnerd")
: null;
// The transport hashes runnerBinary on the controller before an external
// launcher starts runnerd. Keep that artifact identity in the controller's
// filesystem; the remote launcher separately owns the sandbox command path.
// When an explicit remote artifact is configured, prepareRemoteRunner stages
// these exact bytes at remoteBinary before launch.
const controllerRunnerBinary = remoteTarget
? input.runnerRemoteBinaryPath?.trim() || remotePiCompanion?.runnerBinary || (useBundledRemoteImageAssets ? bundledRemoteRunnerBinary() : resolvePaperclipRunnerBinary())
: resolvePaperclipRunnerBinary();
const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null;
const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() ||
(remoteTarget?.transport === "computer" && !explicitRemoteCodex
? `@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`
: null);
if (explicitRemoteCodex && remoteCodexNpmSpec) {
throw new Error("runner_remote_codex_source_conflict");
}
const remoteCodexBinary =
remoteRuntimeRoot && input.execution.provider.kind === "codex"
? remoteCodexNpmSpec
? posix.join(
remoteRuntimeRoot,
"harnesses",
"codex",
"node_modules",
".bin",
"codex",
)
: posix.join(remoteRuntimeRoot, "bin", "codex")
: null;
const remoteSessionRoot = remoteStorage?.sessionRoot ?? null;
const remoteStateDirectory = remoteSessionRoot
? posix.join(remoteSessionRoot, "runner")
: undefined;
const remoteRunnerFilesystemRoot = remoteSessionRoot
? posix.join(remoteSessionRoot, "filesystem")
: null;
const persistenceProfile = resolveNativeHarnessPersistenceProfile(
input.execution,
);
const sandboxLeaseAcquisition =
remoteTarget?.transport === "sandbox"
? (remoteTarget.sandboxLeaseAcquisition ?? null)
: null;
const remotePersistencePath = (
directory: NativeHarnessPersistenceDirectory,
): string | null =>
directory.location === "runner"
? (remoteStateDirectory ?? null)
: remoteRunnerFilesystemRoot
? posix.join(remoteRunnerFilesystemRoot, directory.name)
: null;
const sourceRuntimeContext =
"runtimeContext" in input.execution ? input.execution.runtimeContext : null;
// Computer bundles are immutable. Each bootstrap attempt publishes new
// references instead of overlaying a previous read-only snapshot. Old
// snapshots remain until future bounded maintenance; provider HOME and user
// files never participate in this materialization.
const remoteContextRoot = remoteRunnerFilesystemRoot
? posix.join(remoteRunnerFilesystemRoot, "context", ...(remoteTarget?.transport === "computer" ? ["snapshots", randomUUID()] : []))
: null;
const remoteRuntimeContext: NativeRuntimeContextSnapshot | null =
remoteRunnerFilesystemRoot && sourceRuntimeContext
? {
...sourceRuntimeContext,
...(remoteTarget?.transport === "computer" ? {
persistentAgentHome: { rootPath: remoteTarget.fileAuthority.agentHome },
} : {}),
instructions: {
...sourceRuntimeContext.instructions,
bundle: {
...sourceRuntimeContext.instructions.bundle,
rootPath: posix.join(
remoteContextRoot!,
"instructions",
),
},
},
skills: sourceRuntimeContext.skills.map((skill, index) => ({
...skill,
bundle: {
...skill.bundle,
rootPath: posix.join(
remoteContextRoot!,
"skills",
`${index}-${skill.bundle.digest.slice(0, 12)}`,
),
},
})),
}
: sourceRuntimeContext;
let remotePrepared = false;
let remoteHarnessStatePrepared = false;
let selectedRemoteMode: "dial_wss" | "listen_ws" | null = null;
let remoteCaBundleMapping: { sourcePath: string; targetPath: string } | null =
null;
let resolveRemoteRunnerProcessSpawned: (() => void) | null = null;
const remoteRunnerProcessSpawned = remoteTarget
? new Promise<void>((resolveSpawned) => {
resolveRemoteRunnerProcessSpawned = resolveSpawned;
})
: Promise.resolve();
const verifyRemoteRunner = async (
requiredMode: "dial_wss" | "listen_ws",
executable = remoteBinary,
) => {
if (!remoteTarget || !remoteCommandRunner || !executable) return;
const metadataResult = await remoteCommandRunner.execute({
command: executable,
args: ["--build-metadata"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 30_000,
});
if (metadataResult.exitCode !== 0 || metadataResult.timedOut) {
throw new Error("runner_remote_artifact_verification_failed");
}
let metadata: Record<string, unknown>;
try {
metadata = JSON.parse(metadataResult.stdout) as Record<string, unknown>;
} catch (error) {
throw new Error("runner_remote_artifact_metadata_invalid", {
cause: error,
});
}
assertRemoteRunnerBuildMetadata(metadata, requiredMode);
};
const reportedCodexVersions = new Set<string>();
const verifyRemoteCodex = async (executable = remoteCodexBinary) => {
if (!remoteTarget || !remoteCommandRunner || !executable) return;
const versionResult = await remoteCommandRunner.execute({
command: executable,
args: ["--version"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 30_000,
});
if (versionResult.exitCode !== 0 || versionResult.timedOut) {
throw new Error("runner_remote_codex_artifact_verification_failed");
}
const versionOutput = `${versionResult.stdout}\n${versionResult.stderr}`;
const version = parseCodexCliVersion(versionOutput);
if (!version || !isSupportedRemoteCodexVersion(version)) {
throw new Error(
`runner_remote_provider_artifact_incompatible: supported Codex versions ${REMOTE_CODEX_SUPPORTED_RANGE}, received ${version ?? "an unrecognized or prerelease version"}; install a supported stable Codex release or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`,
);
}
// A Codex inside the compatibility window can still be too old for the
// configured model: the ChatGPT backend rejects a model from clients
// below the model's floor on every turn. Fail before launch with the
// exact gap, so a stale sandbox image is not reported as an account
// problem. When a preinstalled Codex fails here and an npm spec is
// configured, the caller falls back to installing the pinned release.
const configuredModel = input.execution.provider.kind === "codex"
? input.execution.provider.model
: null;
const modelMinimum = minimumCodexCliVersionForModel(configuredModel);
if (modelMinimum && !codexCliVersionAtLeast(version, modelMinimum)) {
throw new Error(
`runner_remote_provider_artifact_incompatible: ${normalizeCodexModel(configuredModel)} requires Codex ${modelMinimum} or newer with ChatGPT sign-in, received ${version} from the sandbox image; promote a sandbox image with Codex ${REMOTE_PROVIDER_PACK_PINS.codex} or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC=@openai/codex@${REMOTE_PROVIDER_PACK_PINS.codex}`,
);
}
if (version !== REMOTE_PROVIDER_PACK_PINS.codex && !reportedCodexVersions.has(version)) {
reportedCodexVersions.add(version);
await input.onLog?.(
"stderr",
`[paperclip-runner] using compatible Codex ${version} (supported ${REMOTE_CODEX_SUPPORTED_RANGE}; install pin ${REMOTE_PROVIDER_PACK_PINS.codex})\n`,
);
}
};
const verifyRemoteProviderPack = async (packRoot: string) => {
if (!remoteTarget || !remoteCommandRunner || !expectedProviderPackManifest)
return;
const expected = Buffer.from(
canonicalJson(expectedProviderPackManifest),
"utf8",
).toString("base64");
const providerNodeCommand = posix.join(
packRoot,
expectedProviderPackManifest.payload.artifacts.nodeCommand.path,
);
const verifyScript = [
"const fs=require('node:fs')",
"const crypto=require('node:crypto')",
"const path=require('node:path')",
"const root=process.argv[1]",
"const expected=Buffer.from(process.argv[2],'base64').toString('utf8')",
"const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
"const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
"const manifest=JSON.parse(actual)",
"if(canonical(manifest)!==expected)throw new Error('manifest mismatch')",
"const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
"const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
"for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar','grokLauncher']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
"if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
"for(const candidate of Object.values({...manifest.payload.providers,...manifest.payload.candidateProviders})){if(tree(path.join(root,candidate.path))!==candidate.sha256)throw new Error('candidate asset tree digest mismatch')}",
"const version=process.versions.node.split('.').map(Number)",
`const minimums=[manifest.payload.pins.nodeMinimum,${JSON.stringify(REMOTE_PROVIDER_PACK_PINS.nodeMinimum)}]`,
"for(const value of minimums){const minimum=value.split('.').map(Number);if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')}",
"if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
"const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
"const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
"for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
].join(";");
const verified = await remoteCommandRunner.execute({
command: providerNodeCommand,
args: ["-e", verifyScript, packRoot, expected],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 30_000,
});
if (verified.exitCode !== 0 || verified.timedOut) {
throw new Error(
`runner_remote_provider_artifact_incompatible: provider pack verification failed (${verified.stderr.trim().slice(-1_024)})`,
);
}
const opencodeCommand = posix.join(
packRoot,
expectedProviderPackManifest.payload.artifacts.opencodeCommand.path,
);
const opencodeVersion = await remoteCommandRunner.execute({
command: opencodeCommand,
args: ["--version"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 30_000,
});
if (
opencodeVersion.exitCode !== 0 ||
opencodeVersion.timedOut ||
opencodeVersion.stdout.trim() !== expectedProviderPackManifest.payload.pins.opencode
) {
throw new Error(
"runner_remote_provider_artifact_incompatible: OpenCode version mismatch",
);
}
};
const discoverPreinstalledProviderPack = async () => {
if (!remoteTarget || !remoteCommandRunner) return null;
const result = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
'for candidate in /opt/paperclip-runner/provider-pack "$HOME/.local/share/paperclip-runner/provider-pack"; do if [ -f "$candidate/provider-pack.json" ]; then printf \'%s\\n\' "$candidate"; break; fi; done',
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
if (result.exitCode !== 0 || result.timedOut) return null;
return parseRemoteExecutableCandidate(result.stdout);
};
// Image policy: keep one latest stable CLI installation shared by native and
// local adapters. Preferred bin entries must point to that same installation;
// never bake an older global CLI alongside a private runner-only version.
const discoverPreinstalledExecutable = async (
name: "paperclip-runnerd" | "codex",
) => {
if (!remoteTarget || !remoteCommandRunner) return null;
return discoverRemoteExecutable(remoteCommandRunner, remoteTarget.remoteCwd, name);
};
const linkPreinstalledExecutable = async (
sourcePath: string,
targetPath: string,
) => {
if (!remoteTarget || !remoteCommandRunner) return;
const escapedSource = sourcePath.replaceAll("'", "'\\''");
const escapedTarget = targetPath.replaceAll("'", "'\\''");
const escapedDirectory = posix.dirname(targetPath).replaceAll("'", "'\\''");
const result = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
targetPath === remoteCodexBinary
? buildRemoteCodexLauncherCommand(sourcePath, targetPath)
: `umask 077; mkdir -p '${escapedDirectory}' && ` +
`ln -sfn '${escapedSource}' '${escapedTarget}'`,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error("runner_remote_preinstalled_link_failed");
}
};
const prepareRemoteRunner = async (
requiredMode: "dial_wss" | "listen_ws",
) => {
if (
!remoteTarget ||
!remoteCommandRunner ||
!remoteBinary ||
remotePrepared
)
return;
selectedRemoteMode = requiredMode;
if (input.restartRecovery?.kind === "reattach_remote_runner") {
await verifyRemoteRunner(requiredMode);
if (requiresRemoteProviderPack && stagedRemoteProviderPackRoot) {
await verifyRemoteProviderPack(stagedRemoteProviderPackRoot);
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
}
remotePrepared = true;
return;
}
// Compatibility metadata does not prove artifact identity. Reuse only the
// exact controller-owned bytes when the controller artifact is available.
const matchesControllerRunnerArtifact = async (executable: string): Promise<boolean> => {
const expected = createHash("sha256")
.update(readFileSync(controllerRunnerBinary))
.digest("hex");
try {
const probe = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
'test -x "$1" || exit 1; if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1"; else shasum -a 256 "$1"; fi',
"paperclip-runner-artifact",
executable,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
return probe.exitCode === 0 && !probe.timedOut &&
/^[a-f0-9]{64}\s/.test(probe.stdout) &&
probe.stdout.trim().split(/\s+/)[0] === expected;
} catch {
// An unavailable checksum uses the verified staging path.
return false;
}
};
let runnerArtifactPrepared = false;
if (
sandboxLeaseAcquisition?.outcome === "resumed" &&
existsSync(controllerRunnerBinary)
) {
runnerArtifactPrepared = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_retained",
() => matchesControllerRunnerArtifact(remoteBinary),
);
}
const explicitRemoteBinary = input.runnerRemoteBinaryPath?.trim() || null;
if (!runnerArtifactPrepared && mayUsePreinstalledRunnerArtifact(explicitRemoteBinary)) {
const preinstalledRunner = await measureNativeRunnerSpan(
input.trace,
"runner.artifact.discover",
() => discoverPreinstalledExecutable("paperclip-runnerd"),
);
if (preinstalledRunner) {
try {
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.verify_preinstalled",
() => verifyRemoteRunner(requiredMode, preinstalledRunner),
);
if (existsSync(controllerRunnerBinary) &&
!await matchesControllerRunnerArtifact(preinstalledRunner)) {
throw new Error("runner_remote_preinstalled_artifact_mismatch");
}
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.link",
() => linkPreinstalledExecutable(preinstalledRunner, remoteBinary),
);
runnerArtifactPrepared = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled runnerd from the sandbox image\n",
);
} catch {
runnerArtifactPrepared = false;
}
}
}
if (!runnerArtifactPrepared) {
// Upload the same artifact used for the controller identity. The server
// vendors the runner under vendor/paperclip-runner/bin, so the package
// development fallback cannot locate it in a deployed server.
const sourceBinary = controllerRunnerBinary;
if (!existsSync(sourceBinary)) {
throw new Error("runner_remote_artifact_unavailable");
}
if (!explicitRemoteBinary && !remotePiCompanion && !useBundledRemoteImageAssets) {
const platform = await remoteCommandRunner.execute({
command: "sh",
args: ["-c", "uname -s; uname -m"],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
const [remoteOs = "", remoteArch = ""] = platform.stdout
.trim()
.split(/\r?\n/);
const localOs =
process.platform === "darwin"
? "Darwin"
: process.platform === "linux"
? "Linux"
: process.platform;
const localArch =
process.arch === "x64"
? "x86_64"
: process.arch === "arm64"
? "aarch64"
: process.arch;
const archMatches =
remoteArch === localArch ||
(localArch === "aarch64" && remoteArch === "arm64");
if (
platform.exitCode !== 0 ||
platform.timedOut ||
remoteOs !== localOs ||
!archMatches
) {
throw new Error(
"runner_remote_artifact_platform_mismatch: configure PAPERCLIP_RUNNER_REMOTE_BINARY_PATH for the remote OS and architecture",
);
}
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: sourceBinary,
targetPath: remoteBinary,
mode: 0o700,
});
}
await measureNativeRunnerSpan(input.trace, "runner.artifact.verify", () =>
verifyRemoteRunner(requiredMode),
);
if (remoteCodexBinary && explicitRemoteCodex) {
if (!existsSync(explicitRemoteCodex)) {
throw new Error("runner_remote_codex_artifact_unavailable");
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: explicitRemoteCodex,
targetPath: remoteCodexBinary,
mode: 0o700,
});
await verifyRemoteCodex();
}
if (remoteCodexBinary && remoteCodexNpmSpec) {
let usedPreinstalledCodex = false;
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (preinstalledCodex) {
try {
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.link",
() =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
usedPreinstalledCodex = true;
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
} catch {
usedPreinstalledCodex = false;
}
}
if (!usedPreinstalledCodex) {
const installRoot = posix.join(
remoteRuntimeRoot!,
"harnesses",
"codex",
);
const installResult = await remoteCommandRunner.execute({
command: "npm",
args: [
"install",
"--prefix",
installRoot,
"--no-audit",
"--no-fund",
remoteCodexNpmSpec,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 180_000,
});
if (installResult.exitCode !== 0 || installResult.timedOut) {
throw new Error(`runner_remote_codex_install_failed exitCode=${installResult.exitCode} timedOut=${installResult.timedOut}: ${redactSensitiveText(installResult.stderr).slice(-2_000).trim()}`);
}
}
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
}
if (remoteCodexBinary && !explicitRemoteCodex && !remoteCodexNpmSpec) {
const preinstalledCodex = await measureNativeRunnerSpan(
input.trace,
"harness.artifact.discover",
() => discoverPreinstalledExecutable("codex"),
);
if (!preinstalledCodex) {
throw new Error(
"runner_remote_codex_artifact_unavailable: install codex in the sandbox image or configure PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC",
);
}
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify_preinstalled",
() => verifyRemoteCodex(preinstalledCodex),
);
await measureNativeRunnerSpan(input.trace, "harness.artifact.link", () =>
linkPreinstalledExecutable(preinstalledCodex, remoteCodexBinary),
);
await measureNativeRunnerSpan(
input.trace,
"harness.artifact.verify",
() => verifyRemoteCodex(),
);
await input.onLog?.(
"stderr",
"[paperclip-runner] using preinstalled Codex from the sandbox image\n",
);
}
if (
requiresRemoteProviderPack &&
expectedProviderPackManifest &&
stagedRemoteProviderPackRoot
) {
const packSource = remoteTarget.transport === "computer"
? await prepareComputerProviderPackCache({
cacheRoot: stagedRemoteProviderPackRoot,
sessionKey: nativeSessionKey(input.execution),
owner: remoteTarget.resourceAuthority,
runner: remoteCommandRunner,
verify: (root) => measureNativeRunnerSpan(input.trace, "provider_pack.verify", () => verifyRemoteProviderPack(root)),
stage: async (root, runner) => {
if (retainedComputerPack) throw new Error("runner_remote_provider_pack_retained_artifacts_unavailable");
if (!configuredProviderPackRoot) {
throw new Error("runner_remote_provider_artifact_incompatible: configure the build-owned provider pack for first use on this computer");
}
await input.onLog?.("stderr", "[paperclip-runner] Preparing the provider pack for first use.\n");
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner,
sourcePath: configuredProviderPackRoot,
targetPath: root,
mode: 0o700,
onProgress: async (completed, total) => {
await input.onLog?.("stderr", `[paperclip-runner] Uploading provider pack: ${Math.round(completed / 1024 / 1024)} / ${Math.round(total / 1024 / 1024)} MiB.\n`);
},
});
},
})
: await prepareVerifiedRemoteProviderPack({
verifyStaged: () => measureNativeRunnerSpan(
input.trace,
"provider_pack.verify",
() => verifyRemoteProviderPack(stagedRemoteProviderPackRoot),
),
usePreinstalled: async () => {
let preinstalledProviderPack = await discoverPreinstalledProviderPack();
if (preinstalledProviderPack) {
try {
await measureNativeRunnerSpan(
input.trace,
"provider_pack.verify_preinstalled",
() => verifyRemoteProviderPack(preinstalledProviderPack!),
);
const escapedSource = preinstalledProviderPack.replaceAll(
"'",
"'\\''",
);
const escapedTarget = stagedRemoteProviderPackRoot.replaceAll(
"'",
"'\\''",
);
const escapedParent = posix
.dirname(stagedRemoteProviderPackRoot)
.replaceAll("'", "'\\''");
const linked = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
`umask 077; mkdir -p '${escapedParent}' && rm -rf '${escapedTarget}' && ln -s '${escapedSource}' '${escapedTarget}'`,
],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
if (linked.exitCode !== 0 || linked.timedOut) {
throw new Error(
"runner_remote_provider_artifact_incompatible: preinstalled provider pack could not be linked",
);
}
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
await input.onLog?.(
"stderr",
"[paperclip-runner] using manifest-matched provider pack from the sandbox image\n",
);
} catch {
preinstalledProviderPack = null;
}
}
return preinstalledProviderPack !== null;
},
stageAndVerify: async () => {
if (!configuredProviderPackRoot) {
throw new Error("runner_remote_provider_artifact_incompatible: install the matching Paperclip package and Daytona image; the image provider pack did not match the bundled release identity");
}
const escapedPackRoot = stagedRemoteProviderPackRoot.replaceAll(
"'",
"'\\''",
);
const cleared = await remoteCommandRunner.execute({
command: "sh",
args: ["-c", `rm -rf '${escapedPackRoot}'`],
cwd: remoteTarget.remoteCwd,
bypassSession: true,
timeoutMs: 10_000,
});
if (cleared.exitCode !== 0 || cleared.timedOut) {
throw new Error(
"runner_remote_provider_artifact_incompatible: stale provider pack could not be replaced",
);
}
await input.onLog?.("stderr", "[paperclip-runner] Preparing the provider pack for first use.\n");
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: configuredProviderPackRoot,
targetPath: stagedRemoteProviderPackRoot,
mode: 0o700,
onProgress: async (completed, total) => {
await input.onLog?.("stderr", `[paperclip-runner] Uploading provider pack: ${Math.round(completed / 1024 / 1024)} / ${Math.round(total / 1024 / 1024)} MiB.\n`);
},
});
await measureNativeRunnerSpan(input.trace, "provider_pack.verify", () =>
verifyRemoteProviderPack(stagedRemoteProviderPackRoot),
);
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
},
});
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
if (packSource === "staged" || packSource === "reused") {
await input.onLog?.(
"stderr",
"[paperclip-runner] reusing manifest-matched provider pack from the workspace\n",
);
}
}
remotePrepared = true;
};
const inspectRemoteHarnessState = async (): Promise<{
complete: boolean;
runnerState: Record<string, unknown> | null;
providerSessionIdentity: Record<string, unknown> | null;
incompleteReason: "unavailable" | "not_suspended" | null;
}> => {
if (!remoteCommandRunner || !remoteStateDirectory) {
return {
complete: false,
runnerState: null,
providerSessionIdentity: null,
incompleteReason: "unavailable",
};
}
const requirements = persistenceProfile.directories.flatMap((directory) => {
const path = remotePersistencePath(directory);
if (!path) return [];
const escaped = path.replaceAll("'", "'\\''");
return directory.name === "runner"
? [`test -f '${escaped}/runner-state.json'`]
: [`test -d '${escaped}'`];
});
const escapedRunnerState = posix
.join(remoteStateDirectory, "runner-state.json")
.replaceAll("'", "'\\''");
const escapedProviderState = posix
.join(remoteStateDirectory, runnerProviderStateFilename(input.execution))
.replaceAll("'", "'\\''");
const inspected = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
`${requirements.join(" && ")} && test -f '${escapedProviderState}' && base64 < '${escapedRunnerState}'`,
],
bypassSession: true,
timeoutMs: 10_000,
});
if (inspected.exitCode !== 0 || inspected.timedOut) {
return {
complete: false,
runnerState: null,
providerSessionIdentity: null,
incompleteReason: "unavailable",
};
}
let runnerState: Record<string, unknown>;
try {
runnerState = record(
JSON.parse(
Buffer.from(inspected.stdout.replace(/\s+/g, ""), "base64").toString(
"utf8",
),
),
);
} catch {
throw new Error("runner_harness_state_mismatch: runner_state_invalid_json");
}
if (
runnerState.runnerInstanceId !== input.runnerInstanceId ||
runnerState.normalizedSessionId !== nativeSessionKey(input.execution)
) {
throw new Error("runner_harness_state_mismatch: runner_identity");
}
if (runnerState.lifecycle !== "suspended") {
return {
complete: false,
runnerState: null,
providerSessionIdentity: null,
incompleteReason: "not_suspended",
};
}
let providerState: Record<string, unknown>;
try {
providerState = await readRemoteRunnerProviderState({
runner: remoteCommandRunner,
stateDirectory: remoteStateDirectory,
execution: input.execution,
});
} catch {
throw new Error("runner_harness_state_mismatch: provider_state_unreadable");
}
const providerSessionIdentity =
providerSessionIdentityFromDurableProviderState({
execution: input.execution,
providerState,
});
if (!providerSessionIdentityIsPresent(providerSessionIdentity)) {
throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)})`);
}
const previousManifest = compatibleNativeHarnessBackupManifests({
root,
execution: input.execution,
runnerInstanceId: input.runnerInstanceId,
})[0]?.manifest;
if (
previousManifest &&
!providerSessionIdentityTransitionIsAllowed({
execution: input.execution,
previous: previousManifest.providerSessionIdentity,
current: providerSessionIdentity,
})
) {
throw new Error("runner_harness_state_mismatch: provider_identity_changed");
}
return {
complete: true,
runnerState,
providerSessionIdentity,
incompleteReason: null,
};
};
const claimUntouchedSessionInResumedLease = async (): Promise<boolean> => {
if (!remoteCommandRunner || !remoteRuntimeRoot || !remoteSessionRoot) return false;
const identity = readRunnerdDurableIdentity(root);
if (!durableIdentityMatchesExecution(identity, input.execution) ||
identity?.runnerInstanceId !== effectiveRunnerInstanceId ||
identity?.environmentLeaseId !== effectiveEnvironmentLeaseId ||
!runnerdStateProvesIncompleteBootstrap(root)) return false;
// A reusable workspace may have failed before any harness was created.
// Claim this exact new session atomically under readable real directories.
// Missing files inside an existing session never authorize a fresh start.
const probe = await remoteCommandRunner.execute({
command: "sh",
args: ["-c",
'set -eu; umask 077; test -d "$1" && test ! -L "$1" && test -r "$1" && test -x "$1" || exit 1; if test ! -e "$2" && test ! -L "$2"; then mkdir -- "$2"; fi; test -d "$2" && test ! -L "$2" && test -r "$2" && test -x "$2" || exit 1; mkdir -- "$3"',
"paperclip-runner-claim-unstarted-session", remoteRuntimeRoot,
posix.dirname(remoteSessionRoot), remoteSessionRoot],
bypassSession: true,
timeoutMs: 10_000,
});
return probe.exitCode === 0 && !probe.timedOut;
};
const recordInPlaceHarnessReuse = async (
providerSessionIdentity: Record<string, unknown>,
startedAtMs = Date.now(),
) => {
const now = Date.now();
const attributes = {
provider: input.execution.provider.kind,
harness: input.execution.session.driverKind,
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
stateSource: "sandbox_filesystem",
bytesTransferred: 0,
};
await input.trace?.record({
name: "harness_state.reuse",
startedAtMs,
endedAtMs: now,
attributes,
});
await input.trace?.record({
name: "provider.session.resume",
startedAtMs: now,
endedAtMs: now,
attributes: {
provider: input.execution.provider.kind,
harness: input.execution.session.driverKind,
identityPresent: providerSessionIdentityIsPresent(
providerSessionIdentity,
),
},
});
};
const recordHarnessBackupStampForCurrentLease = async (
backup: VerifiedHarnessBackup,
) => {
if (remoteTarget?.transport !== "sandbox" || !remoteTarget.leaseId) {
return;
}
const leaseRow = await input.db
.select({
metadata: environmentLeases.metadata,
providerLeaseId: environmentLeases.providerLeaseId,
})
.from(environmentLeases)
.where(eq(environmentLeases.id, remoteTarget.leaseId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!leaseRow?.providerLeaseId) {
throw new Error("runner_harness_backup_lease_missing");
}
if (
sandboxLeaseAcquisition?.providerLeaseId &&
sandboxLeaseAcquisition.providerLeaseId !== leaseRow.providerLeaseId
) {
throw new Error("runner_harness_backup_lease_mismatch");
}
const stamp = createNativeHarnessBackupStamp({
manifestPath: resolve(backup.root, "manifest.json"),
sessionScopeId,
authorizedProviderLeaseId: leaseRow.providerLeaseId,
normalizedSessionId: backup.manifest.normalizedSessionId,
runnerInstanceId: backup.manifest.runnerInstanceId,
completedAt: backup.manifest.completedAt,
});
const updated = await input.db
.update(environmentLeases)
.set({
metadata: {
...(leaseRow.metadata ?? {}),
nativeHarnessBackup: stamp,
},
updatedAt: new Date(),
})
.where(eq(environmentLeases.id, remoteTarget.leaseId))
.returning({ id: environmentLeases.id })
.then((rows) => rows[0] ?? null);
if (!updated) throw new Error("runner_harness_backup_lease_missing");
};
const restoreVerifiedHarnessBackup = async () => {
if (!remoteTarget || !remoteCommandRunner) {
throw new Error("runner_harness_backup_unavailable");
}
const backup = verifyNativeHarnessBackup({
root,
execution: input.execution,
runnerInstanceId: input.runnerInstanceId,
});
if (!backup) throw new Error("runner_harness_backup_unavailable");
await measureNativeRunnerSpan(
input.trace,
"harness_state.failover_restore",
async () => {
for (const directory of persistenceProfile.directories) {
const targetPath = remotePersistencePath(directory);
if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable");
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: resolve(backup.root, directory.name),
targetPath,
mode: 0o700,
});
}
},
{
attributes: {
provider: input.execution.provider.kind,
harness: input.execution.session.driverKind,
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
stateSource: "verified_failover_backup",
bytesTransferred: backup.bytes,
},
},
);
const restored = await inspectRemoteHarnessState();
if (
!restored.complete ||
!restored.runnerState ||
!restored.providerSessionIdentity ||
canonicalJson(restored.providerSessionIdentity) !==
canonicalJson(backup.manifest.providerSessionIdentity)
) {
throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed");
}
// A deliberately non-reusable environment receives a fresh provider lease
// for every turn. Stamp that new lease as soon as the verified host backup
// has been restored so a later provider/bootstrap failure can still clean
// the ephemeral sandbox up without discarding the only durable copy.
await recordHarnessBackupStampForCurrentLease(backup);
return backup;
};
const materializeRemoteHarnessLaunchState = async () => {
if (!remoteTarget || !remoteCommandRunner) return;
for (const directory of persistenceProfile.directories) {
if (directory.location !== "filesystem") continue;
const targetPath = remotePersistencePath(directory);
if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable");
const escapedTarget = targetPath.replaceAll("'", "'\\''");
const created = await remoteCommandRunner.execute({
command: "sh",
args: ["-c", `umask 077; install -d -m 0700 '${escapedTarget}'`],
bypassSession: true,
timeoutMs: 10_000,
});
if (created.exitCode !== 0 || created.timedOut) {
throw new Error(`runner_remote_directory_staging_failed: exit=${created.exitCode} timedOut=${created.timedOut}${created.stderr.trim() ? ` ${redactSensitiveText(created.stderr).trim().slice(-512)}` : ""}`);
}
// Codex launch credentials are intentionally excluded from failover
// backups. Re-materialize only those launch-time files into a fresh or
// replacement sandbox after the durable history has been restored.
if (directory.name !== "codex-home") continue;
const localDirectory = resolve(root, directory.name);
for (const name of ["auth.json", "config.toml"] as const) {
const sourcePath = resolve(localDirectory, name);
if (!existsSync(sourcePath)) continue;
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath,
targetPath: posix.join(targetPath, name),
mode: 0o600,
});
}
}
};
const ensureRemoteRunner = async (stageLaunchAssets = true) => {
await measureNativeRunnerSpan(
input.trace,
"stage.sync",
async () => {
if (!selectedRemoteMode) {
throw new Error("runner_remote_transport_mode_unresolved");
}
try {
await verifyRemoteRunner(selectedRemoteMode);
await verifyRemoteCodex();
if (requiresRemoteProviderPack) {
if (!activeRemoteProviderPackRoot) {
throw new Error(
"runner_remote_provider_artifact_incompatible: provider pack was not prepared",
);
}
await verifyRemoteProviderPack(activeRemoteProviderPackRoot);
}
} catch {
remotePrepared = false;
await prepareRemoteRunner(selectedRemoteMode);
}
if (!remoteHarnessStatePrepared) {
await measureNativeRunnerSpan(input.trace, "stage.asset.home", () =>
measureNativeRunnerSpan(
input.trace,
"session.checkpoint.restore",
async () => {
if (
remoteTarget?.transport === "sandbox" &&
remoteCommandRunner
) {
const acquisitionRecordedAtMs = Date.now();
const backupAvailable = harnessBackupCandidates(root).some(
(candidate) =>
existsSync(resolve(candidate, "manifest.json")),
);
const restoreIntoCreatedSandbox =
shouldRestoreNativeHarnessBackupIntoSandbox({
acquisitionOutcome:
sandboxLeaseAcquisition?.outcome ?? null,
reusableLeaseConfigured:
remoteTarget.reusableLeaseConfigured,
backupAvailable,
});
await input.trace?.record({
name: "sandbox.lease.acquisition",
startedAtMs: acquisitionRecordedAtMs,
endedAtMs: acquisitionRecordedAtMs,
attributes: {
provider: remoteTarget.providerKey ?? "sandbox",
harness: input.execution.session.driverKind,
lifecycleMode:
input.execution.session.lifecyclePolicy.mode,
outcome: sandboxLeaseAcquisition?.outcome ?? "unknown",
stateSource:
sandboxLeaseAcquisition?.outcome === "replacement" ||
restoreIntoCreatedSandbox
? "verified_failover_backup"
: sandboxLeaseAcquisition?.outcome === "resumed"
? "sandbox_filesystem"
: "new_sandbox",
bytesTransferred: 0,
},
});
if (sandboxLeaseAcquisition?.outcome === "resumed") {
const reuseStartedAtMs = Date.now();
const state = await measureNativeRunnerSpan(
input.trace,
"sandbox.lease.resume",
inspectRemoteHarnessState,
{
attributes: {
provider: remoteTarget.providerKey ?? "sandbox",
harness: input.execution.session.driverKind,
lifecycleMode:
input.execution.session.lifecyclePolicy.mode,
outcome: "resumed",
},
},
);
if (
!state.complete ||
!state.runnerState ||
!state.providerSessionIdentity
) {
if (state.incompleteReason !== "unavailable" || backupAvailable ||
!(await claimUntouchedSessionInResumedLease())) {
throw new Error(`runner_harness_state_mismatch: resumed_${state.incompleteReason}`);
}
} else {
await recordInPlaceHarnessReuse(
state.providerSessionIdentity,
reuseStartedAtMs,
);
}
} else if (
sandboxLeaseAcquisition?.outcome === "replacement"
) {
await measureNativeRunnerSpan(
input.trace,
"sandbox.lease.replacement",
restoreVerifiedHarnessBackup,
{
attributes: {
provider: remoteTarget.providerKey ?? "sandbox",
harness: input.execution.session.driverKind,
lifecycleMode:
input.execution.session.lifecyclePolicy.mode,
outcome: "replacement",
reason: sandboxLeaseAcquisition.reason ?? "unknown",
},
},
);
} else if (restoreIntoCreatedSandbox) {
await measureNativeRunnerSpan(
input.trace,
"sandbox.lease.replacement",
restoreVerifiedHarnessBackup,
{
attributes: {
provider: remoteTarget.providerKey ?? "sandbox",
harness: input.execution.session.driverKind,
lifecycleMode:
input.execution.session.lifecyclePolicy.mode,
outcome: "created",
reason: "reuse_disabled",
},
},
);
} else {
const reuseStartedAtMs = Date.now();
const state = await inspectRemoteHarnessState();
if (
state.complete &&
state.runnerState &&
state.providerSessionIdentity
) {
// Re-entry while this newly-created lease is already running (for
// example a transport reconnect) still uses the in-place state.
await recordInPlaceHarnessReuse(
state.providerSessionIdentity,
reuseStartedAtMs,
);
} else if (backupAvailable) {
// A continuation that has a durable backup but no recorded reusable
// lease was not provider-confirmed lost. Never silently create a new
// provider session from that ambiguous state.
throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease");
}
}
} else if (remoteTarget && remoteCommandRunner) {
// Local and generic SSH execution retain their existing checkpoint
// behavior. The manifest-only failover gate applies to managed sandbox
// replacement, where provider lease provenance is available.
for (const directory of persistenceProfile.directories) {
const localDirectory = resolve(root, directory.name);
const remoteDirectory = remotePersistencePath(directory);
if (
remoteDirectory &&
existsSync(localDirectory) &&
!(await remoteRunnerPathExists({
runner: remoteCommandRunner,
path:
directory.name === "runner"
? posix.join(remoteDirectory, "runner-state.json")
: remoteDirectory,
kind:
directory.name === "runner" ? "file" : "directory",
}))
) {
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: localDirectory,
targetPath: remoteDirectory,
mode: 0o700,
excludeEntries: directory.excludeEntries,
});
}
}
}
},
{
attributes: {
mode: remoteTarget?.transport ?? "local",
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
},
},
),
);
remoteHarnessStatePrepared = true;
}
// Authority rotation needs durable history, before the transport writes
// this turn's launch files. Stage assets only at the actual launch so we
// neither upload stale context nor transfer every bundle twice on resume.
if (!stageLaunchAssets) return;
// Resume can prepare/rotate durable state before the transport creates
// this invocation's isolated Codex auth/config. The later launch must
// still stage those fresh files even when history was already restored.
// Launch material is never recovered from a failover backup.
await materializeRemoteHarnessLaunchState();
if (
remoteTarget &&
remoteCommandRunner &&
remoteRunnerFilesystemRoot &&
sourceRuntimeContext &&
remoteRuntimeContext
) {
await measureNativeRunnerSpan(
input.trace,
"stage.asset.runtime_context",
async () => {
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: sourceRuntimeContext.instructions.bundle.rootPath,
targetPath: remoteRuntimeContext.instructions.bundle.rootPath,
mode: 0o555,
});
for (
let index = 0;
index < sourceRuntimeContext.skills.length;
index += 1
) {
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath:
sourceRuntimeContext.skills[index]!.bundle.rootPath,
targetPath:
remoteRuntimeContext.skills[index]!.bundle.rootPath,
mode: 0o555,
});
}
await stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: resolve(root, "runtime-context.json"),
targetPath: posix.join(
remoteRunnerFilesystemRoot,
"runtime-context.json",
),
mode: 0o600,
});
},
);
}
if (remoteTarget && remoteCommandRunner && remoteCaBundleMapping) {
const caBundleMapping = remoteCaBundleMapping;
await measureNativeRunnerSpan(
input.trace,
"stage.asset.ca_bundle",
() =>
stageRemoteRunnerFile({
target: remoteTarget,
runner: remoteCommandRunner,
sourcePath: caBundleMapping.sourcePath,
targetPath: caBundleMapping.targetPath,
mode: 0o600,
}),
);
}
},
{
attributes: {
target: remoteTarget?.transport ?? "local",
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
},
},
);
};
const checkpointRemoteRunner = async (
settlement: "settled" | "unsettled",
) => {
if (
!remoteCommandRunner ||
!remoteStateDirectory ||
!remoteRunnerFilesystemRoot
)
return;
// Transport release also runs when provider bootstrap failed. In that case
// runnerd has no durable provider identity (and may not have created the
// provider persistence directory at all), so attempting a failover backup
// would replace the original provider error with
// `runner_harness_state_mismatch`. Only checkpoint a harness that runnerd
// has proved complete. A malformed or identity-conflicting state still
// throws from inspectRemoteHarnessState and therefore fails closed.
let checkpointable = await inspectRemoteHarnessState();
// The remote runner writes its suspended lifecycle and provider state
// before the outer process-owner RPC necessarily observes completion.
// Allow a very small bounded visibility window without ever accepting an
// active, incomplete, malformed, or identity-conflicting checkpoint.
for (let attempt = 1; !checkpointable.complete && attempt < 3; attempt++) {
await new Promise<void>((resolve) => setTimeout(resolve, 100));
checkpointable = await inspectRemoteHarnessState();
}
if (!checkpointable.complete) {
const incompleteFailure = remoteCheckpointIncompleteFailure(
settlement,
checkpointable.incompleteReason,
);
await input.onLog?.(
"stderr",
`[paperclip-runner] remote checkpoint ${incompleteFailure ? "failed" : "skipped"}: exact suspended harness state unavailable (process=${settlement} reason=${checkpointable.incompleteReason})\n`,
);
if (incompleteFailure) throw incompleteFailure;
return;
}
// The computer ledger and exact remote placement are the recovery authority.
// Its durable provider state stays on the persistent machine; importing the
// sandbox archive would create a second authority and delay bounded shutdown.
if (remoteTarget?.transport === "computer") return;
const backupSpanAttributes = {
provider: input.execution.provider.kind,
harness: input.execution.session.driverKind,
lifecycleMode: input.execution.session.lifecyclePolicy.mode,
stateSource: "sandbox_filesystem",
bytesTransferred: 0,
};
try {
await measureNativeRunnerSpan(
input.trace,
"session.checkpoint.persist",
() =>
measureNativeRunnerSpan(
input.trace,
"harness_state.backup.persist",
async () => {
const verified = await inspectRemoteHarnessState();
if (
!verified.complete ||
!verified.runnerState ||
!verified.providerSessionIdentity
) {
throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete");
}
const providerSessionIdentity = verified.providerSessionIdentity;
const backupRoot = harnessBackupRoot(root);
mkdirSync(backupRoot, { recursive: true, mode: 0o700 });
const pendingRoot = resolve(
backupRoot,
`.pending-${randomUUID()}`,
);
mkdirSync(pendingRoot, { recursive: true, mode: 0o700 });
try {
for (const directory of persistenceProfile.directories) {
const sourcePath = remotePersistencePath(directory);
if (!sourcePath)
throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable");
const targetPath = resolve(pendingRoot, directory.name);
await syncRemoteRunnerDirectoryOut({
runner: remoteCommandRunner,
sourcePath,
targetPath,
mode: 0o700,
excludeEntries: directory.excludeEntries,
});
if (!existsSync(targetPath)) {
throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing");
}
}
const manifest = buildNativeHarnessBackupManifest({
backupRoot: pendingRoot,
execution: input.execution,
runnerInstanceId: input.runnerInstanceId,
providerSessionIdentity,
sourceProviderLeaseId:
sandboxLeaseAcquisition?.providerLeaseId ??
remoteTarget?.leaseId ??
input.durableEnvironmentLeaseId ??
"unknown",
});
backupSpanAttributes.bytesTransferred =
manifest.directories.reduce(
(total, directory) => total + directory.bytes,
0,
);
const temporaryManifest = resolve(
pendingRoot,
"manifest.json.tmp",
);
const manifestPath = resolve(pendingRoot, "manifest.json");
writeFileSync(temporaryManifest, JSON.stringify(manifest), {
encoding: "utf8",
mode: 0o600,
});
renameSync(temporaryManifest, manifestPath);
const currentRoot = resolve(backupRoot, "current");
const previousRoot = resolve(backupRoot, "previous");
removeNativeHarnessBackup(previousRoot);
let movedCurrent = false;
if (existsSync(currentRoot)) {
renameSync(currentRoot, previousRoot);
movedCurrent = true;
}
try {
renameSync(pendingRoot, currentRoot);
if (
remoteTarget?.transport === "sandbox" &&
remoteTarget.leaseId
) {
await recordHarnessBackupStampForCurrentLease({
root: currentRoot,
manifest,
bytes: backupSpanAttributes.bytesTransferred,
});
}
} catch (error) {
if (existsSync(currentRoot)) {
removeNativeHarnessBackup(currentRoot);
}
if (
movedCurrent &&
existsSync(previousRoot) &&
!existsSync(currentRoot)
) {
renameSync(previousRoot, currentRoot);
}
throw error;
}
removeNativeHarnessBackup(previousRoot);
} finally {
removeNativeHarnessBackup(pendingRoot);
}
},
{
attributes: backupSpanAttributes,
},
),
{ parentName: "task.settle" },
);
} catch (error) {
const detail = redactSensitiveText(
error instanceof Error ? error.message : String(error),
)
.replace(/[\r\n]+/g, " ")
.slice(0, 512);
await input.onLog?.(
"stderr",
`[paperclip-runner] remote checkpoint failed: ${detail || "unknown failure"}\n`,
);
throw error;
}
};
const prepareExternalRunnerState =
remoteTarget && remoteCommandRunner
? async () => {
selectedRemoteMode ??= resolveRemoteRunnerTransportMode({
target: remoteTarget,
runnerIngressAuthorized: input.runnerIngressAuthorized === true,
});
await ensureRemoteRunner(false);
if (remoteTarget.transport === "computer" && remoteStateDirectory) {
await sealRetiredComputerRunnerState({ db: input.db, execution: input.execution, root,
target: remoteTarget, runner: remoteCommandRunner, stateDirectory: remoteStateDirectory });
}
}
: undefined;
const archiveExternalRunnerState =
remoteCommandRunner && remoteStateDirectory && remoteSessionRoot
? async (archive: { archiveKey: string }) => {
if (!/^[0-9a-f]{24}$/.test(archive.archiveKey)) {
throw new Error("runner_remote_authority_archive_invalid");
}
const sourcePath = posix.join(
remoteStateDirectory,
"runner-state.json",
);
const archiveDirectory = posix.join(
remoteSessionRoot,
"authority-epochs",
`epoch-${archive.archiveKey}`,
);
const archivedStatePath = posix.join(
archiveDirectory,
"runner-state.json",
);
const result = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
'set -eu; if test -L "$2" || { test -e "$2" && test ! -d "$2"; }; then exit 1; fi; if test -f "$1" && test ! -L "$1" && test ! -e "$3" && test ! -L "$3"; then umask 077; install -d -m 0700 "$2"; mv -- "$1" "$3"; elif test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"; then :; else exit 1; fi; base64 < "$3"',
"paperclip-runner-authority-archive",
sourcePath,
archiveDirectory,
archivedStatePath,
],
bypassSession: true,
timeoutMs: 10_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error("runner_remote_authority_archive_failed");
}
try {
return record(
JSON.parse(
Buffer.from(
result.stdout.replace(/\s+/g, ""),
"base64",
).toString("utf8"),
),
);
} catch {
throw new Error("runner_remote_authority_archive_failed");
}
}
: undefined;
const remoteProcessLauncher =
remoteTarget && remoteCommandRunner && remoteBinary && remoteStateDirectory
? createRemoteRunnerProcessLauncher({
target: remoteTarget,
runner: remoteCommandRunner,
remoteBinary,
processIdentityPath: posix.join(
remoteStateDirectory,
"runner-process.identity",
),
stateDirectory: remoteStateDirectory,
diagnosticsDirectory: posix.join(remoteSessionRoot!, "diagnostics"),
runnerInstanceId: input.runnerInstanceId,
ensureArtifact: ensureRemoteRunner,
onSpawn: input.onSpawn,
onLog: input.onLog,
trace: input.trace,
onRunnerProcessSpawned: () => resolveRemoteRunnerProcessSpawned?.(),
onLaunchAttempt: input.onRemoteLaunchAttempt,
})
: undefined;
const runnerExecution: NativeExecutionInput = remoteTarget
? {
...input.execution,
workspace: {
...input.execution.workspace,
cwd: remoteTarget.remoteCwd,
},
}
: input.execution;
const isGrok = input.execution.provider.kind === "acpx" && input.execution.provider.agent === "grok";
let effectiveRunnerEnvironmentBase: NodeJS.ProcessEnv = {
...(isGrok
? input.runnerEnvironment ?? {}
: resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment)),
};
const grokCredential = isGrok ? await prepareGrokRunnerCredentials({
companyId: input.execution.binding.companyId, environment: effectiveRunnerEnvironmentBase, remote: Boolean(remoteTarget),
managedHome: input.managedAiCredentialHome,
}) : null;
if (grokCredential) effectiveRunnerEnvironmentBase = grokCredential.environment;
if (relayAssignedMcp) {
// The server-held tool authority owns this credential. Do not deliver a
// duplicate HTTP MCP server or its bearer token to the remote provider.
delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_NAME;
delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_URL;
delete effectiveRunnerEnvironmentBase.PAPERCLIP_NATIVE_MCP_TOKEN;
}
// This authority bit is derived only from the selected execution target.
// Never let an agent, environment binding, or host variable disable the
// Codex sandbox for a local runner by supplying the same key.
delete effectiveRunnerEnvironmentBase.PAPERCLIP_RUNNER_EXTERNAL_SANDBOX;
const effectiveRunnerEnvironment: NodeJS.ProcessEnv = remoteRuntimeRoot
? {
...effectiveRunnerEnvironmentBase,
// The provider home is runner-owned state, not the execution workspace.
// Codex's permission profile explicitly denies HOME and CODEX_HOME. If
// either points at remoteCwd, that deny rule shadows the workspace write
// grant and the provider cannot initialize its shell sandbox or edit.
HOME: posix.join(remoteRunnerFilesystemRoot!, "codex-home"),
CODEX_HOME: posix.join(remoteRunnerFilesystemRoot!, "codex-home"),
PAPERCLIP_WORKSPACE_CWD: remoteTarget!.remoteCwd,
...(adapterExecutionTargetIsCommandBacked(remoteTarget)
? { PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1" }
: {}),
}
: {
...effectiveRunnerEnvironmentBase,
PAPERCLIP_WORKSPACE_CWD: input.execution.workspace.cwd,
};
const archiveContinuityState = async () => {
if (hasRetainedWarmTransitionEvidence(root)) {
throw new Error("native_runner_warm_transition_recovery_unproven");
}
const archiveToken = `${Date.now()}-${randomUUID()}`;
const archiveRoot = resolve(root, "continuity-breaks", archiveToken);
mkdirSync(archiveRoot, { recursive: true, mode: 0o700 });
for (const name of [
"control-plane",
"runner",
"codex-home",
"opencode",
"acpx",
// Backups belong to the retired provider session. Leaving them active
// makes the fresh replacement look like ambiguous lost harness state.
// Keep their evidence inside the same continuity-break archive.
"failover-backups",
]) {
const source = resolve(root, name);
if (existsSync(source)) renameSync(source, resolve(archiveRoot, name));
}
if (remoteCommandRunner && remoteSessionRoot) {
const escapedSource = remoteSessionRoot.replaceAll("'", "'\\''");
const archivedRemote = `${remoteSessionRoot}.continuity-break-${archiveToken}`;
const escapedArchive = archivedRemote.replaceAll("'", "'\\''");
const result = await remoteCommandRunner.execute({
command: "sh",
args: [
"-c",
`if test -d '${escapedSource}'; then mv '${escapedSource}' '${escapedArchive}'; fi`,
],
bypassSession: true,
timeoutMs: 30_000,
});
if (result.exitCode !== 0 || result.timedOut) {
throw new Error("runner_continuity_break_archive_failed");
}
}
remotePrepared = false;
};
const localRecoveryProcess =
input.restartRecovery?.kind === "reattach_existing_runner"
? input.restartRecovery.process
: null;
const adoptedProcess =
input.restartRecovery?.kind === "reattach_remote_runner"
? await verifyRemoteRunnerReattachment({
claim: input.restartRecovery,
target,
identity: durableIdentity ?? {},
runId: input.execution.binding.runId,
normalizedSessionId: nativeSessionKey(input.execution),
})
: target.kind === "local" && localRecoveryProcess
? {
...localRecoveryProcess,
isAlive: () => verifiedRecoveryProcessIsAlive(localRecoveryProcess),
signal: (signal: NodeJS.Signals) =>
signalVerifiedRecoveryProcess(localRecoveryProcess, signal),
}
: undefined;
if (adoptedProcess && remoteTarget) {
const markSpawned = resolveRemoteRunnerProcessSpawned as (() => void) | null;
markSpawned?.();
}
const executeCurrentToolAuthority = (
call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
return current.execute(call);
};
const backend = createNativeSessionBackend(runnerExecution, {
runnerInstanceId:
retainedTransition?.runnerInstanceId ?? input.runnerInstanceId,
environment: effectiveRunnerEnvironment,
workingDirectoryAuthority: remoteTarget
? "remote_runner"
: "local_filesystem",
onSpawn: input.onSpawn,
dynamicTools,
completionFeedback: async (result) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
await current.definitions(); // Reject a revoked run authority before reading task state.
return nativeCompletionFeedback(input.db, current.runId, result);
},
dynamicToolHandler: executeCurrentToolAuthority,
acpxDynamicToolHandler: executeCurrentToolAuthority,
opencodeRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"opencode",
),
acpxRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"acpx",
),
codexTransportFactory: (recoveryContext) =>
createRunnerdCodexTransport({
onSpawn: input.onSpawn,
provider:
input.execution.provider.kind === "codex"
? "codex"
: input.execution.provider.kind === "opencode"
? "opencode"
: input.execution.provider.kind === "claude_managed"
? "claude_managed"
: input.execution.provider.kind === "aws_agentcore"
? "aws_agentcore"
: input.execution.provider.kind === "acpx"
? "acpx"
: undefined,
...(input.execution.provider.kind === "acpx"
? {
acpxAgent: input.execution.provider.agent,
// Read only the server operator environment, never agent/runtime env.
acpxCandidateProfile: input.execution.provider.agent === "pi"
? undefined
: resolveAcpxQualification(input.execution.provider, process.env),
acpxPermissionMode: input.execution.provider.permissionMode,
acpxMode: input.execution.provider.mode,
piThinkingLevel: input.execution.provider.piThinkingLevel,
acpxPermissionModePinned:
input.execution.schema === "paperclip.native-execution-input.v4" ||
input.execution.schema === "paperclip.native-execution-input.v5",
acpxRuntimeDirectory: remoteRunnerFilesystemRoot
? posix.join(remoteRunnerFilesystemRoot, "acpx")
: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
"paperclip-runner",
"acpx",
),
}
: {}),
...(input.execution.provider.kind === "opencode"
? {
opencodePermissionMode: input.execution.provider.permissionMode,
}
: {}),
...(input.execution.provider.kind === "claude_managed"
? {
managedProfile: {
...input.execution.provider.managedProfile,
maxSessionListCostUsd:
input.execution.provider.maxSessionListCostUsd,
model: input.execution.provider.model,
},
}
: {}),
...(input.execution.provider.kind === "aws_agentcore"
? {
agentCoreProfile: {
...input.execution.provider.agentCoreProfile,
maxEstimatedSessionCostUsd:
input.execution.provider.maxEstimatedSessionCostUsd,
maxIterations:
input.execution.provider.invocationLimits.maxIterations,
maxOutputTokens:
input.execution.provider.invocationLimits.maxOutputTokens,
timeoutSeconds:
input.execution.provider.invocationLimits.timeoutSeconds,
model: input.execution.provider.model,
},
}
: {}),
...(expectedProviderPackManifest && stagedRemoteProviderPackRoot
? {
providerNodeCommand: posix.join(
stagedRemoteProviderPackRoot,
expectedProviderPackManifest.payload.artifacts.nodeCommand.path,
),
providerNodeCommandSha256:
expectedProviderPackManifest.payload.artifacts.nodeCommand
.sha256,
providerPackAuthorityDigest: expectedProviderPackManifest.digest,
opencodeCommand: posix.join(
stagedRemoteProviderPackRoot,
expectedProviderPackManifest.payload.artifacts
.opencodeExecutable.path,
),
opencodeCommandSha256:
expectedProviderPackManifest.payload.artifacts
.opencodeExecutable.sha256,
opencodeProxyPath: posix.join(
stagedRemoteProviderPackRoot,
expectedProviderPackManifest.payload.artifacts.opencodeProxy
.path,
),
opencodeProxySha256:
expectedProviderPackManifest.payload.artifacts.opencodeProxy
.sha256,
acpxSidecarPath: posix.join(
stagedRemoteProviderPackRoot,
expectedProviderPackManifest.payload.artifacts.acpxSidecar.path,
),
acpxSidecarSha256:
expectedProviderPackManifest.payload.artifacts.acpxSidecar
.sha256,
}
: {}),
stateDirectory: root,
runnerStateDirectory: remoteStateDirectory,
readRunnerState:
remoteStateDirectory && remoteCommandRunner
? () =>
readRemoteRunnerState({
runner: remoteCommandRunner,
stateDirectory: remoteStateDirectory,
})
: undefined,
prepareExternalRunnerState,
archiveExternalRunnerState,
runnerBinary: controllerRunnerBinary,
codexCommand: remoteCodexBinary ?? undefined,
sourceCodexHome: remoteTarget
? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, input.runnerEnvironment))
: undefined,
runnerProcessLauncher: remoteProcessLauncher,
runnerReconnectGraceMs: remoteTarget ? 120_000 : undefined,
adoptExistingRunner: adoptedProcess,
environment: effectiveRunnerEnvironment,
onDiagnostic: (message) => {
void input.onLog?.(
"stderr",
`[paperclip-runner] runnerd diagnostic: ${redactSensitiveText(message).slice(-4_096)}\n`,
);
},
lifecyclePolicy: input.execution.session.lifecyclePolicy,
runtimeContext:
"runtimeContext" in input.execution
? input.execution.runtimeContext
: null,
runnerRuntimeContext: remoteRuntimeContext,
baseInstructions: recoveryContext?.baseInstructions,
runnerFilesystemRoot: remoteRunnerFilesystemRoot ?? undefined,
resumeWorkingDirectory: runnerExecution.workspace.cwd,
externallySandboxed: adapterExecutionTargetIsCommandBacked(remoteTarget),
computerTool: nativeComputerToolForExecution(input.execution, remoteTarget),
persistentAgentHome: remoteTarget?.transport === "computer" ? remoteTarget.fileAuthority.agentHome : undefined,
opencodeRuntimeDirectory: remoteRunnerFilesystemRoot
? posix.join(remoteRunnerFilesystemRoot, "opencode")
: undefined,
resumeDynamicTools: dynamicTools,
resumeCompletionContract: {
revision: input.execution.completionContract.contract.revision,
criterionIds:
input.execution.completionContract.contract.criteria.map(
(criterion) => criterion.id,
),
},
resumeActiveTurnId:
recoveryContext?.persistedSession?.activeTurnId ?? null,
resumeProviderSession: recoveryContext?.persistedSession,
providerRecoveryPolicy:
recoveryContext?.providerRecoveryPolicy ??
(input.execution.provider.kind === "acpx" &&
input.execution.interactionResponses.length > 0
? "allow_replacement_after_governed_wait"
: undefined),
prpIdentity: {
runnerInstanceId: effectiveRunnerInstanceId,
environmentLeaseId: effectiveEnvironmentLeaseId,
runId: input.execution.binding.runId,
normalizedSessionId:
input.execution.session.normalizedSessionId ??
`session-${input.execution.binding.runId}`,
turnId: `turn-${input.execution.binding.runId}`,
itemId: `item-${input.execution.binding.runId}`,
},
warmTransitionRegistrationMode: retainedTransition
? "routed_connect"
: undefined,
authorizeWarmTransitionRecovery: retainedTransition
? async (
stage:
"before_bootstrap" | "before_spawn" | "before_authentication",
) => {
if (!recoveryPending) return;
const current = await verifyWarmTransitionRestart(input);
if (
current.transitionId !== retainedTransition.transitionId ||
(stage === "before_bootstrap" &&
current.stateFingerprint !==
retainedTransition.stateFingerprint)
) {
throw new Error(
"native_runner_warm_transition_recovery_unproven",
);
}
}
: undefined,
onWarmTransitionRecoveryCompleted: retainedTransition
? (completed: { transitionId: string }) => {
// Only the package's fresh completed new-authority snapshot can
// reach this callback. A tombstone or caller hint cannot retire
// the server's pending-recovery admission checks.
if (completed?.transitionId !== retainedTransition.transitionId) {
throw new Error(
"native_runner_warm_transition_recovery_unproven",
);
}
recoveryPending = false;
}
: undefined,
controlPlaneRegistration: async (authority, attachmentIdentity) => {
if (retainedTransition && recoveryPending) {
const current = await verifyWarmTransitionRestart(input);
if (
current.stateFingerprint !== retainedTransition.stateFingerprint
) {
throw new Error(
"native_runner_warm_transition_recovery_unproven",
);
}
}
return measureNativeRunnerSpan(
input.trace,
"runner.transport.connect",
async () => {
if (target.kind === "local") {
const selectedAtMs = Date.now();
await input.trace?.record({
name: "runner.transport.selected",
parentName: "runner.transport.connect",
startedAtMs: selectedAtMs,
endedAtMs: selectedAtMs,
attributes: {
mode: "local_loopback",
connectionOwner: "runnerd",
},
});
await input.onLog?.(
"stderr",
"[paperclip-runner] transport mode=local_loopback state=connecting\n",
);
const registration = await measureNativeRunnerSpan(
input.trace,
"runner.prp.route.register",
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
agentId: input.execution.binding.agentId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
);
return {
...registration,
startupFailureCode: "runner_local_connect_failed" as const,
};
}
const requiredMode = resolveRemoteRunnerTransportMode({
target,
runnerIngressAuthorized: input.runnerIngressAuthorized === true,
});
let transport: PaperclipRunnerTransport;
if (requiredMode === "dial_wss") {
// Validate eligibility before staging any artifact.
transport = await measureNativeRunnerSpan(
input.trace,
"runner.transport.resolve",
() =>
resolvePaperclipRunnerTransport({
target,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
runnerIngressAuthorized:
input.runnerIngressAuthorized === true,
}),
);
if (
transport.mode === "direct_outbound" &&
transport.caBundlePath &&
!existsSync(transport.caBundlePath)
) {
throw new Error(
"runner_direct_wss_failed: configured runner CA bundle is unavailable",
);
}
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.prepare",
() => prepareRemoteRunner(requiredMode),
);
} else {
await measureNativeRunnerSpan(
input.trace,
"runner.artifact.prepare",
() => prepareRemoteRunner(requiredMode),
);
// Provider endpoint acquisition happens only after runnerd is staged
// and its listener capability has been verified.
transport = await measureNativeRunnerSpan(
input.trace,
"runner.ingress.acquire",
() =>
resolvePaperclipRunnerTransport({
target,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
runnerIngressAuthorized: true,
}),
);
}
const selectedAtMs = Date.now();
await input.trace?.record({
name: "runner.transport.selected",
parentName: "runner.transport.connect",
startedAtMs: selectedAtMs,
endedAtMs: selectedAtMs,
attributes: {
mode: transport.mode,
connectionOwner:
transport.mode === "provider_ingress"
? "paperclip"
: "runnerd",
},
});
await input.onLog?.(
"stderr",
`[paperclip-runner] transport mode=${transport.mode} state=connecting\n`,
);
if (transport.mode === "direct_outbound") {
const inbound = await measureNativeRunnerSpan(
input.trace,
"runner.prp.route.register",
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
agentId: input.execution.binding.agentId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
{ parentName: "runner.transport.connect" },
);
let caBundlePath = transport.caBundlePath;
if (caBundlePath && remoteBinary) {
const remoteCaBundlePath = posix.join(
posix.dirname(remoteBinary),
"runner-ca-bundle.pem",
);
remoteCaBundleMapping = {
sourcePath: caBundlePath,
targetPath: remoteCaBundlePath,
};
caBundlePath = remoteCaBundlePath;
}
return {
connection: {
mode: "connect" as const,
connectUrl: transport.connectUrl,
...(caBundlePath ? { caBundlePath } : {}),
},
startupFailureCode: "runner_direct_wss_failed" as const,
checkpoint: checkpointRemoteRunner,
release: () => inbound.release(),
};
}
if (transport.mode !== "provider_ingress") {
throw new Error("runner_transport_mode_changed_after_dispatch");
}
let outbound: ReturnType<typeof connectRunnerPrpIngress> | null =
null;
let activation: Promise<void> | null = null;
return {
connection: {
mode: "listen" as const,
listenAddress: transport.listenAddress,
listenPort: transport.listenPort,
listenPath: transport.listenPath,
},
activate: () => {
activation = measureNativeRunnerSpan(
input.trace,
"runner.transport.activation",
async () => {
await measureNativeRunnerSpan(
input.trace,
"runner.ingress.wait_for_process",
() => remoteRunnerProcessSpawned,
);
outbound = connectRunnerPrpIngress({
authority,
endpoint: transport.ingress,
onStateChange: (state, failureCode) => {
void input.onLog?.(
"stderr",
`[paperclip-runner] transport mode=provider_ingress state=${state}${failureCode ? ` failure=${failureCode}` : ""}\n`,
);
},
});
},
{ parentName: "runner.session.startup" },
);
// Cancellation can release the sandbox while activation is
// still waiting for the remote runner process. Attach a
// rejection observer immediately; `ready()` still awaits the
// original promise and reports the same startup failure, but
// an early teardown can no longer crash the controller with
// an unhandled plugin RPC rejection.
void activation.catch(() => undefined);
},
ready: async () => {
await measureNativeRunnerSpan(
input.trace,
"runner.transport.ready",
async () => {
await activation;
if (!outbound)
throw new Error("runner_ingress_unavailable");
await measureNativeRunnerSpan(
input.trace,
"runner.prp.authenticate",
() => outbound!.ready,
);
},
{ parentName: "runner.session.startup" },
);
},
get failure() {
return outbound?.failure;
},
startupFailureCode: "runner_ingress_unavailable" as const,
checkpoint: checkpointRemoteRunner,
release: async () => {
if (outbound) await outbound.close();
else await transport.ingress.close();
},
};
},
);
},
}).transport,
});
const boundManagedSessions = new WeakSet<NativeSession>();
const wrapManagedSession = (session: NativeSession): NativeSession => {
if (isGrok && grokCredential?.home && !boundManagedSessions.has(session)) {
boundManagedSessions.add(session);
// The launch runtime directory already ends in "acpx"; ACPX adds its
// own namespace beneath it in resolveAcpxRuntimeRoot.
const relativeHome = `acpx/acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(input.execution))}/grok-home`;
const localHome = resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", relativeHome);
const remoteHome = remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, relativeHome) : null;
const readAuth = async (name: string): Promise<Buffer> => {
if (!remoteHome || !remoteCommandRunner) return Buffer.from(await readLocalAiCredentialFile(join(localHome, name)));
const script = `const fs=require('node:fs'),path=require('node:path');let fd;try{const file=process.argv[1];let parent=path.dirname(file);while(true){if(!fs.lstatSync(parent).isDirectory())throw Error('directory');const next=path.dirname(parent);if(next===parent)break;parent=next;}fd=fs.openSync(file,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);const st=fs.fstatSync(fd);if(!st.isFile()||st.uid!==process.getuid()||(st.mode&511)!==384||st.size>65536)throw Error('credential');const b=Buffer.alloc(65537);let n=0;while(n<b.length){const k=fs.readSync(fd,b,n,b.length-n,n);if(!k)break;n+=k;}if(n>65536)throw Error('size');process.stdout.write(b.subarray(0,n).toString('base64'));b.fill(0);}catch(e){process.exitCode=e.code==='ENOENT'?66:1;}finally{if(fd!==undefined)fs.closeSync(fd);}`;
const result = await remoteCommandRunner.execute({ command: "node", args: ["-e", script, posix.join(remoteHome, name)], bypassSession: true, timeoutMs: 10000 });
if (result.exitCode !== 0 || result.timedOut) throw Object.assign(new Error("Grok credential refresh handoff unavailable"), { code: result.exitCode === 66 ? "ENOENT" : "INVALID_CREDENTIAL" });
return Buffer.from(result.stdout, "base64");
};
return bindManagedNativeCredentialTurn(session, {
copyBack: async () => {
await copyBackGrokAuth({ hostHomeDir: grokCredential.home!, log: () => {},
readSandboxAuth: () => readAuth("auth.json").catch((error) => {
if (error.code !== "ENOENT") throw error;
return readAuth("auth-refresh.json");
}),
});
},
remove: async () => {
for (const name of ["auth.json", "auth-refresh.json", "auth-refresh.json.tmp"]) {
rmSync(join(localHome, name), { force: true });
if (remoteHome && remoteCommandRunner) {
const result = await remoteCommandRunner.execute({ command: "rm", args: ["-f", "--", posix.join(remoteHome, name)], bypassSession: true, timeoutMs: 10000 });
if (result.exitCode !== 0 || result.timedOut) throw new Error("Grok credential cleanup failed");
}
}
},
});
}
if (!input.managedAiCredentialHome || input.execution.provider.kind !== "codex" || boundManagedSessions.has(session)) return session;
boundManagedSessions.add(session);
const remoteAuth = remoteRunnerFilesystemRoot ? posix.join(remoteRunnerFilesystemRoot, "codex-home", "auth.json") : null;
const localAuth = join(root, "codex-home", "auth.json");
return bindManagedNativeCredentialTurn(session, {
copyBack: async () => {
await copyBackCodexAuth({
hostAuthPath: join(input.managedAiCredentialHome!, "auth.json"),
readSandboxAuth: async () => {
if (!remoteAuth || !remoteCommandRunner) return readFileSync(localAuth);
const result = await remoteCommandRunner.execute({ command: "base64", args: [remoteAuth], bypassSession: true, timeoutMs: 10000 });
if (result.exitCode !== 0 || result.timedOut) throw new Error("AI credential copy-back failed");
return Buffer.from(result.stdout, "base64");
},
log: () => {},
});
},
remove: async () => {
rmSync(localAuth, { force: true });
if (remoteAuth && remoteCommandRunner) await remoteCommandRunner.execute({ command: "rm", args: ["-f", "--", remoteAuth], bypassSession: true, timeoutMs: 10000 });
},
});
};
const priorAuthorityEpoch = sessionToolAuthorityEpochs.get(sessionScopeId);
if (priorAuthorityEpoch && priorAuthorityEpoch !== authorityEpoch) {
priorAuthorityEpoch.revoke();
}
sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch);
return {
bindManagedSession: wrapManagedSession,
descriptor: () => backend.descriptor(),
openSession: async (sessionInput) => wrapManagedSession(await backend.openSession(sessionInput)),
recoverSession: async (snapshot, options) => {
const result = backend.recoverSession ? await backend.recoverSession(snapshot, options) : { recovered: false, reason: "driver does not support recovery" };
return result.session ? { ...result, session: wrapManagedSession(result.session) } : result;
},
openReplacementSession: async (sessionInput) => {
await measureNativeRunnerSpan(
input.trace,
"provider.session.archive_before_replacement",
archiveContinuityState,
{ parentName: "native.session.execute" },
);
return wrapManagedSession(await backend.openSession(sessionInput));
},
} satisfies NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession };
}