PaperClipAI/server/src/services/heartbeat/runtime-execution.ts

import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js";
import { readContextCompactionRequest } from "../context-compaction.js";
import { remoteWorkspaceFailureCause } from "../remote-workspace-transport.js";
import { readNativePostResultFailureDiagnostic } from "../native-runtime/native-post-result-failure.js";
import { acceptedNativeAdapterResult } from "../native-runtime/native-accepted-result-settlement.js";
import { NativeStartupCleanupPendingError } from "../native-runtime/native-startup-cleanup.js";
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
import { describeFailedAttachment } from "../native-runtime/remote-warm-session-recovery.js";

import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js";
import { environmentService } from "../environments.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
import { AsyncLocalStorage } from "node:async_hooks";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { parseNativeSessionGoalControl } from "./scheduling.js";
import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";
import {
  PAPERCLIP_WAKE_PAYLOAD_KEY,
  configuredPaperclipApiBaseUrl,
  buildPaperclipRuntimeMcpServers,
  createAdapterRuntimeToolAccess,
  paperclipApiBaseUrl,
  createAdapterRuntimeMcpAccess,
  createManagedMcpRunConfig,
  revokeHeartbeatRunGatewayTokens,
} from "./run-preparation.js";
import {
  WorkspaceValidationFailure,
  isWorkspaceValidationFailure,
  fingerprintFinalizeWorkspaceBranchValidation,
  attachPaperclipSessionMetadataToSessionParams,
} from "./workspaces.js";
import { recordLegacyWorkspaceRestoreFailure } from "../legacy-execution-recovery.js";
import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js";
import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils";
import { externalObjectService } from "../external-objects.js";
import { isAgentDirectoryCopy } from "../agent-directory-working-copies.js";
import { restoreNativeWorkspaceBestEffort } from "../native-runtime/native-workspace-best-effort.js";
import {
  withNativeWorkspaceFinalizationOwnership,
  NativeWorkspaceFinalizationBusyError,
  NativeWorkspaceFinalizationOwnershipLostError,
  type NativeWorkspaceFinalizationOwnership,
} from "../native-runtime/native-workspace-finalization-ownership.js";
import { createRunUsageRecorder, readAcceptedRunUsageCheckpoint } from "../usage-receipts.js";
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { isConversation } from "../agent-conversations.js";
import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase";
import { remoteExecutionHasStopped } from "../remote-execution-termination.js";
import { registerAdapterExecutionControl } from "../adapter-execution-control.js";
import { resolveManagedOpenAiBilling } from "@paperclipai/adapter-utils";
import { and, eq, isNull, sql } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db";
import { conflict } from "../../errors.js";
import { getStartupTraceContext } from "../../instrumentation.js";
import { logger } from "../../middleware/logger.js";
import {
  buildNativeProviderEnvironment,
  executePaperclipNativeSession,
  finalizeNativeRun,
  isRunnerIngressAuthorized,
  materializeLegacyQuestionResponseWakeProjection,
  NativeCancellationPendingRecoveryError,
  NativeControllerDetachedForRestartError,
  recordNativeFinalizationFailure,
} from "../native-runtime/index.js";
import {
  buildNativeHeartbeatPreparationSpans,
  buildNativeWakeIngressSpan,
} from "../native-runtime/native-run-trace.js";
import { PROVIDER_TRACE_MAX_BYTES } from "../provider-trace-store.js";
import type { AdapterExecutionResult } from "../../adapters/index.js";
import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
import { parseObject, asNumber } from "../../adapters/utils.js";
import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js";
import {
  ensureGitWorktreeBranchCoherent,
  formatManagedGitWorktreeBranchInspection,
  inspectManagedGitWorktreeBranch,
} from "../workspace-runtime.js";
import {
  blockRunnerGoalRecovery,
  failRunnerGoalAction,
  isRunnerGoalActionCompleted,
  settleLiveRunnerGoalBeforeInterrupt,
} from "../runner-goals.js";
import type { Db, agents, executionWorkspaces } from "@paperclipai/db";
import type { AdapterExecutionContext, ServerAdapterModule } from "../../adapters/index.js";
import type { HeartbeatRuntimeSelectionInput, HeartbeatRuntimeSelectionResult } from "./runtime-selection.js";
import type { createHeartbeatRunState } from "./run-state.js";
import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
import type { createHeartbeatRunControl } from "./run-control.js";
import type { createHeartbeatQueue } from "./queue.js";
import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import type { createAdapterExecutionControl } from "../adapter-execution-control.js";
import type { executionWorkspaceService } from "../execution-workspaces.js";
import type { workspaceOperationService } from "../workspace-operations.js";
import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";
import type { instanceSettingsService } from "../instance-settings.js";
import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js";
import type { ProviderResourceDisposition } from "../environment-runtime.js";
import type { NativeExecutionInput, NativeSessionBackend } from "../../vendor/paperclip-runner/index.js";
import type { PluginWorkerManager } from "../plugin-worker-manager.js";
import type { RealizedExecutionWorkspace } from "../workspace-runtime.js";
import type { reserveWarmNativeInstructionDirectory } from "../native-runtime/index.js";

type Selection = HeartbeatRuntimeSelectionInput;
type Selected = Extract<HeartbeatRuntimeSelectionResult, { selected: true }>;
type NativeSessionInput = Parameters<typeof executePaperclipNativeSession>[0];
type RunState = ReturnType<typeof createHeartbeatRunState>;
type Lifecycle = ReturnType<typeof createHeartbeatLifecycle>;

/** Dispatches a prepared run; admission and terminal run status remain owned by the executor. */
export interface HeartbeatRuntimeExecutionInput {
  run: typeof heartbeatRuns.$inferSelect;
  agent: typeof agents.$inferSelect;
  options: {
    nativeSessionBackendFactory?: (execution: NativeExecutionInput, context?: { recoveryPurpose: "execution" | "stop_cleanup" }) => NativeSessionBackend;
    pluginWorkerManager?: PluginWorkerManager;
  };
  task: Pick<Selection["task"], "issueContext" | "context" | "executionContinuation"> & {
    issueRef: (NonNullable<Selection["task"]["issueRef"]> & { projectId: string | null }) | null;
    issueId: string | null;
    taskKey: string | null;
  };
  runtime: Pick<Selected, "nativeExecution" | "nativeRunnerInstanceId" | "getNativeFreshSessionHandoff"> & {
    nativeRuntimeResolution: Selection["nativeRuntimeResolution"];
    adapter: ServerAdapterModule;
    runtimeForAdapter: AdapterExecutionContext["runtime"];
    getFreshSessionHandoff: AdapterExecutionContext["getFreshSessionHandoff"];
    runOptions: Selection["recovery"];
  };
  workspace: {
    persistedExecutionWorkspace: typeof executionWorkspaces.$inferSelect | null;
    executionWorkspace: RealizedExecutionWorkspace;
    executionTarget: Selection["workspace"]["executionTarget"];
    workspaceOperationRecorder: ReturnType<ReturnType<typeof workspaceOperationService>["createRecorder"]>;
    nativeWorkspaceSync: Selected["nativeWorkspaceSync"];
    workspaceRestoreSource: Parameters<typeof recordLegacyWorkspaceRestoreFailure>[3] | null;
    remoteExecution: unknown;
  };
  config: Pick<Selection["config"], "runtimeConfig" | "resolvedConfig" | "managedAiRuntime"> & {
    resolvedInstanceSettings: { experimental: Pick<Awaited<ReturnType<ReturnType<typeof instanceSettingsService>["getExperimental"]>>, "enableWorkspaceBranchReconcileForward" | "enableWorkspaceDirtyQuarantineRepair"> };
    configuredTaskEnvironment: Parameters<typeof configuredEnvironmentProjection>[0];
    adapterEnv: Parameters<typeof buildNativeProviderEnvironment>[0];
    agentIdentity: AdapterExecutionContext["agentIdentity"];
    observability?: AdapterExecutionContext["observability"];
    runtimeEnv: Selection["runtimeEnv"];
    useHostGitHub: boolean;
    githubSelection: { configured: boolean };
  };
  session: {
    runtimeSessionParamsForAdapter: Record<string, unknown> | null;
    configuredModel: string | null;
    sessionConfigMetadata: EffectiveRunSessionConfigMetadata;
    goalCheckpointSession: { current: { params: Record<string, unknown>; displayId: string } | null };
  };
  instructions: {
    // Instruction callbacks can replace the copy and receipt while a provider is running.
    readonly instructionCopy: Selection["config"]["instructionCopy"];
    getInstructionSave: () => Record<string, unknown> | null;
    nativeInstructionWorkingCopy: () => NativeSessionInput["instructionWorkingCopy"];
    collectStoppedInstructions: AdapterExecutionContext["onProviderStopped"];
    nativeInstructionReservation: Awaited<ReturnType<typeof reserveWarmNativeInstructionDirectory>>;
    releaseInstructionCopy: () => Promise<void>;
  };
  trace: {
    providerTraceCapture: { path: string } | null;
    nativeRunnerPreparationSpans: NonNullable<NativeSessionInput["preparationSpans"]>;
    attestedQuestionResponseAtMs: Parameters<typeof buildNativeWakeIngressSpan>[0]["attestedQuestionResponseAtMs"];
    attemptStartedAtMs: number;
    environmentAcquireStartedAtMs: number;
    environmentRealizeEndedAtMs: number;
  };
  control: {
    executionControl: ReturnType<typeof createAdapterExecutionControl>;
    executionPhaseContext: Parameters<typeof withAdapterExecutionPhase>[0];
    dispatchResolvedInteractionContinuationWithAtomicGate: <T>(dispatch: (markDispatchStarted: () => void) => Promise<T>) => Promise<
      { dispatched: true; resultPromise: Promise<T> } | { dispatched: false }
    >;
  };
  output: Pick<AdapterExecutionContext, "onLog"> & {
    onPersistedNativeLog?: (cursor: number, chunk: string) => Promise<void>;
    onAdapterEvent: NonNullable<AdapterExecutionContext["onEvent"]>;
    onAdapterMeta: AdapterExecutionContext["onMeta"];
    identityRedactor: ReturnType<typeof createAgentIdentityRedactor>;
    appendIdentityRedactedLog: AdapterExecutionContext["onLog"];
  };
  services: Pick<RunState, "upsertTaskSession" | "getRun"> &
    Pick<Lifecycle, "persistRunProcessMetadata" | "recordCurrentHeartbeatRunRuntimeProgress"> &
    Pick<ReturnType<typeof createHeartbeatRunControl>, "cancelRunInternal"> &
    Pick<ReturnType<typeof createHeartbeatQueue>, "enqueueWakeup" | "dispatchPendingNativeStatusWakeups"> & {
    executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "getStoredById">;
    instanceSettings: Pick<ReturnType<typeof instanceSettingsService>, "getExperimental">;
    envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "releaseForRun">;
  };
  // Teardown must observe these updates immediately, including when execution later throws.
  effects: {
    onUsageCaptureReady: (persistFailure: () => Promise<void>) => void;
    onNativeDispatchStarted: () => void;
    onLegacyAdapterEntered: () => void;
    onWorkspaceRestoreFailure: (evidence: Record<string, unknown>) => void;
    onProviderResourceDisposition: (disposition: ProviderResourceDisposition) => void;
    onNativeWorkspaceFinalizeScheduled: () => void;
    onNativeSessionResumeScheduled: () => void;
    onNativeOwnershipHeld: () => void;
  };
}

export type HeartbeatRuntimeExecutionResult = { dispatched: false } | {
  dispatched: true;
  adapterResult: AdapterExecutionResult;
};

export class NativeSessionResumeScheduledError extends Error {
  constructor(readonly original: unknown) {
    super("Native session recovery has been scheduled for the same run.");
    this.name = "NativeSessionResumeScheduledError";
  }
}

export class NativeWorkspaceFinalizeScheduledError extends Error {
  constructor(
    readonly original: unknown,
    readonly terminalFailure: boolean,
    readonly reasonCode:
      "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
  ) {
    super("Native workspace finalization recovery has been scheduled.");
    this.name = "NativeWorkspaceFinalizeScheduledError";
  }
}

function readNonEmptyString(value: unknown): string | null {
  return typeof value === "string" && value.trim().length > 0 ? value : null;
}

export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExecutionInput): Promise<HeartbeatRuntimeExecutionResult> {
  const { run, agent, options } = input;
  const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task;
  const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime;
  const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace;
  const remoteWorkspaceAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
  const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, observability, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config;
  const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session;
  const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;
  const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace;
  const { executionControl, executionPhaseContext, dispatchResolvedInteractionContinuationWithAtomicGate } = input.control;
  const dispatchWithTiming: HeartbeatRuntimeExecutionInput["control"]["dispatchResolvedInteractionContinuationWithAtomicGate"] = dispatch => {
    const providerDispatch = AsyncLocalStorage.bind(dispatch);
    return measureRunPhase("execution.dispatch_admission", () => dispatchResolvedInteractionContinuationWithAtomicGate(providerDispatch));
  };
  const { identityRedactor, appendIdentityRedactedLog, onLog, onAdapterEvent, onAdapterMeta } = input.output;
  const { executionWorkspacesSvc, instanceSettings, envOrchestrator, upsertTaskSession, getRun, cancelRunInternal, enqueueWakeup, persistRunProcessMetadata, recordCurrentHeartbeatRunRuntimeProgress, dispatchPendingNativeStatusWakeups } = input.services;
  const localAgentJwtScope =
    issueRef?.workMode === "skill_test"
      ? { kind: "skill_test" as const, issueId: issueRef.id }
      : { kind: "standard" as const };
  const authToken =
    nativeRuntimeResolution.kind === "legacy" &&
    adapter.supportsLocalAgentJwt
      ? createLocalAgentJwt(
          agent.id,
          agent.companyId,
          agent.adapterType,
          run.id,
          run.responsibleUserId,
          localAgentJwtScope,
        )
      : null;
  if (
    nativeRuntimeResolution.kind === "legacy" &&
    adapter.supportsLocalAgentJwt &&
    !authToken
  ) {
    logger.warn(
      {
        companyId: agent.companyId,
        agentId: agent.id,
        runId: run.id,
        adapterType: agent.adapterType,
      },
      "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY",
    );
  }
  let adapterFinalizeOutcome: "succeeded" | "failed" | null = null;
  let preserveRunGatewayTokens = false;
  let pluginWorkspaceFinalization: Record<string, unknown> | null = null;
  const inspectFinalizeWorkspaceBranch = async () => {
    const workspaceRecord = persistedExecutionWorkspace?.id
      ? await executionWorkspacesSvc.getStoredById(
          agent.companyId,
          persistedExecutionWorkspace.id,
        )
      : persistedExecutionWorkspace;
    pluginWorkspaceFinalization = workspaceRecord
      ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id)
      : null;
    if (pluginWorkspaceFinalization || readRemoteWorkspaceAuthority(workspaceRecord?.metadata)
      || workspaceRecord?.strategyType !== "git_worktree") return null;

    const worktreePath =
      readNonEmptyString(workspaceRecord.providerRef) ??
      readNonEmptyString(workspaceRecord.cwd) ??
      readNonEmptyString(executionWorkspace.worktreePath) ??
      readNonEmptyString(executionWorkspace.cwd);
    const expectedBranchName =
      readNonEmptyString(workspaceRecord.branchName) ??
      readNonEmptyString(executionWorkspace.branchName);
    if (!worktreePath || !expectedBranchName) return null;

    const inspection = await inspectManagedGitWorktreeBranch({
      worktreePath,
      expectedBranchName,
    });
    return { workspaceRecord, inspection };
  };
  const recordWorkspaceFinalize = async (
    status: "succeeded" | "failed",
    metadata?: Record<string, unknown>,
    restore?: () => Promise<void>,
  ) => {
    if (adapterFinalizeOutcome) return;
    await workspaceOperationRecorder.recordOperation({
      phase: "workspace_finalize", cwd: executionWorkspace.cwd,
      metadata: { adapterType: agent.adapterType, executionTargetKind: executionTarget?.kind ?? "local",
        ...(restore ? { owningService: "native_workspace_finalizer" } : {}), ...metadata },
      run: async () => {
        await restore?.();
        let finalizeBranchMetadata: Record<string, unknown> | null = null;
        let finalizeBranchRepairMetadata: Record<string, unknown> | null =
          null;
        if (status === "succeeded") {
          const branchInspection = await inspectFinalizeWorkspaceBranch();
          if (branchInspection) {
            let inspection = branchInspection.inspection;
            const initialManagedGitWorktreeBranch =
              formatManagedGitWorktreeBranchInspection(inspection);
            if (
              !inspection.valid &&
              inspection.reasonCode === "branch_mismatch" &&
              inspection.repoRoot
            ) {
              let repairedExpectedBranchName = inspection.expectedBranchName;
              try {
                const coherence = await ensureGitWorktreeBranchCoherent({
                  db,
                  repoRoot: inspection.repoRoot,
                  worktreePath: inspection.worktreePath,
                  expectedBranchName: inspection.expectedBranchName,
                  actualBranchName: inspection.actualBranchName,
                  sourceIssue: issueRef
                    ? {
                        id: issueRef.id,
                        identifier: issueRef.identifier,
                        title: issueRef.title,
                        workMode: issueRef.workMode,
                      }
                    : null,
                  executionWorkspaceId: branchInspection.workspaceRecord.id,
                  heartbeatRunId: run.id,
                  enableWorkspaceBranchReconcileForward:
                    resolvedInstanceSettings.experimental
                      .enableWorkspaceBranchReconcileForward,
                  enableWorkspaceDirtyQuarantineRepair:
                    resolvedInstanceSettings.experimental
                      .enableWorkspaceDirtyQuarantineRepair,
                  persistForwardReconcile: false,
                  reconcileOperationPhase: "workspace_finalize",
                  recorder: workspaceOperationRecorder,
                });
                if (
                  coherence.branchName &&
                  coherence.branchName !==
                    branchInspection.workspaceRecord.branchName
                ) {
                  repairedExpectedBranchName = coherence.branchName;
                  executionWorkspace.branchName = coherence.branchName;
                  executionWorkspace.warnings.push(...coherence.warnings);
                }
              } catch (repairErr) {
                const workspaceValidationFailure =
                  isWorkspaceValidationFailure(repairErr) ? repairErr : null;
                finalizeBranchMetadata = {
                  executionWorkspaceId: branchInspection.workspaceRecord.id,
                  ...initialManagedGitWorktreeBranch,
                };
                finalizeBranchRepairMetadata = {
                  attempted: true,
                  succeeded: false,
                  initial: initialManagedGitWorktreeBranch,
                  reason:
                    repairErr instanceof Error
                      ? repairErr.message
                      : String(repairErr),
                };
                await workspaceOperationRecorder.recordOperation({
                  phase: "workspace_finalize",
                  cwd: executionWorkspace.cwd,
                  metadata: {
                    adapterType: agent.adapterType,
                    executionTargetKind: executionTarget?.kind ?? "local",
                    ...metadata,
                    managedGitWorktreeBranch: finalizeBranchMetadata,
                    managedGitWorktreeBranchRepair:
                      finalizeBranchRepairMetadata,
                    ...(workspaceValidationFailure?.resultJson
                      ? {
                          workspaceValidation:
                            workspaceValidationFailure.resultJson
                              .workspaceValidation ??
                            workspaceValidationFailure.resultJson,
                        }
                      : {}),
                  },
                  run: async () => ({
                    status: "failed",
                    stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`,
                  }),
                });
                adapterFinalizeOutcome = "failed";
                throw repairErr;
              }

              const repairedInspection =
                await inspectManagedGitWorktreeBranch({
                  worktreePath: inspection.worktreePath,
                  expectedBranchName: repairedExpectedBranchName,
                  repoRoot: inspection.repoRoot,
                });
              finalizeBranchRepairMetadata = {
                attempted: true,
                succeeded: repairedInspection.valid,
                initial: initialManagedGitWorktreeBranch,
                repaired:
                  formatManagedGitWorktreeBranchInspection(
                    repairedInspection,
                  ),
              };
              inspection = repairedInspection;
            }

            const managedGitWorktreeBranch =
              formatManagedGitWorktreeBranchInspection(inspection);
            finalizeBranchMetadata = {
              executionWorkspaceId: branchInspection.workspaceRecord.id,
              ...managedGitWorktreeBranch,
            };
            if (!inspection.valid) {
              const workspaceValidationFingerprint =
                fingerprintFinalizeWorkspaceBranchValidation({
                  issueId: issueRef?.id ?? null,
                  executionWorkspaceId: branchInspection.workspaceRecord.id,
                  inspection: managedGitWorktreeBranch,
                });
              await workspaceOperationRecorder.recordOperation({
                phase: "workspace_finalize",
                cwd: executionWorkspace.cwd,
                metadata: {
                  adapterType: agent.adapterType,
                  executionTargetKind: executionTarget?.kind ?? "local",
                  ...metadata,
                  managedGitWorktreeBranch: finalizeBranchMetadata,
                  ...(finalizeBranchRepairMetadata
                    ? {
                        managedGitWorktreeBranchRepair:
                          finalizeBranchRepairMetadata,
                      }
                    : {}),
                },
                run: async () => ({
                  status: "failed",
                  stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`,
                }),
              });
              adapterFinalizeOutcome = "failed";
              throw new WorkspaceValidationFailure(
                `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`,
                {
                  workspaceValidation: {
                    reason: "git_worktree_branch_incoherence",
                    fingerprint: workspaceValidationFingerprint,
                    adapterType: agent.adapterType,
                    issueId: issueRef?.id ?? null,
                    issueIdentifier: issueRef?.identifier ?? null,
                    persistedExecutionWorkspaceId:
                      branchInspection.workspaceRecord.id,
                    executionWorkspaceCwd: executionWorkspace.cwd,
                    managedGitWorktreeBranch: finalizeBranchMetadata,
                  },
                },
              );
            }
          }
        }
        return { status, metadata: {
          ...(pluginWorkspaceFinalization ? { pluginWorkspace: pluginWorkspaceFinalization } : {}),
          ...(finalizeBranchMetadata ? { managedGitWorktreeBranch: finalizeBranchMetadata } : {}),
          ...(finalizeBranchRepairMetadata ? { managedGitWorktreeBranchRepair: finalizeBranchRepairMetadata } : {}),
        } };
      },
    });
    // Only mark the outcome after the row landed, so a transient write
    // failure on the succeeded path can still be recovered by recording
    // finalize=failed from the catch path below.
    adapterFinalizeOutcome = status;
  };

  const producerOwnsUsage = usesNativeUsageDelivery({
    runnerd: nativeRuntimeResolution.kind === "native" && agent.adapterType === "paperclip_runner",
    remote: executionTarget?.kind === "remote", provider: nativeExecution?.provider,
  });
  const usageRecorder = producerOwnsUsage ? undefined : await measureRunPhase("execution.usage_startup", () =>
    createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));
  input.effects.onUsageCaptureReady(usageRecorder?.persistFailure ?? (async () => {}));
  let adapterResult: AdapterExecutionResult;
  const runGoalControlRequestId = readNonEmptyString(
    context.goalControlRequestId,
  );
  try {
    if (nativeRuntimeResolution.kind === "native") {
      if (!nativeExecution || !nativeRunnerInstanceId)
        throw new Error("native_runtime_selection_not_persisted");
      const expectedNativeMcpDigest =
        "runtimeContext" in nativeExecution &&
        nativeExecution.runtimeContext.mcp.bindingId
          ? nativeExecution.runtimeContext.mcp.digest
          : null;
      const nativeMcpServers = "homeMode" in nativeExecution && nativeExecution.homeMode === "preconfigured" ? [] : await measureRunPhase("execution.mcp_handoff", () => buildPaperclipRuntimeMcpServers({
        db,
        agent,
        runId: run.id,
        expectedAssignmentDigest: expectedNativeMcpDigest,
      }));
      if ("runtimeContext" in nativeExecution) {
        if (nativeMcpServers.length > 1)
          throw new Error(
            "native MCP realization must produce one aggregate gateway",
          );
        const server = nativeMcpServers[0] ?? null;
        const digest = server?.connectionId.startsWith("assignment:")
          ? server.connectionId.slice("assignment:".length)
          : null;
        if (digest && digest !== expectedNativeMcpDigest) {
          throw new Error("native MCP assignment digest mismatch");
        }
      }
      const nativeMcpServer = nativeMcpServers[0] ?? null;
      let sessionGoalControl = parseNativeSessionGoalControl(
        context.runnerGoalControl,
      );
      if (runGoalControlRequestId && !sessionGoalControl) {
        throw new Error("session_goal_control_payload_invalid");
      }
      // A hard restart replays the heartbeat context, not a new user
      // action. Do not repeat a completed create/replace/edit (which
      // could reactivate or clear a goal that finished while detached).
      const completedGoalControl =
        sessionGoalControl !== null &&
        taskKey !== null &&
        (await isRunnerGoalActionCompleted(
          db,
          {
            companyId: agent.companyId,
            agentId: agent.id,
            issueId: taskKey,
          },
          sessionGoalControl.requestId,
        ));
      if (completedGoalControl) sessionGoalControl = null;
      // Goal checkpoints already publish their task session. Goal-less turns
      // need the same durable pointer before cleanup can delay host completion.
      const taskSessionPublication = taskKey && !sessionGoalControl &&
        context.resumeSessionGoalHeartbeat !== true && !completedGoalControl
        ? {
            taskKey,
            adapterType: agent.adapterType,
            sessionParamsJson: attachPaperclipSessionMetadataToSessionParams(
              { ...runtimeSessionParamsForAdapter, cwd: executionWorkspace.cwd },
              configuredModel,
              sessionConfigMetadata,
            ),
          }
        : undefined;
      const nativeDispatchAtMs = Date.now();
      const runCreatedAtMs = run.createdAt.getTime();
      const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();
      const wakeComments = Array.isArray(
        parseObject(context.paperclipWake).comments,
      )
        ? (parseObject(context.paperclipWake).comments as unknown[])
        : [];
      const wakeIngressSpan = buildNativeWakeIngressSpan({
        runCreatedAtMs,
        wakeComments,
        attestedQuestionResponseAtMs,
      });
      if (wakeIngressSpan)
        nativeRunnerPreparationSpans.unshift(wakeIngressSpan);
      nativeRunnerPreparationSpans.push(
        ...buildNativeHeartbeatPreparationSpans({
          runCreatedAtMs,
          runStartedAtMs,
          attemptStartedAtMs,
          environmentAcquireStartedAtMs,
          environmentRealizeEndedAtMs,
          nativeDispatchAtMs,
        }),
      );
      const stopCleanup = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner"
        && await readClaimedNativeStopRecovery(db, run, runOptions.nativeLeaseOwner);
      const guardedDispatch =
        await dispatchWithTiming(
          (markDispatchStarted) => {
            return executePaperclipNativeSession({
              casesEnabled: context.paperclipCasesEnabled === true,
              db,
              execution: nativeExecution,
              observability,
              getFreshSessionHandoff: getNativeFreshSessionHandoff,
              refreshTools: context.refreshTools === true,
              conversationMode: isConversation(issueContext),
              turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000,
              runnerInstanceId: nativeRunnerInstanceId,
              leaseOwner: runOptions.nativeLeaseOwner,
              restartRecovery: runOptions.nativeRestartRecovery,
              onExecutionLeaseAcquired: runOptions.onNativeExecutionLeaseAcquired,
              backend:
                stopCleanup
                  ? options.nativeSessionBackendFactory?.(nativeExecution, { recoveryPurpose: "stop_cleanup" })
                  : options.nativeSessionBackendFactory?.(nativeExecution),
              useRunnerd: agent.adapterType === "paperclip_runner",
              adapterType: agent.adapterType,
              sessionGoalControl,
              sessionCompaction: readContextCompactionRequest(context),
              resumeSessionGoalHeartbeat:
                context.resumeSessionGoalHeartbeat === true ||
                completedGoalControl,
              taskSessionPublication,
              onGoalCheckpoint: async (snapshot) => {
                if (!taskKey) return;
                const params =
                  attachPaperclipSessionMetadataToSessionParams(
                    {
                      ...runtimeSessionParamsForAdapter,
                      sessionId: snapshot.identity.sessionId,
                      cwd: executionWorkspace.cwd,
                    },
                    configuredModel,
                    sessionConfigMetadata,
                  )!;
                const displayId =
                  snapshot.providerSessionId ?? snapshot.sessionId;
                await upsertTaskSession({
                  companyId: agent.companyId,
                  agentId: agent.id,
                  adapterType: agent.adapterType,
                  taskKey,
                  sessionParamsJson: params,
                  sessionDisplayId: displayId,
                  lastRunId: run.id,
                  lastError: null,
                });
                goalCheckpointSession.current = { params, displayId };
              },
              onLog,
              onPersistedNativeLog: input.output.onPersistedNativeLog,
              onEvent: onAdapterEvent,
              instructionWorkingCopy: nativeInstructionWorkingCopy(),

              onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
              preparationSpans: nativeRunnerPreparationSpans,
              // Bootstrap with executable/home discovery while keeping
              // configured provider values and the server-selected
              // workspace boundary authoritative.
              managedGitHub: !useHostGitHub && githubSelection.configured,
              billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined,
              managedAiCredentialIdentity: managedAiRuntime?.identity,
              managedAiAccountIdentity: managedAiRuntime?.sessionIdentity,
              managedAiCredentialWriteback: managedAiRuntime?.credentialWriteback,
              managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record<string, unknown>).CODEX_HOME) : undefined,
              dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined,
              runnerEnvironment: {
                ...configuredEnvironmentProjection(configuredTaskEnvironment),
                ...buildNativeProviderEnvironment(
                  adapterEnv,
                  process.env,
                  executionWorkspace.cwd,
                ),
                ...buildAgentIdentityEnv(agentIdentity),
                ...(input.instructions.instructionCopy && isAgentDirectoryCopy(input.instructions.instructionCopy) ? { AGENT_HOME: input.instructions.instructionCopy.executionRoot } : {}),
                ...(nativeMcpServer
                  ? {
                      PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name,
                      PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url,
                      PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token,
                    }
                  : {}),
                ...(providerTraceCapture
                  ? {
                      PAPERCLIP_PROVIDER_TRACE_PATH:
                        providerTraceCapture.path,
                      PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(
                        PROVIDER_TRACE_MAX_BYTES,
                      ),
                    }
                  : {}),
              },
              onTaskDispatched: markDispatchStarted,
              runnerExecutionTarget: executionTarget,
              runnerIngressAuthorized: isRunnerIngressAuthorized(
                nativeRuntimeResolution,
              ),
              runnerPublicUrl:
                runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null,
              runnerCaBundlePath:
                runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() ||
                null,
              runnerRemoteBinaryPath:
                runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() ||
                null,
              runnerRemoteCodexPath:
                runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() ||
                null,
              runnerRemoteCodexNpmSpec:
                runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() ||
                null,
              runnerRemoteProviderPackPath:
                runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() ||
                null,
              stopTaskForReassignment: async (target) => {
                await settleLiveRunnerGoalBeforeInterrupt(db, target);
                if (!target.runId) return;
                const prior = await getRun(target.runId);
                if (!prior || prior.companyId !== target.companyId || prior.agentId !== target.agentId) {
                  throw conflict("Reassignment run binding changed");
                }
                const stopped = await cancelRunInternal(target.runId, "Cancelled for task reassignment", {
                  errorCode: "issue_reassigned", suppressImmediateRecovery: true,
                  resultJson: { reassignmentStopConfirmed: true },
                });
                if (stopped && ["running", "queued", "scheduled_retry"].includes(stopped.status)) {
                  throw conflict("The previous run did not stop; reassignment was not applied");
                }
              },
              enqueueWakeup,
              syncIssueExternalObjects: externalObjectService(db, {
                pluginWorkerManager: options.pluginWorkerManager,
                enabled: async () => (await instanceSettings.getExperimental()).enableExternalObjects === true,
              }).syncIssueSafely,
              onSpawn: async (meta) => {
                markDispatchStarted();
                await persistRunProcessMetadata(run.id, { ...meta, targetKind: executionTarget?.kind ?? "local" });
              },
            });
          },
        );
      if (!guardedDispatch.dispatched) return { dispatched: false };
      input.effects.onNativeDispatchStarted();
      adapterResult = await guardedDispatch.resultPromise;
    } else {
      const interactionId = readNonEmptyString(context.interactionId);
      const legacyQuestionResponse =
        issueRef &&
        interactionId &&
        readNonEmptyString(context.interactionKind) ===
          "ask_user_questions" &&
        readNonEmptyString(context.interactionStatus) === "answered"
          ? await materializeLegacyQuestionResponseWakeProjection({
              db,
              companyId: agent.companyId,
              issueId: issueRef.id,
              runId: run.id,
              agentId: agent.id,
              interactionId,
            })
          : null;
      // Do not write the answer projection back to `context`: legacy
      // adapters need it in their prompt, but the authoritative answers
      // remain on the interaction instead of being duplicated in the
      // heartbeat run snapshot.
      const adapterContext: Record<string, unknown> = {
        ...context,
        ...(legacyQuestionResponse
          ? {
              [PAPERCLIP_WAKE_PAYLOAD_KEY]: {
                ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]),
                questionResponse: legacyQuestionResponse,
              },
            }
          : {}),
      };
      const runtimeTools = createAdapterRuntimeToolAccess({
        agentId: agent.id,
        companyId: agent.companyId,
        runId: run.id,
        responsibleUserId: run.responsibleUserId,
      });
      if (!runtimeTools) {
        logger.warn(
          {
            companyId: agent.companyId,
            agentId: agent.id,
            runId: run.id,
          },
          "runtime connection tools could not be delivered",
        );
      }
      const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({
        db,
        agent,
        runId: run.id,
      });
      const runtimeToolDelivery =
        adapter.runtimeToolDelivery ?? "invocation_context";
      if (runtimeTools && runtimeToolDelivery === "native_mcp") {
        runtimeMcpServers.unshift({
          name: "Paperclip connections",
          url: runtimeTools.mcpEndpoint,
          token: runtimeTools.bearerToken,
          connectionId: "paperclip-runtime-tools",
        });
      }
      if (authToken && configuredPaperclipApiBaseUrl() && issueRef) {
        runtimeMcpServers.unshift({ name: "Paperclip projects", url: `${paperclipApiBaseUrl()}/api/mcp/project-tools`,
          token: authToken, connectionId: "paperclip-project-tools" });
      }
      const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers);
      if (runtimeTools && runtimeToolDelivery === "invocation_context") {
        adapterContext.paperclipRuntimeTools = runtimeTools;
      }
      const managedMcpConfig = await measureRunPhase("execution.mcp_handoff", () => createManagedMcpRunConfig({
        db,
        agent,
        runId: run.id,
        config: runtimeConfig,
        projectId: issueRef?.projectId ?? null,
        issueId: issueRef?.id ?? null,
      }));
      if (managedMcpConfig) {
        adapterContext.paperclipManagedMcp = managedMcpConfig;
      }
      const guardedDispatch =
        await dispatchWithTiming(
          (markDispatchStarted) => {
            input.effects.onLegacyAdapterEntered();
            return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({
              getFreshSessionHandoff,
              agentIdentity,
              observability,
              runId: run.id,
              agent,
              runtime: runtimeForAdapter,
              config: runtimeConfig,
              context: adapterContext,
              executionContinuation: executionContinuation ?? null,
              runtimeCommandSpec:
                adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null,
              executionTarget,
              executionTransport: remoteExecution
                ? {
                    remoteExecution: remoteExecution as unknown as Record<
                      string,
                      unknown
                    >,
                  }
                : undefined,
              runtimeMcp,
              runtimeTools,
              onLog,
              onMeta: onAdapterMeta,
              onEvent: onAdapterEvent,
              onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
              onExecutionPhase: executionControl.phases.enter,
              startupTraceContext: getStartupTraceContext(),
              onRuntimeProgress: async (progress) => {
                await recordCurrentHeartbeatRunRuntimeProgress(
                  run,
                  progress,
                  issueId,
                );
              },
              onProviderStopped: collectStoppedInstructions,
              onDispatch: markDispatchStarted,
              signal: executionControl.controller.signal,
              ...(executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" ? {
                stopRemoteStartup: async () => {
                  // Scope comes from the running host invocation, never agent
                  // config. Keep adapter ownership until setup has unwound.
                  if (!executionControl.controller.signal.aborted) {
                    throw new Error("Remote startup stop requires a cancelled run");
                  }
                  const release = await envOrchestrator.releaseForRun({
                    heartbeatRunId: run.id,
                    companyId: agent.companyId,
                    agentId: agent.id,
                    status: "released",
                    providerResourceDisposition: "stop_and_retain",
                    cancelActiveWork: true,
                  });
                  if (release.errors.length || !await remoteExecutionHasStopped(db, agent.companyId, run.id)) {
                    throw new Error("Could not verify remote startup stopped");
                  }
                },
              } : {}),
              onCancellationReady: async () => {
                await registerAdapterExecutionControl(run.id, executionControl);
                const current = await getRun(run.id);
                if (!current || isHeartbeatRunTerminalStatus(current.status)) {
                  executionControl.controller.abort(new Error("Run stopped before provider startup"));
                }
              },
              onSpawn: async (meta) => {
                markDispatchStarted();
                await persistRunProcessMetadata(run.id, {
                  pid: meta.pid,
                  processGroupId:
                    "processGroupId" in meta &&
                    typeof meta.processGroupId === "number"
                      ? meta.processGroupId
                      : null,
                  startedAt: meta.startedAt,
                });
              },
              authToken: authToken ?? undefined,
            }));
          },
        );
      if (!guardedDispatch.dispatched) return { dispatched: false };
      adapterResult = await guardedDispatch.resultPromise;
    }
    adapterResult = identityRedactor.redact(adapterResult);
    if (run.runtimeMode === "legacy" && hasWorkspaceRestoreFailure(adapterResult.resultJson)
        && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") {
      const requiredWorkspaceRestoreEvidence = {
        workspaceRestoreFailure: adapterResult.resultJson!.workspaceRestoreFailure,
        ...(adapterResult.resultJson?.workspaceRestoreDiagnostic ? { workspaceRestoreDiagnostic: adapterResult.resultJson.workspaceRestoreDiagnostic } : {}),
      };
      input.effects.onWorkspaceRestoreFailure(requiredWorkspaceRestoreEvidence);
      // Retention is the fallback even if recording this receipt fails.
      input.effects.onProviderResourceDisposition("stop_and_retain");
      if (workspaceRestoreSource === null) throw new Error("legacy_workspace_restore_source_unavailable");
      await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);
    }
    for (const stream of ["stdout", "stderr"] as const) {
      const tail = identityRedactor.finish(stream);
      if (tail) await appendIdentityRedactedLog(stream, tail);
    }
    const instructionSave = input.instructions.getInstructionSave();
    if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };

    if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {
      if (usageRecorder) {
        const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
        adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
      } else {
        const accepted = await measureRunPhase("execution.usage", () => readAcceptedRunUsageCheckpoint(db, { companyId: run.companyId, runId: run.id }));
        adapterResult = { ...adapterResult, ...accepted, usageComplete: accepted?.complete ?? false,
          ...(!accepted ? { costStatus: "unpriced" as const } : {}) };
      }
    } else {
      // Stop may already own the terminal result. Preserve its metadata
      // while durably recording the proof needed to release admission.
      await db.update(heartbeatRuns).set({ costAccountingPending: true,
        usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`,
      }).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));
    }
    adapterResult = applyWorkspaceRestoreFailure(adapterResult);
    // A returned result can include a failed restore. Keep the workspace
    // barrier closed until required files have been restored.
    // If recording the barrier itself fails, propagate as a run failure
    // rather than silently leaving dependents stranded behind a missing
    // finalize row.
    const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => {
      try {
        if (remoteWorkspaceAuthority && persistedExecutionWorkspace) {
          if (!ownership || !executionTarget) throw new Error("remote_workspace_ownership_required");
          const current = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id);
          const authority = readRemoteWorkspaceAuthority(current?.metadata);
          const lease = authority?.location
            ? await environmentService(db).getLeaseById(authority.location.leaseId) : null;
          if (!current || !lease || lease.companyId !== run.companyId || lease.heartbeatRunId !== run.id) {
            throw new Error("workspace_sync_out_unrecoverable");
          }
          await recordWorkspaceFinalize("succeeded", { owningService: "native_workspace_finalizer", storage: "remote" },
            async () => { await saveRemoteWorkspace({ db, workspace: current, runId: run.id, lease,
              target: executionTarget, assertOwned: ownership.assertHeld }); });
        }
        if (nativeWorkspaceSync) {
          const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
            eq(workspaceOperations.companyId, run.companyId),
            eq(workspaceOperations.heartbeatRunId, run.id),
            eq(workspaceOperations.phase, "workspace_finalize"),
            eq(workspaceOperations.status, "succeeded"),
          )).limit(1);
          if (exported.length) adapterFinalizeOutcome = "succeeded";
          else await recordWorkspaceFinalize(hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded", undefined,
            async () => { await measureRunPhase("execution.workspace_finalize", () => restoreNativeWorkspaceBestEffort({
              db, runId: run.id, assertOwnership: ownership?.assertHeld,
              restore: () => nativeWorkspaceSync!.restoreWorkspace(ownership?.assertHeld),
            })); });
        }
        await ownership?.assertHeld();
        await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
          .update(heartbeatRuns)
          .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })
          .where(
            and(
              eq(heartbeatRuns.id, run.id),
              eq(heartbeatRuns.status, "running"),
            ),
          ), { companyId: run.companyId });
        const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded";
        await recordWorkspaceFinalize(workspaceFinalizeStatus);
        if (adapterResult.nativeFinalization) {
          adapterResult.nativeFinalization.workspaceFinalizeStatus =
            workspaceFinalizeStatus;
          try {
            const finalized = await measureRunPhase("execution.native_finalize", () => finalizeNativeRun({
              db,
              runId: run.id,
              workspaceFinalizeStatus,
              preserveProviderAttempt: Boolean(nativeWorkspaceSync || remoteWorkspaceAuthority),
            }));
            await dispatchPendingNativeStatusWakeups({
              companyId: run.companyId,
            });
            if (finalized.phase === "committed") {
              await nativeWorkspaceSync?.cleanup();
            }
          } catch (finalizeErr) {
            logger.warn(
              { err: finalizeErr, runId: run.id },
              "native result persisted but finalization did not apply; the reconciliation loop will retry",
            );
          }
        }
      } catch (error) {
        if (ownership) {
          await ownership.assertHeld();
          await recordWorkspaceFinalize("failed");
        }
        throw error;
      }
    };
    if (nativeWorkspaceSync || remoteWorkspaceAuthority) {
      const owned = await measureRunPhase("execution.workspace_ownership", () => withNativeWorkspaceFinalizationOwnership({
        db, companyId: run.companyId, runId: run.id,
      }, completeWorkspace));
      if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError();
    } else {
      await completeWorkspace();
    }
  } catch (adapterErr) {
    if (adapterErr instanceof NativeCancellationPendingRecoveryError) {
      // Durable cancellation is settled by the outer recovery handler;
      // it does not imply a failed workspace or a persisted run result.
      throw adapterErr;
    }
    if (adapterErr instanceof NativeWorkspaceFinalizationBusyError
      || adapterErr instanceof NativeWorkspaceFinalizationOwnershipLostError) {
      input.effects.onNativeWorkspaceFinalizeScheduled();
      throw adapterErr;
    }
    if (adapterErr instanceof NativeControllerDetachedForRestartError) {
      // Preserve the provider and its run for the new controller. This
      // also keeps generic teardown from terminalizing/releasing its lease.
      input.effects.onNativeSessionResumeScheduled();
      preserveRunGatewayTokens = true;
      throw adapterErr;
    }
    if (adapterErr instanceof NativeControllerRecoveryWaitingError) {
      input.effects.onNativeSessionResumeScheduled();
      throw new NativeSessionResumeScheduledError(adapterErr);
    }
    if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {
      input.effects.onNativeOwnershipHeld();
      throw adapterErr;
    }
    if (nativeRuntimeResolution.kind === "native" && (nativeWorkspaceSync?.reference.state === "prepared" || remoteWorkspaceAuthority?.dirty)
      && !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) {
      input.effects.onProviderResourceDisposition("stop_and_retain");
    }
    try {
      await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
        .update(heartbeatRuns)
        .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })
        .where(
          and(
            eq(heartbeatRuns.id, run.id),
            eq(heartbeatRuns.status, "running"),
          ),
        ), { companyId: run.companyId });
    } catch (writeError) {
      if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr;
      throw writeError;
    }
    if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr;
    if (
      issueRef &&
      context.resumeSessionGoalHeartbeat === true &&
      !runGoalControlRequestId
    ) {
      await blockRunnerGoalRecovery(
        db,
        {
          companyId: run.companyId,
          issueId: issueRef.id,
          agentId: agent.id,
          adapterType: agent.adapterType,
        },
        "provider_session_goal_recovery_failed",
      ).catch(() => undefined);
    }
    if (issueRef && runGoalControlRequestId) {
      await failRunnerGoalAction(
        db,
        {
          companyId: run.companyId,
          issueId: issueRef.id,
          agentId: agent.id,
          adapterType: agent.adapterType,
        },
        runGoalControlRequestId,
        adapterErr instanceof Error
          ? adapterErr.message
          : "session_goal_control_failed",
      ).catch(() => undefined);
    }
    const nativeResumeScheduled =
      nativeRuntimeResolution.kind === "native"
        ? await db
            .select({
              phase: nativeRunFinalizations.phase,
              resultId: nativeRunFinalizations.resultId,
            })
            .from(nativeRunFinalizations)
            .where(eq(nativeRunFinalizations.runId, run.id))
            .limit(1)
            .then(
              (rows) =>
                rows[0]?.phase === "retryable_failure" &&
                rows[0]?.resultId === null,
            )
        : false;
    if (nativeResumeScheduled) {
      input.effects.onNativeSessionResumeScheduled();
      const failedAttachment = describeFailedAttachment(adapterErr instanceof Error ? adapterErr.message : null);
      if (failedAttachment) await Promise.resolve(onLog("stderr", `[paperclip] ${failedAttachment}\n`)).catch(() => undefined);
      throw new NativeSessionResumeScheduledError(adapterErr);
    }
    // Adapter (or its restore finally) threw — or the finalize record
    // write itself threw. Either way the workspace may be in a partial
    // state. Best-effort record finalize=failed so the dependent readiness
    // check keeps the gate closed instead of waking on stale local state,
    // and surface the original error to the caller.
    // The control code can be a fencing label; the closed diagnostic says
    // what actually failed after an accepted result (no provider text).
    const postResultFailure = readNativePostResultFailureDiagnostic(adapterErr);
    // A runner that could not prove suspension after an accepted result left a
    // local workspace untouched: there is nothing to copy back. Settle the
    // accepted result instead of failing the run over runner bookkeeping.
    // Rebuild the accepted result before writing any finalize status: only a
    // proven, complete accepted result may record workspace finalization as
    // succeeded.
    let acceptedResult = nativeRuntimeResolution.kind === "native"
      && postResultFailure?.category === "runner_settlement_incomplete"
      && !nativeWorkspaceSync && !remoteWorkspaceAuthority
      ? await (async () => {
          const resultId = await db.select({ resultId: nativeRunFinalizations.resultId }).from(nativeRunFinalizations)
            .where(eq(nativeRunFinalizations.runId, run.id)).limit(1).then(rows => rows[0]?.resultId ?? null);
          return resultId ? acceptedNativeAdapterResult(db, {
            runId: run.id, companyId: run.companyId, resultId,
            model: nativeExecution?.provider.model ?? null, driverKind: nativeExecution?.session.driverKind ?? "unknown",
            postResultFailure: postResultFailure!,
          }) : null;
        })().catch((readErr: unknown) => {
          logger.warn({ err: readErr, runId: run.id }, "accepted native result could not be read; keeping the failure path");
          return null;
        })
      : null;
    let settleAcceptedResult = acceptedResult !== null;
    try {
      await recordWorkspaceFinalize(settleAcceptedResult ? "succeeded" : "failed", settleAcceptedResult
        ? { runnerSettlementDiagnostic: postResultFailure }
        : {
            errorMessage:
              adapterErr instanceof Error
                ? adapterErr.message
                : String(adapterErr),
            ...(postResultFailure ? { failureDiagnostic: postResultFailure } : {}),
          });
    } catch (recordErr) {
      logger.warn(
        {
          err: recordErr,
          runId: run.id,
          executionWorkspaceId: persistedExecutionWorkspace?.id ?? null,
        },
        "failed to record workspace_finalize=failed operation; dependents may remain gated",
      );
    }
    // Settle only when the workspace finalization record itself succeeded. A
    // failed branch or plugin workspace check keeps the failure path.
    if (adapterFinalizeOutcome !== "succeeded") { settleAcceptedResult = false; acceptedResult = null; }
    if (nativeRuntimeResolution.kind === "native") {
      const proposedResult = await db
        .select({ resultId: nativeRunFinalizations.resultId })
        .from(nativeRunFinalizations)
        .where(eq(nativeRunFinalizations.runId, run.id))
        .limit(1)
        .then((rows) => rows[0]?.resultId ?? null);
      if (proposedResult && (nativeWorkspaceSync || remoteWorkspaceAuthority)) {
        const workspaceFailureMessage =
          adapterErr instanceof Error ? adapterErr.message : "";
        const unrecoverable =
          workspaceFailureMessage ===
            "workspace_sync_out_unrecoverable" ||
          workspaceFailureMessage.includes("daytona_sandbox_not_found");
        const failure = await recordNativeFinalizationFailure({
          db,
          runId: run.id,
          error: new Error(
            unrecoverable
              ? "native_workspace_sync_out_unrecoverable"
              : "native_workspace_sync_out_failed",
          ),
          projectRunStatus: true,
          failureScope: "workspace",
          permanent: unrecoverable,
          failureCause: postResultFailure
            ? `native_post_result_failure: category=${postResultFailure.category}`
            : remoteWorkspaceFailureCause(adapterErr),
        });
        input.effects.onNativeWorkspaceFinalizeScheduled();
        throw new NativeWorkspaceFinalizeScheduledError(
          adapterErr,
          failure.phase === "terminal_failure",
          unrecoverable
            ? "workspace_sync_out_unrecoverable"
            : "workspace_sync_out_failed",
        );
      }
      if (settleAcceptedResult && acceptedResult) {
        // Finalize exactly as a returned result would, then let the ordinary
        // completion path settle the run, session, accounting and agent.
        await onLog("stderr", "[paperclip] runner settlement was incomplete after the result was accepted; finalizing the accepted result\n")
          .catch(() => undefined);
        try {
          await measureRunPhase("execution.native_finalize", () => finalizeNativeRun({
            db, runId: run.id, workspaceFinalizeStatus: "succeeded",
          }));
          await dispatchPendingNativeStatusWakeups({ companyId: run.companyId });
        } catch (finalizeErr) {
          logger.warn(
            { err: finalizeErr, runId: run.id },
            "native result persisted but finalization did not apply; the reconciliation loop will retry",
          );
        }
        return { dispatched: true, adapterResult: acceptedResult };
      }
      try {
        await measureRunPhase("execution.native_finalize", () => finalizeNativeRun({
          db,
          runId: run.id,
          workspaceFinalizeStatus: "failed",
        }));
        await dispatchPendingNativeStatusWakeups({
          companyId: run.companyId,
        });
      } catch (finalizeErr) {
        logger.warn(
          { err: finalizeErr, runId: run.id },
          "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results",
        );
      }
    }
    throw adapterErr;
  } finally {
    try {
      if (!preserveRunGatewayTokens) await revokeHeartbeatRunGatewayTokens({
        db,
        companyId: agent.companyId,
        runId: run.id,
      });
    } catch (revokeErr) {
      logger.warn(
        { err: revokeErr, runId: run.id, companyId: agent.companyId },
        "failed to revoke heartbeat-run MCP gateway tokens",
      );
    }
    await nativeInstructionReservation?.release();
    await withAdapterExecutionPhase(executionPhaseContext, "instruction_cleanup", releaseInstructionCopy);
  }
  return { dispatched: true, adapterResult };
}