[BUGFIX] Ubuntu 20.04: support for in-distro Intel MKL libraries (#19766

)

* Improve MxNet framework building on Ubuntu 20.04

Ubuntu 20.04:
  - add support for intel-mkl distribution version;
  - use FindBLAS cmake function to look for the intel-mkl;
  - add FindBLAS.cmake file upstream version (due to Intel10_64_dyn &
    --start-group/--end-gropup options support)
  - remove FindMKL.cmake;
  - preserve Ubuntu 18.04 images for TensorRT pipeline;
  - preserve MKL libraries linking support on non-Ubuntu 20.04 OSes
    (w/o FindMKL.cmake file).

* Temporarily use numpy < 1.20 on CI

ci/docker/Dockerfile.build.ubuntu

@@ -33,11 +33,16 @@ FROM $BASE_IMAGE AS base
 
 WORKDIR /work/deps
 
+SHELL ["/bin/bash", "-c"]
 RUN export DEBIAN_FRONTEND=noninteractive && \
+    export OS_RELEASE="$(cat /etc/os-release)" && \
     apt-get update && \
     apt-get install -y wget software-properties-common && \
-    wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add - && \
-    apt-add-repository "deb https://apt.repos.intel.com/mkl all main" &&  \
+    if [[ ${OS_RELEASE} == *"Bionic"* ]]; then \
+        wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add -; \
+        apt-add-repository "deb https://apt.repos.intel.com/mkl all main"; \
+        INTEL_MKL="-2020.0-088"; \
+    fi && \
     apt-get update && \
     apt-get install -y \
         ## Utilities
@@ -58,7 +63,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         g++ \
         g++-7 \
         g++-8 \
-        intel-mkl-2020.0-088 \
+        intel-mkl${INTEL_MKL} \
         libomp-dev \
         ## Dependencies
         libgomp1 \

ci/docker/install/requirements

@@ -19,7 +19,7 @@
 # the whole docker cache for the image
 
 # Required dependencies
-numpy>=1.17
+numpy<1.20.0
 requests>=2.20.0,<3
 graphviz<0.9.0,>=0.8.1
 contextvars;python_version<"3.7"

cmake/ChooseBlas.cmake

@@ -121,17 +121,49 @@ set(FORTRAN_DIR \\\"\$\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\}\\\")
     endif()
   endif()
 elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
-  if (USE_INT64_TENSOR_SIZE)
-    set(MKL_USE_ILP64 ON CACHE BOOL "enable using ILP64 in MKL" FORCE)
+  # ---[ Root folders
+  set(INTEL_HOME_ROOT "$ENV{HOME}/intel" CACHE PATH "Folder contains user-installed intel libs")
+  set(INTEL_OPT_ROOT "/opt/intel" CACHE PATH "Folder contains root-installed intel libs")
+
+  # ---[ MKL Options
+  if(UNIX)
+    # Single dynamic library interface leads to conflicts between intel omp and llvm omp
+    # https://github.com/apache/incubator-mxnet/issues/17641
+    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
   else()
-    if(MKL_USE_ILP64)
-      message(FATAL_ERROR "MKL_USE_ILP64 cannot be set without USE_INT64_TENSOR_SIZE; "
-              "Please set USE_INT64_TENSOR_SIZE instead of MKL_USE_ILP64.")
+    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
+  endif()
+  cmake_dependent_option(BLA_STATIC "Use static libraries" ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF)
+  cmake_dependent_option(MKL_MULTI_THREADED  "Use multi-threading"  ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF)
+
+  if(MKL_USE_SINGLE_DYNAMIC_LIBRARY)
+    set(BLA_VENDOR Intel10_64_dyn)
+    add_definitions(-DMKL_USE_SINGLE_DYNAMIC_LIBRARY=1)
+  else()
+    if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+      set(BLA_VENDOR Intel10_32)
+    else()
+      if(MKL_MULTI_THREADED)
+        if(USE_INT64_TENSOR_SIZE)
+          set(BLA_VENDOR Intel10_64ilp)
+        else()
+          set(BLA_VENDOR Intel10_64lp)
+        endif()
+      else()
+        if(USE_INT64_TENSOR_SIZE)
+          set(BLA_VENDOR Intel10_64ilp_seq)
+        else()
+          set(BLA_VENDOR Intel10_64lp_seq)
+        endif()
+      endif()
     endif()
   endif()
-  find_package(MKL REQUIRED)
+  # Setting up BLAS_mkl_MKLROOT for non-Ubuntu 20.04 OSes
+  find_path(BLAS_mkl_MKLROOT mkl PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT})
+  find_package(BLAS)
+  find_path(MKL_INCLUDE_DIR mkl.h HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl PATHS ENV MKLROOT PATH_SUFFIXES include mkl REQUIRED)
   include_directories(SYSTEM ${MKL_INCLUDE_DIR})
-  list(APPEND mshadow_LINKER_LIBS ${MKL_LIBRARIES})
+  list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})
   add_definitions(-DMSHADOW_USE_CBLAS=0)
   add_definitions(-DMSHADOW_USE_MKL=1)
   add_definitions(-DMXNET_USE_BLAS_MKL=1)