Skip to content

Using ALCF

Jump to bottom Edit New page
Nichols A. Romero edited this page Apr 3, 2015 · 15 revisions

Table of Contents

Links

Helpful Blue Gene/Q web sites:

Kevin's Stock MTXM code

Whether you are on Blue Gene/P or Blue Gene/Q, you will want to compile Kevin Stock's MTXM code. This must be done outside of the MADNESS build system proper with the XL compiler as GCC does not support double hummer or QPX intrinsic. This is straightforward to accomplish, but the code is buried in a non-intuitive directory. 

cd ${MADNESS_TOP_LEVEL}/src/madness/tensor/new_mtxmq/bests
setenv TARGET BGQ)
make

MPI wrapper scripts

MADNESS uses MPI_THREAD_MULTIPLE mode. In principle, this should work with the legacy MPI wrapper scripts. However, we have discovered some hangs so we recommend using the non-legacy MPI wrapper scripts instead. This is the default wrapper script on the ALCF Blue Gene/Q.

Compiling and Running with the Elemental, a parallel dense linear algebra package

Most moldft calculations for a decent size molecule will require the use of Elemental. Presently, Elemental 0.84 is only supported. 

wget http://libelemental.org/pub/releases/Elemental-0.84-p1.tgz

Elemental support can be activated with the following configure flag: 

--with-elemental=elemental-0.84_path

MADNESS using Elemental MRRR algorithm for solving the Generalized EVP which is part of the LCAO initialization. MRRR algorithm is implemented in an external library which is bundled with Elemental and built by default. MRRR spawns its own set of threads by default which interfere with the MADNESS RT. It is necessary to set the following flag in your job submission script: 

--env PMR_NUM_THREADS=1

Elemental Build Script and Toolchain for Clang + ESSL

Create a build subdirectory. And execute cmake command inside build.sh: 

/soft/buildtools/cmake/current/bin/cmake \
      -D CMAKE_TOOLCHAIN_FILE=../BGQ-Mira-clang-essl.cmake \
      -D ELEM_EXAMPLES=ON -D ELEM_TESTS=ON \
      -D CMAKE_INSTALL_PREFIX=$HOME/public/install/local/elemental-0.84
Where BGQ-Mira-clang-essl.cmake is linked in.

Then run: 

make -j 4 && make install

BGQ-Mira-clang-essl.cmake Toolchain for no threaded version of Elemental 

 #set(CMAKE_SYSTEM_NAME BlueGeneQ-static)
set(CMAKE_BUILD_TYPE "PureRelease")

set(GCC_ROOT  "/bgsys/drivers/ppcfloor/gnu-linux")
set(GCC_NAME  "powerpc64-bgq-linux")
set(CLANG_ROOT "/home/projects/llvm")
set(CLANG_MPI_ROOT "/home/projects/llvm/mpi/bgclang") 
 #set(IBMCMP_ROOT "$ENV{IBM_MAIN_DIR}")
set(IBMCMP_ROOT "/soft/compilers/ibmcmp-feb2014")

set(BLAS_LIB "/soft/libraries/alcf/current/xl/BLAS/lib")
set(LAPACK_LIB "/soft/libraries/alcf/current/xl/LAPACK/lib")
set(ESSL_LIB "/soft/libraries/essl/current/essl/5.1/lib64")

set(MPI_ROOT   "/bgsys/drivers/ppcfloor/comm")
set(PAMI_ROOT  "/bgsys/drivers/ppcfloor/comm/sys")
set(SPI_ROOT   "/bgsys/drivers/ppcfloor/spi")

 #The serial compilers
set(CMAKE_C_COMPILER       "${CLANG_ROOT}/wbin/bgclang")
set(CMAKE_CXX_COMPILER     "${CLANG_ROOT}/wbin/bgclang++11")
set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/${GCC_NAME}-gfortran")

 #The MPI wrappers for the C and C++ compilers
set(MPI_C_COMPILER   "${CLANG_MPI_ROOT}/bin/mpiclang")
set(MPI_CXX_COMPILER "${CLANG_MPI_ROOT}/bin/mpiclang++11")

set(MPI_C_COMPILE_FLAGS    "")
set(MPI_CXX_COMPILE_FLAGS  "")
set(MPI_C_INCLUDE_PATH     "${MPI_ROOT}/include")
set(MPI_CXX_INCLUDE_PATH   "${MPI_ROOT}/include")
set(MPI_C_LINK_FLAGS       "-L${MPI_ROOT}/lib -L${PAMI_ROOT}/lib -L${SPI_ROOT}/lib")
set(MPI_CXX_LINK_FLAGS     "${MPI_C_LINK_FLAGS}")
set(MPI_C_LIBRARIES       "${MPI_C_LINK_FLAGS}   -lmpich-gcc -lopa-gcc -lmpl-gcc -lpami-gcc -lSPI -lSPI_cnk -lrt -lpthread -lstdc++ -lpthread")
set(MPI_CXX_LIBRARIES     "${MPI_CXX_LINK_FLAGS} -lmpichcxx-gcc ${MPI_C_LIBRARIES}")


if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES HybridDebug)
  set(CXX_FLAGS "-g")
else()
  set(CXX_FLAGS "-O3")
endif()

 #set(CMAKE_THREAD_LIBS_INIT "-fopenmp")
 #set(OpenMP_CXX_FLAGS "-fopenmp")

 ##############################################################

 #set the search path for the environment coming with the compiler
 #and a directory where you can install your own compiled software
set(CMAKE_FIND_ROOT_PATH
    /bgsys/drivers/ppcfloor
    /bgsys/drivers/ppcfloor/gnu-linux/powerpc64-bgq-linux
    /bgsys/drivers/ppcfloor/comm/xl
    /bgsys/drivers/ppcfloor/comm/sys
    /bgsys/drivers/ppcfloor/spi
)

 #adjust the default behaviour of the FIND_XXX() commands:
 #search headers and libraries in the target environment, search
 #programs in the host environment
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

 ##############################################################

set(XLF_LIB "${IBMCMP_ROOT}/xlf/bg/14.1/bglib64")
set(XLSMP_LIB "${IBMCMP_ROOT}/xlsmp/bg/3.1/bglib64")
if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES PureRelease)
  set(MATH_LIBS "-L${ESSL_LIB} -lesslbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlomp_ser -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
else()
  set(MATH_LIBS "-L${ESSL_LIB} -lesslsmpbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslsmpbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlsmp -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
endif()

Elemental Build Script and Toolchain for GCC 4.7.2 + ESSL

Both the build script and the toolchain are similar to the case of clang + ESSL. So we only provide the differences here. Because there are no compiler wrappers for GCC 4.7.2, you will need to use the existing GCC 4.4.7 MPI wrapper scripts and override the underlying GCC compilers. This what the first two lines of this batch script below accomplish. Be aware that the absolute path to the GCC 4.7.2 compilers has been known to change with new efixes and driver upgrades.

For the build.sh: 

#!/bin/sh -x
export MPICH_CXX="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-g++"
export MPICH_CC="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-gcc"
/soft/buildtools/cmake/current/bin/cmake \
      -D CMAKE_TOOLCHAIN_FILE=../BGQ-Mira-gnu-essl.cmake \
      -D ELEM_EXAMPLES=ON -D ELEM_TESTS=ON \
      -D CMAKE_INSTALL_PREFIX=$HOME/public/install/local/elemental-0.84

The difference in the BGQ-Mira-gnu-essl.cmake is shown below: 

set(CMAKE_C_COMPILER        ${MPI_ROOT}/bin/mpicc)
set(CMAKE_CXX_COMPILER      ${MPI_ROOT}/bin/mpicxx)
set(CMAKE_Fortran_COMPILER  ${MPI_ROOT}/bin/mpif77)

Building MADNESS

Clang Build Instructions

Environmental variables tested in Cetus, Vesta, and Mira.

Your ~/.soft file must have: 

+mpiwrapper-bgclang
And it must have disabled any other wrappers of MPI, ie, +mpiwrapper-xl.

This is an example of HOWTO compile with Clang+Elemental+LibXC+libMADMXM. 

#!/bin/bash -x
export LANG=C
export IBMCMP_ROOT="/soft/compilers/ibmcmp-aug2014"
export LAPACK_LIB=/soft/libraries/alcf/current/xl/LAPACK/lib
export ESSL_LIB=/soft/libraries/essl/current/essl/5.1/lib64
export XLF_LIB=${IBMCMP_ROOT}/xlf/bg/14.1/bglib64
export XLSMP_LIB=${IBMCMP_ROOT}/xlsmp/bg/3.1/bglib64
export XLMASS_LIB=${IBMCMP_ROOT}/xlmass/bg/7.3/bglib64
export MATH_LIBS="-L${XLMASS_LIB} -lmassv -lmass -L${LAPACK_LIB} -llapack \
                  -L${ESSL_LIB} -lesslbg -L${XLF_LIB} -lxlf90_r \
                  -L${XLSMP_LIB} -lxlsmp -ldl  -lxlopt -lxlfmath -lxl \
                  -Wl,--allow-multiple-definition"

export LIBS="${MATH_LIBS} ${LIBS} -L/home/avazquez/public/install/lib  -lMADMTXM "
export CPPFLAGS="-D"

export CXXFLAGS="${CPPFLAGS} -O2 -std=c++0x "
export CFLAGS="-O2"
export FFLAGS="-O2"

export CLANG_ROOT=/home/projects/llvm
export CLANG_MPI_ROOT=/home/projects/llvm/mpi/bgclang

export CC=${CLANG_ROOT}/wbin/bgclang
export CXX=${CLANG_ROOT}/wbin/bgclang++11
export F77=${GCC_ROOT}/bin/${GCC_NAME}-gfortran
export MPICC=${CLANG_MPI_ROOT}/bin/mpiclang
export MPICXX=${CLANG_MPI_ROOT}/bin/mpiclang++11

./configure \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=GNU \
   --with-elemental="/home/avazquez/public/install/local/elemental-0.84"\
   --with-libxc="/home/avazquez/public/install"\
   --host=powerpc64-bgq-linux \
   --with-fortran-int32

GCC 4.7.2 Build Instructions

At minimum you will need a recent version of the XL compiler and the non-legacy MPI GCC 4.7.2 wrapper scripts in your ~/.soft: 

@ibm-compilers-2014-05
@remove @ibm-compilers-default
+mpiwrapper-gcc

Because there are no compiler wrappers for GCC 4.7.2, you will need to use the existing GCC 4.4.7 MPI wrapper scripts and override the underlying GCC compilers. This what the first two lines of this batch script below accomplish. Be aware that the absolute path to the GCC 4.7.2 compilers has been known to change with new efixes and driver upgrades. 

!/bin/bash
export MPICH_CXX="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-g++"
export MPICH_CC="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-gcc"
export LANG=C

export XL_TOP="${IBM_MAIN_DIR}"

export LIBS="${LIBS} -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack"
export LIBS="${LIBS} -L/home/naromero/madness-git-naromero/src/lib/tensor/new_mtxmq/bests -lMADMTXM"
export LIBS="${LIBS} -L/soft/libraries/essl/5.1.1-1/lib64 -lesslbg"
export LIBS="${LIBS} -L${XL_TOP}/xlmass/bg/7.3/bglib64 -lmass"
export LIBS="${LIBS} -L${XL_TOP}/xlf/bg/14.1/bglib64 -lxlf90_r -lxlopt -lxl -ldl"
export LIBS="${LIBS} -L${XL_TOP}/xlsmp/bg/3.1/bglib64 -lxlsmp "

export NOMALLOC="-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"

 #default
export CXXFLAGS="-mcpu=a2 -O3 -std=c++11 ${NOMALLOC}"
export CFLAGS="-mcpu=a2 -O3 ${NOMALLOC}"
export FFLAGS="-mcpu=a2 -O3"

export MPICC=mpicc
export MPICXX=mpicxx
export CC=mpicc
export CXX=mpicxx
export F77=mpif77
TBB_HOME=tbb43_20150316oss
../madness-git-naromero/configure \
   --with-google-test=/home/naromero/gtest-1.6.0-GCC4.7.2 \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=GNU \
   --host=powerpc64-bgq-linux \
   --with-fortran-int32 \
   --with-libxc="/home/naromero/libxc-2.0.x/bin" \
   --with-elemental=/home/naromero/elemental-0.84-gcc \
   --with-google-perf=/home/naromero/gperftools-2.2.1 \
   --with-tbb=yes \
   --with-tbb-lib=/home/naromero/${TBB_HOME}/build/linux_bg_gcc_4.7.2_release \
   --with-tbb-include=/home/naromero/${TBB_HOME}/include

Running with Intel TBB

This is now robustly tested. Please report any errors to the MADNESS mailing list.

You will need Intel TBB 4.3 Update 4: 

wget --no-check-certificate https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb43_20150316oss_src.tgz

Older version will not compile 'out of the box'.

Build it: 

make extra_inc=big_iron.inc arch=bg

Append the following configure options: 

   --with-tbb=yes \
   --with-tbb-include=/home/naromero/tbb41_20130314oss/include \
   --with-tbb-lib=/home/naromero/tbb41_20130314oss/build/linux_bg_gcc_cc4.4.6_libc2.12_kernel2.6.32_debug

Build as you would normally. Note that you will be limited to running on 1 rank per node (c1 mode).

Add a custom footer
Add a custom sidebar
Clone this wiki locally