From b397329581d06a4e5187c3b47c7ede346a402fa2 Mon Sep 17 00:00:00 2001 From: patrickkenney9801 Date: Thu, 25 Apr 2024 12:11:42 -0500 Subject: [PATCH 1/3] feat: Add tacc install and btl setup scripts --- Makefile | 42 ++++++++++++++++++++++++++------ pando-rt/src/prep/CMakeLists.txt | 2 +- scripts/tacc_build_env.sh | 13 ++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 scripts/tacc_build_env.sh diff --git a/Makefile b/Makefile index 870fc885..7bbc7157 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ DRV_ROOT ?= # Developer variables that should be set as env vars in startup files like .profile PANDO_CONTAINER_MOUNTS ?= PANDO_CONTAINER_ENV ?= +PANDO_TACC_DEPENDENCY_DIR ?= ${WORK}/mono-install .PHONY: docker docker-image-dependencies @@ -154,14 +155,12 @@ apptainer: cmake-mpi: @echo "Must be run from inside the dev Docker container" - @. /dependencies/spack/share/spack/setup-env.sh && \ - spack load gasnet@${GASNET_VERSION}%gcc@${GCC_VERSION} qthreads@${QTHEADS_VERSION}%gcc@${GCC_VERSION} openmpi@${OMPI_VERSION}%gcc@${GCC_VERSION} && \ - cmake \ + @cmake \ -S ${SRC_DIR} \ -B ${BUILD_DIR} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CXX_FLAGS=${PANDO_EXTRA_CXX_FLAGS} \ - -DPANDO_PREP_GASNET_CONDUIT=mpi \ + -DPANDO_PREP_GASNET_CONDUIT=ibv \ -DCMAKE_INSTALL_PREFIX=/opt/pando-lib-galois \ -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \ -DBUILD_WORKFLOWS=${BUILD_WORKFLOWS} \ @@ -172,9 +171,27 @@ cmake-mpi: -DENABLE_BFS_TESTS=${BFS_TESTS} \ -DENABLE_TC_TESTS=${TC_TESTS} \ -DBUILD_DOCS=${PANDO_BUILD_DOCS} \ - -DPANDO_TEST_DISCOVERY_TIMEOUT=${PANDO_TEST_DISCOVERY_TIMEOUT} \ - -DCMAKE_CXX_COMPILER=g++-12 \ - -DCMAKE_C_COMPILER=gcc-12 + -DPANDO_TEST_DISCOVERY_TIMEOUT=${PANDO_TEST_DISCOVERY_TIMEOUT} + +cmake-ibv: + @echo "Must be run from inside the dev Docker container" + @cmake \ + -S ${SRC_DIR} \ + -B ${BUILD_DIR} \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCMAKE_CXX_FLAGS=${PANDO_EXTRA_CXX_FLAGS} \ + -DPANDO_PREP_GASNET_CONDUIT=ibv \ + -DCMAKE_INSTALL_PREFIX=/opt/pando-lib-galois \ + -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \ + -DBUILD_WORKFLOWS=${BUILD_WORKFLOWS} \ + -DBUILD_MICROBENCH=${BUILD_MICROBENCH} \ + -DENABLE_ROOT_TESTS=${ROOT_TESTS} \ + -DENABLE_GALOIS_TESTS=${GALOIS_TESTS} \ + -DENABLE_INFLUENCE_MAX_TESTS=${WF4_TESTS} \ + -DENABLE_BFS_TESTS=${BFS_TESTS} \ + -DENABLE_TC_TESTS=${TC_TESTS} \ + -DBUILD_DOCS=${PANDO_BUILD_DOCS} \ + -DPANDO_TEST_DISCOVERY_TIMEOUT=${PANDO_TEST_DISCOVERY_TIMEOUT} cmake-smp: @echo "Must be run from inside the dev Docker container" @@ -221,6 +238,17 @@ cmake-drv: -DCMAKE_CXX_COMPILER=g++-12 \ -DCMAKE_C_COMPILER=gcc-12 +setup-tacc: + module load gcc/12.2.0 + test -d ${WORK}/gasnet || git clone https://bitbucket.org/berkeleylab/gasnet.git ${WORK}/gasnet -b stable + test -d ${WORK}/qthreads || git clone https://github.com/sandialabs/qthreads.git ${WORK}/qthreads -b 1.19 + cd ${WORK}/qthreads && bash autogen.sh \ + && ./configure --prefix=${PANDO_TACC_DEPENDENCY_DIR} \ + && make -j16 install + cd ${WORK}/gasnet && ./Bootstrap \ + && ./configure --enable-ibv --enable-par --prefix=${PANDO_TACC_DEPENDENCY_DIR} \ + && make -j16 install + setup-ci: cmake-mpi setup: cmake-mpi cmake-smp diff --git a/pando-rt/src/prep/CMakeLists.txt b/pando-rt/src/prep/CMakeLists.txt index 04449d64..ee54e996 100644 --- a/pando-rt/src/prep/CMakeLists.txt +++ b/pando-rt/src/prep/CMakeLists.txt @@ -22,7 +22,7 @@ set(JUST_INSTALL_CEREAL ON CACHE INTERNAL "") FetchContent_MakeAvailable(cereal) # setup GASNet with supported conduits -set(PANDO_PREP_GASNET_CONDUIT_LIST "smp" "mpi") +set(PANDO_PREP_GASNET_CONDUIT_LIST "smp" "mpi" "ibv") set(PANDO_PREP_GASNET_CONDUIT smp CACHE STRING "PREP GASNet conduit") set_property(CACHE PANDO_PREP_GASNET_CONDUIT PROPERTY STRINGS ${PANDO_PREP_GASNET_CONDUIT_LIST}) message(STATUS "PREP GASNet conduit: ${PANDO_PREP_GASNET_CONDUIT}") diff --git a/scripts/tacc_build_env.sh b/scripts/tacc_build_env.sh new file mode 100644 index 00000000..4bffea9a --- /dev/null +++ b/scripts/tacc_build_env.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +module purge +module load gcc/12.2.0 +module load cmake/3.24.2 + +export PANDO_TACC_DEPENDENCY_DIR=$WORK/mono-install +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PANDO_TACC_DEPENDENCY_DIR +export GASNet_ROOT=$PANDO_TACC_DEPENDENCY_DIR +export QTHREADS_ROOT=$PANDO_TACC_DEPENDENCY_DIR +export SRC_DIR=. +export BUILD_DIR=build +export BUILD_MICROBENCH=ON From a99e8e03efbd71c3ee18deaacc69ef8d7c96ca25 Mon Sep 17 00:00:00 2001 From: AdityaAtulTewari Date: Thu, 25 Apr 2024 12:36:32 -0500 Subject: [PATCH 2/3] Instructions and some small changes --- Makefile | 15 +++++++++------ README.md | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 7bbc7157..1261a034 100644 --- a/Makefile +++ b/Makefile @@ -155,12 +155,14 @@ apptainer: cmake-mpi: @echo "Must be run from inside the dev Docker container" - @cmake \ + @. /dependencies/spack/share/spack/setup-env.sh && \ + spack load gasnet@${GASNET_VERSION}%gcc@${GCC_VERSION} qthreads@${QTHEADS_VERSION}%gcc@${GCC_VERSION} openmpi@${OMPI_VERSION}%gcc@${GCC_VERSION} && \ + cmake \ -S ${SRC_DIR} \ -B ${BUILD_DIR} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CXX_FLAGS=${PANDO_EXTRA_CXX_FLAGS} \ - -DPANDO_PREP_GASNET_CONDUIT=ibv \ + -DPANDO_PREP_GASNET_CONDUIT=mpi \ -DCMAKE_INSTALL_PREFIX=/opt/pando-lib-galois \ -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \ -DBUILD_WORKFLOWS=${BUILD_WORKFLOWS} \ @@ -171,17 +173,18 @@ cmake-mpi: -DENABLE_BFS_TESTS=${BFS_TESTS} \ -DENABLE_TC_TESTS=${TC_TESTS} \ -DBUILD_DOCS=${PANDO_BUILD_DOCS} \ - -DPANDO_TEST_DISCOVERY_TIMEOUT=${PANDO_TEST_DISCOVERY_TIMEOUT} + -DPANDO_TEST_DISCOVERY_TIMEOUT=${PANDO_TEST_DISCOVERY_TIMEOUT} \ + -DCMAKE_CXX_COMPILER=g++-12 \ + -DCMAKE_C_COMPILER=gcc-12 -cmake-ibv: - @echo "Must be run from inside the dev Docker container" +cmake-tacc: @cmake \ -S ${SRC_DIR} \ -B ${BUILD_DIR} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CXX_FLAGS=${PANDO_EXTRA_CXX_FLAGS} \ -DPANDO_PREP_GASNET_CONDUIT=ibv \ - -DCMAKE_INSTALL_PREFIX=/opt/pando-lib-galois \ + -DCMAKE_INSTALL_PREFIX=${WORK}/mono-install \ -DBUILD_EXAMPLES=${BUILD_EXAMPLES} \ -DBUILD_WORKFLOWS=${BUILD_WORKFLOWS} \ -DBUILD_MICROBENCH=${BUILD_MICROBENCH} \ diff --git a/README.md b/README.md index fd186de2..fa464759 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repo's `pando-rt` submodule for using root. Before developing make sure you have initialized the `pando-rt` submodule with `git submodule update --init --recursive`. -Quick Setup: +## Docker Setup: ```shell git submodule update --init --recursive @@ -28,16 +28,26 @@ make setup make -C dockerbuild -j8 make run-tests ``` - Developers can run a hello-world smoke test inside their containers by running `bash scripts/run.sh`. -## Running on Drive X +### Running on Drive X Inside the container run `make cmake-drv`. Workloads can then be run via `bash scripts/run-drv.sh`. +## TACC Setup: + +```shell +. ./scripts/tacc_build_env.sh +make setup-tacc +make cmake-tacc +make -C build -j8 +``` + +So far only PReP works on tacc + ## Tools ### [asdf](https://asdf-vm.com) From 41829e072462514e08575ae3e4b99bd710595465 Mon Sep 17 00:00:00 2001 From: AdityaAtulTewari Date: Thu, 25 Apr 2024 17:58:05 +0000 Subject: [PATCH 3/3] Updated for pre-commits --- README.md | 5 +++-- scripts/tacc_build_env.sh | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa464759..3ae491cf 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repo's `pando-rt` submodule for using root. Before developing make sure you have initialized the `pando-rt` submodule with `git submodule update --init --recursive`. -## Docker Setup: +## Docker Setup ```shell git submodule update --init --recursive @@ -28,6 +28,7 @@ make setup make -C dockerbuild -j8 make run-tests ``` + Developers can run a hello-world smoke test inside their containers by running `bash scripts/run.sh`. @@ -37,7 +38,7 @@ Inside the container run `make cmake-drv`. Workloads can then be run via `bash scripts/run-drv.sh`. -## TACC Setup: +## TACC Setup ```shell . ./scripts/tacc_build_env.sh diff --git a/scripts/tacc_build_env.sh b/scripts/tacc_build_env.sh index 4bffea9a..ad963aed 100644 --- a/scripts/tacc_build_env.sh +++ b/scripts/tacc_build_env.sh @@ -1,5 +1,8 @@ #!/bin/bash +# SPDX-License-Identifier: MIT +# Copyright (c) 2023. University of Texas at Austin. All rights reserved. + module purge module load gcc/12.2.0 module load cmake/3.24.2