diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index 2f9ed1556..4e3c2e8b5 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -80,7 +80,6 @@ blueos_cuda_11_gcc_spectrum_build: blueos_cuda_11_gcc_spectrum_test: extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] needs: [blueos_cuda_11_gcc_spectrum_build] - allow_failure: true blueos_cuda_11_gcc_spectrum_cleanup: extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir] diff --git a/.gitmodules b/.gitmodules index d24489245..107101d1a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,4 @@ path = extern/chai url = https://github.com/llnl/chai branch = feature/ManagedSharedPtr + ignore = all diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index df68e6781..bcf4141d2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -15,6 +15,9 @@ Notable changes include: * TPL builds have been split off into a separate Gitlab CI stage to help with timeouts on allocations. * Failed ATS runs are automatically retested once in the Gitlab CI. * Python execute command is centralized in scripts/spheralutils.py now. + * Caliper updated v2.11. + * Adiak added as TPL. + * Created singleton wrapper for cali::ConfigManger and python wrapped Caliper timer and Adiak routines. * Build changes / improvements: * Distributed source directory must always be built now. @@ -23,6 +26,8 @@ Notable changes include: * The FSISPH package is now optional (SPHERAL\_ENABLE\_FSISPH). * The GSPH package is now optional (SPHERAL\_ENABLE\_GSPH). * The SVPH package is now optional (SPHERAL\_ENABLE\_SVPH). + * Cleaner Spheral Spack package. + * ENABLE\_DEV\_BUILD can now export targets properly. * Bug Fixes / improvements: * Wrappers for MPI calls are simplified and improved. diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index ee5317486..9055a0c6f 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -67,9 +67,15 @@ endif() # Find pre-compiled TPLs #----------------------------------------------------------------------------------- +# Any targets that used find package must be added to these lists +set(SPHERAL_FP_TPLS ) +set(SPHERAL_FP_DIRS ) + # Use find_package to get axom (which brings in fmt) and patch fmt find_package(axom REQUIRED NO_DEFAULT_PATH PATHS ${axom_DIR}/lib/cmake) list(APPEND SPHERAL_BLT_DEPENDS axom ) +list(APPEND SPHERAL_FP_TPLS axom) +list(APPEND SPHERAL_FP_DIRS ${axom_DIR}/lib/cmake) # This is a hack to handle transitive issues that come # from using object libraries with newer version of axom @@ -82,6 +88,31 @@ foreach(_comp ${AXOM_COMPONENTS_ENABLED}) list(APPEND SPHERAL_BLT_DEPENDS ${axom_deps}) endforeach() +message("-----------------------------------------------------------------------------") +# Use find_package to get adiak +find_package(adiak REQUIRED NO_DEFAULT_PATH PATHS ${adiak_DIR}/lib/cmake/adiak) +if(adiak_FOUND) + list(APPEND SPHERAL_BLT_DEPENDS adiak::adiak) + list(APPEND SPHERAL_FP_TPLS adiak) + list(APPEND SPHERAL_FP_DIRS ${adiak_DIR}) + message("Found Adiak External Package") +endif() +message("-----------------------------------------------------------------------------") +# Use find_package to get caliper +if (ENABLE_TIMER) + # Save caliper_DIR because it gets overwritten by find_package + if(NOT CONFIG_CALIPER_DIR) + # Only save if it does not exists already + set(CONFIG_CALIPER_DIR "${caliper_DIR}" CACHE PATH "Configuration Caliper directory") + endif() + find_package(caliper REQUIRED NO_DEFAULT_PATH PATHS ${caliper_DIR}/share/cmake/caliper) + if(caliper_FOUND) + list(APPEND SPHERAL_BLT_DEPENDS caliper) + list(APPEND SPHERAL_FP_TPLS caliper) + list(APPEND SPHERAL_FP_DIRS ${caliper_DIR}) + message("Found Caliper External Package") + endif() +endif() message("-----------------------------------------------------------------------------") find_package(RAJA REQUIRED NO_DEFAULT_PATH PATHS ${raja_DIR}) if (RAJA_FOUND) @@ -100,6 +131,8 @@ if(chai_DIR AND USE_EXTERNAL_CHAI) if (chai_FOUND) message("Found chai External Package.") endif() + list(APPEND SPHERAL_FP_TPLS chai) + list(APPEND SPHERAL_FP_DIRS ${chai_DIR}) else() message("Using chai Submodule.") set(chai_DIR "${SPHERAL_ROOT_DIR}/extern/chai") @@ -108,6 +141,10 @@ else() endif() list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) +list(APPEND SPHERAL_FP_TPLS RAJA umpire) +list(APPEND SPHERAL_FP_DIRS ${raja_DIR} ${umpire_DIR}) +set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) +set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) message("-----------------------------------------------------------------------------") @@ -116,7 +153,6 @@ list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo hdf5 polytope) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS aneos IF ENABLE_ANEOS) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS opensubdiv IF ENABLE_OPENSUBDIV) -blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS caliper IF ENABLE_TIMER) # Create and install target library for each external library foreach(lib ${SPHERAL_EXTERN_LIBS}) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index de1024cf2..d93c4fd41 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -153,7 +153,7 @@ endif() # Build C++ tests and install tests to install directory #------------------------------------------------------------------------------- if (ENABLE_TESTS) - add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit/CXXTests) + add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit) # A macro to preserve directory structure when installing files macro(install_with_directory) @@ -189,6 +189,4 @@ if (ENABLE_TESTS) DESTINATION ${SPHERAL_TEST_INSTALL_PREFIX}) endif() -if(NOT ENABLE_DEV_BUILD) - include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake) -endif() +include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake) diff --git a/cmake/spheral/SpheralAddLibs.cmake b/cmake/spheral/SpheralAddLibs.cmake index 6e8b9e545..2d3fca6d7 100644 --- a/cmake/spheral/SpheralAddLibs.cmake +++ b/cmake/spheral/SpheralAddLibs.cmake @@ -46,11 +46,6 @@ function(spheral_add_obj_library package_name obj_list_name) # Install the headers install(FILES ${${package_name}_headers} DESTINATION include/${package_name}) - - if(ENABLE_DEV_BUILD) - install(TARGETS Spheral_${package_name} - DESTINATION lib) - endif() # Append Spheral_${package_name} to the global object list # For example, SPHERAL_OBJ_LIBS or LLNLSPHERAL_OBJ_LIBS set_property(GLOBAL APPEND PROPERTY ${obj_list_name} Spheral_${package_name}) @@ -71,7 +66,7 @@ endfunction() # ---------------------- # INPUT-OUTPUT VARIABLES # ---------------------- -# package_name : REQUIRED : Desired package name +# package_name : REQUIRED : Desired package name (either CXX or LLNLCXX) # _cxx_obj_list : REQUIRED : List of internal targets to include # ----------------------- # OUTPUT VARIABLES TO USE - Made available implicitly after function call @@ -85,28 +80,29 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) get_property(SPHERAL_CXX_DEPENDS GLOBAL PROPERTY SPHERAL_CXX_DEPENDS) # For including files in submodules, currently unused get_property(SPHERAL_SUBMOD_INCLUDES GLOBAL PROPERTY SPHERAL_SUBMOD_INCLUDES) + # Convert package name to lower-case for export target name + string(TOLOWER ${package_name} lower_case_package) + set(export_target_name spheral_${lower_case_package}-targets) - if(ENABLE_SHARED) - # Build shared spheral C++ library - blt_add_library(NAME Spheral_${package_name} - HEADERS ${${package_name}_headers} - SOURCES ${${package_name}_sources} - DEPENDS_ON ${_cxx_obj_list} ${SPHERAL_CXX_DEPENDS} ${SPHERAL_BLT_DEPENDS} - SHARED TRUE) + if(ENABLE_DEV_BUILD) + install(TARGETS ${_cxx_obj_list} + EXPORT ${export_target_name} + DESTINATION lib) + add_library(Spheral_${package_name} INTERFACE) + target_link_libraries(Spheral_${package_name} INTERFACE ${_cxx_obj_list}) else() - # Build static spheral C++ library + # Build static or shared spheral C++ library blt_add_library(NAME Spheral_${package_name} HEADERS ${${package_name}_headers} SOURCES ${${package_name}_sources} DEPENDS_ON ${_cxx_obj_list} ${SPHERAL_CXX_DEPENDS} ${SPHERAL_BLT_DEPENDS} - SHARED FALSE) + SHARED ${ENABLE_SHARED}) endif() target_include_directories(Spheral_${package_name} SYSTEM PRIVATE ${SPHERAL_SUBMOD_INCLUDES}) if(ENABLE_CUDA) set_target_properties(Spheral_${package_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) endif() - ## This cleans up library targets created with object libs. It is turned off as it triggers ## a failure on Werror and pedantic builds. #set(_properties COMPILE_DEFINITIONS LINK_LIBRARIES LINK_OPTIONS INTERFACE_LINK_OPTIONS COMPILE_OPTIONS INTERFACE_COMPILE_OPTIONS) @@ -118,15 +114,12 @@ function(spheral_add_cxx_library package_name _cxx_obj_list) #set_target_properties(Spheral_${package_name} PROPERTIES INTERFACE_LINK_LIBRARIES "") - # Convert package name to lower-case for export target name - string(TOLOWER ${package_name} lower_case_package) - # Install Spheral C++ target and set it as an exportable CMake target install(TARGETS Spheral_${package_name} DESTINATION lib - EXPORT spheral_${lower_case_package}-targets) + EXPORT ${export_target_name}) # Export Spheral target - install(EXPORT spheral_${lower_case_package}-targets DESTINATION lib/cmake) + install(EXPORT ${export_target_name} DESTINATION lib/cmake) # Set the r-path of the C++ lib such that it is independent of the build dir when installed set_target_properties(Spheral_${package_name} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") diff --git a/cmake/spheral/SpheralInstallPythonFiles.cmake b/cmake/spheral/SpheralInstallPythonFiles.cmake index 01f90b749..9294f8fd9 100644 --- a/cmake/spheral/SpheralInstallPythonFiles.cmake +++ b/cmake/spheral/SpheralInstallPythonFiles.cmake @@ -14,7 +14,7 @@ function(spheral_install_python_files) install(FILES ${ARGV} DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral) install(CODE "execute_process( \ - COMMAND ${PYTHON_EXE} -m compileall DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral \ + COMMAND ${PYTHON_EXE} -m compileall ${SPHERAL_SITE_PACKAGES_PATH}/Spheral \ WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX})") endif() diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index daf35fdaa..2781c4895 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -23,10 +23,14 @@ if(NOT SPHERAL_FOUND) find_package(Python3 COMPONENTS Interpreter Development) set(PYTHON_EXE ${Python3_EXECUTABLE}) endif() - - if(NOT TARGET axom) - find_package(axom REQUIRED QUIET NO_DEFAULT_PATH PATHS ${axom_DIR} ${axom_DIR}/lib ${axom_DIR}/lib/cmake) - endif() + # Loop over TPLs that use find_package + set(SPHERAL_FP_TPLS "@SPHERAL_FP_TPLS@") + set(SPHERAL_FP_DIRS "@SPHERAL_FP_DIRS@") + foreach(tpl dir IN ZIP_LISTS SPHERAL_FP_TPLS SPHERAL_FP_DIRS) + if(NOT TARGET ${tpl}) + find_package(${tpl} REQUIRED QUIET NO_DEFAULT_PATH PATHS ${dir}) + endif() + endforeach() if(NOT TARGET chai) if (@USE_EXTERNAL_CHAI@) set(SPHERAL_CHAI_DIR "@chai_DIR@/lib/cmake/chai") @@ -47,7 +51,6 @@ if(NOT SPHERAL_FOUND) endif() endif() include("${SPHERAL_CXX_INSTALL_PREFIX}/lib/cmake/spheral_cxx-targets.cmake") - set_property(TARGET Spheral_CXX APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SPHERAL_CXX_INCLUDE_DIRS}) diff --git a/cmake/tpl/caliper.cmake b/cmake/tpl/caliper.cmake deleted file mode 100644 index 22a8e98e4..000000000 --- a/cmake/tpl/caliper.cmake +++ /dev/null @@ -1 +0,0 @@ -set(${lib_name}_libs libcaliper.a) diff --git a/docs/build_guide/include/appendecies/cmake_config.rst.inc b/docs/build_guide/include/appendecies/cmake_config.rst.inc index bcf0e8f60..4fb343fa6 100644 --- a/docs/build_guide/include/appendecies/cmake_config.rst.inc +++ b/docs/build_guide/include/appendecies/cmake_config.rst.inc @@ -28,6 +28,8 @@ For just the C++ compiled Spheral a number of TPLs are required: - Polyclipper - Conduit - Axom +- Adiak +- Caliper There are also a number of libraries / python packages that are required for compiling the python bindings and executing Spheral at runtime: @@ -66,58 +68,58 @@ OpenMP and MPI support is handled through BLT. Use the option flags ``-DENABLE_ CMake variables -------------------- -In this section we list the CMake variables that can be tweaked for a Spheral build. Where appropriate the options are listed, with the default value in *italics*. +In this section we list the CMake variables that can be tweaked for a Spheral build. Where appropriate the options are listed, with the default value in **bold**. -``CMAKE_BUILD_TYPE`` (Debug, *Release*, RelWithDebInfo, MinSizeRel) +``CMAKE_BUILD_TYPE`` (Debug, **Release**, RelWithDebInfo, MinSizeRel) Choose the type of build -- for more information see the `CMake documentation `_. ``CMAKE_INSTALL_PREFIX`` The top-level path for installing Spheral include files, libraries, and any Python modules or documentation. -``ENABLE_CXXONLY`` (On, *Off*) +``ENABLE_CXXONLY`` (On, **Off**) Do not build python wrappers for Spheral. -``ENABLE_STATIC_CXXONLY`` (On, *Off*) +``ENABLE_STATIC_CXXONLY`` (On, **Off**) Do not build python wrappers for Spheral. Build and export static library files for Spheral. -``ENABLE_SHARED`` (*On*, Off) +``ENABLE_SHARED`` (**On**, Off) Build Spheral C++ libraries as shared libraries. -``ENABLE_DEV_BUILD`` (On, *Off*) +``ENABLE_DEV_BUILD`` (On, **Off**) Builds separate internal C++ libraries for faster code development. ``_DIR`` Directory of previously built TPL. -``ENABLE_STATIC_TPL`` (On, *Off*) +``ENABLE_STATIC_TPL`` (On, **Off**) Link static libraries instead of shared for HDF5 and Conduit. -``ENABLE_OPENMP`` (*On*, Off) +``ENABLE_OPENMP`` (**On**, Off) Support for OpenMP. -``ENABLE_MPI`` (*On*, Off) +``ENABLE_MPI`` (**On**, Off) Support for MPI. -``ENABLE_1D`` (*On*, Off) +``ENABLE_1D`` (**On**, Off) Build Spheral with 1D support. -``ENABLE_2D`` (*On*, Off) +``ENABLE_2D`` (**On**, Off) Build Spheral with 2D support. -``ENABLE_3D`` (*On*, Off) +``ENABLE_3D`` (**On**, Off) Build Spheral with 3D support. -``ENABLE_ANEOS`` (*On*, Off) +``ENABLE_ANEOS`` (**On**, Off) Install the ANEOS (Analytics Equation of State) package along with the Spheral interface to it. This is a legacy equation of state frequently used for geophysical materials. See descriptions in the `iSALE `_ documentation. -``ENABLE_HELMHOLTZ`` (*On*, Off) +``ENABLE_HELMHOLTZ`` (**On**, Off) Compile the included Helmholtz equation of state, typically used in astrophysical calculations. See a discussion `here `_. -``ENABLE_OPENSUBDIV`` (*On*, Off) +``ENABLE_OPENSUBDIV`` (**On**, Off) Install the Opensubdiv library along with the Spheral interface to it. Opensubdiv is a `Pixar provided library `_, which Spheral uses to implement refinement of polyhedra for some specialized problem generation capabilities. -``ENABLE_TIMER`` (*On*, Off) - Enable timer information from Spheral. +``ENABLE_TIMER`` (On, **Off**) + Enable Caliper timer information for Spheral. ``DBC_MODE`` (None, All, Pre) Set the compile time design by contract (DBC) mode for Spheral. Design by contract statements are very useful developer tools, whereby the developer can insert tests in the code as they write it. These statements are both useful for tracking down bugs with fine-grained testing throughout the code, as well as useful documentation in the code about what sort of conditions are expected to hold. @@ -138,23 +140,23 @@ In this section we list the CMake variables that can be tweaked for a Spheral bu It is worth noting ``DBC_MODE=All`` is quite expensive at run time (of order 4x more), so this is not intended to be active for a release/production compilation of Spheral. -``ENABLE_WARNINGS`` (On, *Off*) +``ENABLE_WARNINGS`` (On, **Off**) Enable compiler warnings. -``ENABLE_BOUNDCHECKING`` (On, *Off*) +``ENABLE_BOUNDCHECKING`` (On, **Off**) If building with the Gnu compilers enable STL bound checking by passing -D_GLIBCXX_DEBUG=1 to the compiler. Note, this is a very expensive option at runtime! -``ENABLE_NAN_EXCEPTIONS`` (On, *Off*) +``ENABLE_NAN_EXCEPTIONS`` (On, **Off**) Raise exceptions in the C++ code when floating-point exceptions occur. Gnu compilers only. -``ENABLE_DOCS`` (On, *Off*) +``ENABLE_DOCS`` (On, **Off**) Choose whether or not to build this documentation. ``SPHINX_EXECUTABLE`` Specify where the Sphinx executable is that should be used to build documentation. If not given, assumes the Spheral built Sphinx will be used. -``SPHINX_THEME`` (*sphinx_rtd_theme*) +``SPHINX_THEME`` (**sphinx_rtd_theme**) Give the Sphinx theme to use when generating documentation. Default based on read the docs theme. ``SPHINX_THEME_DIR`` diff --git a/docs/conf.py b/docs/conf.py index df9b1060e..a8619454d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ # -- Project information ----------------------------------------------------- project = 'Spheral' -copyright = '2012, LLNS' +copyright = '2024, LLNS' author = 'J. Michael Owen' # The short X.Y version diff --git a/docs/conf.py.in b/docs/conf.py.in index b7d23f29a..2f7bd11ad 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -22,7 +22,7 @@ import sphinx_rtd_theme # -- Project information ----------------------------------------------------- project = 'Spheral' -copyright = '2012, LLNS' +copyright = '2024, LLNS' author = 'J. Michael Owen' # The short X.Y version diff --git a/docs/developer/dev/diagnostic_tools.rst b/docs/developer/dev/diagnostic_tools.rst index 07d4d2265..f79cc14b2 100644 --- a/docs/developer/dev/diagnostic_tools.rst +++ b/docs/developer/dev/diagnostic_tools.rst @@ -1,24 +1,46 @@ -Diagnostics -########### +Code Performance Diagnostics +############################ -Spheral uses Caliper to preform diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ENABLE_TIMER=On. Otherwise the diagnostic regions are no-ops for improved preformance. +Spheral uses Caliper to preform code diagnostics, such as timing. To enable this functionality in the code, Spheral needs to be configured with ``ENABLE_TIMER=ON``. Otherwise, the timing regions are no-ops for improved preformance. :: - ./scripts/devtools/host-config-build.py -.cmake -DENABLE_TIMER=On + ./scripts/devtools/host-config-build.py -.cmake -DENABLE_TIMER=ON Querying using Caliper ====================== -By defualt, even when configured with ENABLE_TIMER=On, there is no information being recorded. Caliper uses command line options to report data, the simplest of which is ``CALI_CONFIG=runtime-report`` which reports the timing for the regions and prints them out in the terminal. For example: +Caliper is configured and started through the ``cali::ConfigManager``. +The ``cali::ConfigManager`` is wrapped in a ``TimerMgr`` singleton class, which has a python interface. + +.. note:: + ``TimerMgr`` is initialized and started during ``commandLine()`` in ``src/SimulationControl/SpheralOptionParser.py``. This is because ``commandLine()`` is almost always invoked directly near the start of a problem. However, if ``commandLine()`` is not called, the timers would need to be configured and started directly using the ``TimerMgr`` class. See :ref:`below ` for more details. + +By default, the Caliper configuration is set to ``spot`` and outputs Caliper files (``.cali``). +For the default configuration, the Caliper files are named based on what file is being run, for example: +:: + + python Noh-cylindrical-2d.py + +will produce a timing file called ``Noh-cylindrical-2d_YEAR_MONTH_DATE_TIME.cali`` where the file name includes the current date and time. + +The Caliper file name can be specified using the command line :: - CALI_CONFIG=runtime-report python Noh-cylindrical-2d.py + python Noh-cylindrical-2d.py --caliperFilename 'new_test_name.cali' + +Different Caliper configurations can be set at the command line using ``--caliperConfig`` like so +:: + + python Noh-cylindrical-2d.py --caliperConfig 'runtime-report(output=time.txt),calc.inclusive,region.count' .. note:: - To obtain a similar result to that of the removed Spheral::Timer use :kbd:`CALI_CONFIG=runtime\-report(output=time.txt),calc.inclusive,region.count` this will result in a file named time.txt with cumulative times for the nested regions as well as a count of how many times each region ran. + The above configuration produces timing results similar to the previous ``Spheral::Timer`` method. This results in a file named ``time.txt`` with cumulative times for the nested regions as well as a count of how many times each region ran. + +Similarly, a non-default Caliper configuration can be read in from a JSON file using ``--caliperConfigJSON`` and providing the file name. +Lastly, Caliper timers can be turned off using ``--caliperConfig none``. -There are many different options that can be used with ``CALI_CONFIG`` to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorperated into Spheral in the future: +There are many different Caliper configurations to view various information. Here are some extra links for those who want to read or experiment with other features in Caliper that can be incorporated into Spheral: * `Configuration basics `_ * `Builtin Configuration `_ @@ -26,16 +48,48 @@ There are many different options that can be used with ``CALI_CONFIG`` to view v * `Output Format `_ -Adding Regions -============== +Adding Region Timers in C++ +=========================== So far there are two different types of regions in Spheral, using the following macros: :: TIME_FUNCTION + +or + +:: + TIME_BEGIN("timer_name") TIME_END("timer_name") + - ``TIME_FUNCTION`` can be added to the very beginning of a function and creates a region for the entire function using the function's name. ``TIME_FUNCTION`` uses just the function name and no class or parameter information, so be careful when using this method with functions that could share names. -- ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case timer_name) as the name. +- ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case ``timer_name``) as the name. + + +Adding Region Timers in Python +============================== + +Region timers can be added inside the python code using the following function calls: +:: + + TimerMgr.timer_start("timer_name") + some_function_call() + TimerMgr.timer_end("timer_name") + +.. note:: + IMPORTANT: All timers must have both a start and end call. Otherwise, memory issues will occur. + +.. _manual_caliper: + +Starting Caliper Manually +======================== + +As mentioned above, Caliper (not an individual Caliper timer) is normally configured and started in ``commandLine()`` python routine. However, Caliper can be directly configured and started through the python interface, if desired. This can be done by putting the following into the python file: +:: + + caliper_config = "some_configuration(output=some_filename.txt)" + TimerMgr.add(caliper_config) + TimerMgr.start() diff --git a/docs/intro/introduction.rst.inc b/docs/intro/introduction.rst.inc index 1de8df747..47211f5d1 100644 --- a/docs/intro/introduction.rst.inc +++ b/docs/intro/introduction.rst.inc @@ -25,7 +25,7 @@ Some useful features are Release and License: #################### -Copyright (c) 2012, Lawrence Livermore National Security, LLC. +Copyright (c) 2024, Lawrence Livermore National Security, LLC. Produced at the Lawrence Livermore National Laboratory Written by J. Michael Owen mikeowen@llnl.gov LLNL-CODE-561852 diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 2fc646513..aac9e6fd1 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -54,7 +54,12 @@ if (NOT ENABLE_CXXONLY) configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/lcatstest.in" "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" - ) + ) + + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/performance/performance.py.in" + "${CMAKE_CURRENT_BINARY_DIR}/performance/performance.py" + ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/spheral-setup-venv.sh" @@ -63,7 +68,12 @@ if (NOT ENABLE_CXXONLY) "${CMAKE_CURRENT_BINARY_DIR}/lcatstest.sh" "${CMAKE_CURRENT_SOURCE_DIR}/lc/lcats" DESTINATION "${CMAKE_INSTALL_PREFIX}/scripts" - ) + ) + + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/performance/performance.py" + DESTINATION "${CMAKE_INSTALL_PREFIX}/tests" + ) install(CODE "execute_process( \ COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/spheral-setup-venv.sh \ diff --git a/scripts/devtools/valgrind_python_suppression b/scripts/devtools/valgrind_python_suppression new file mode 100644 index 000000000..619a5f106 --- /dev/null +++ b/scripts/devtools/valgrind_python_suppression @@ -0,0 +1,163 @@ +# ************************************************************************** +# Tool Gear (www.llnl.gov/CASC/tool_gear) +# Version 2.12 Jan 14, 2009 +# Please see COPYRIGHT AND LICENSE information at the end of this file. +# ************************************************************************** +# Based on python supplied valgrind suppression file, more informative +# suppression name and tweaked for x86-64 messages. -JCG 12/8/2009 +{ + _PyObject_Free ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Free +} + +{ + _PyObject_Free ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Free +} +{ + _PyObject_Free Use of uninitialised value of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Free +} + +{ + _PyObject_Free ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + ... + fun:_PyObject_Free +} +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Realloc +} + +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Realloc +} +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Realloc +} + +{ + _PyObject_Realloc ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + ... + fun:_PyObject_Realloc +} +{ + _PyObject_Malloc Leak + Memcheck:Leak + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 4 + Memcheck:Addr4 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 4 + Memcheck:Value4 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid read of size 8 + Memcheck:Value8 + ... + fun:_PyObject_Malloc +} +{ + _PyObject_Malloc Invalid jump condition + Memcheck:Cond + ... + fun:_PyObject_Malloc +} +{ + strdup Leak + Memcheck:Leak + ... + fun:strdup +} +{ + _PyFunction_VectorCall Leak + Memcheck:Leak + ... + fun:_PyFunction_Vectorcall +} +################################################################################ +## COPYRIGHT AND LICENSE +## +## Copyright (c) 2012, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory +## Written by John Gyllenhaal (gyllen@llnl.gov), John May (johnmay@llnl.gov), +## and Martin Schulz (schulz6@llnl.gov). +## LLNL-CODE-582512 +## OCEC-12-057 +## All rights reserved. +## This file is part of Tool Gear. +## For details, see computation.llnl.gov/casc/tool_gear. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright notice, +## this list of conditions and the disclaimer below. +## +## * Redistributions in binary form must reproduce the above copyright notice, +## this list of conditions and the disclaimer (as noted below) in the +## documentation and/or other materials provided with the distribution. +## +## * Neither the name of the LLNS/LLNL nor the names of its contributors may +## be used to endorse or promote products derived from this software without +## specific prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +## THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +## PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL +## SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE +## FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +## DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +## CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +## OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +## USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## +## Additional BSD Notice +## +## 1. This notice is required to be provided under our contract with the +## U.S. Department of Energy (DOE). This work was produced at +## Lawrence Livermore National Laboratory under Contract +## No. DE-AC52-07NA27344 with the DOE. +## +## 2. Neither the United States Government nor Lawrence Livermore National +## Security, LLC nor any of their employees, makes any warranty, express +## or implied, or assumes any liability or responsibility for the accuracy, +## completeness, or usefulness of any information, apparatus, product, or +## process disclosed, or represents that its use would not infringe +## privately-owned rights. +## +## 3. Also, reference herein to any specific commercial products, process, or +## services by trade name, trademark, manufacturer or otherwise does not +## necessarily constitute or imply its endorsement, recommendation, or +## favoring by the United States Government or Lawrence Livermore National +## Security, LLC. The views and opinions of authors expressed herein do not +## necessarily state or reflect those of the United States Government or +## Lawrence Livermore National Security, LLC, and shall not be used for +## advertising or product endorsement purposes. +################################################################################# diff --git a/scripts/lc/lcats b/scripts/lc/lcats index 83ea7145b..eb16e4171 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -642,9 +642,6 @@ parser.add_option("--wcid", action="store", type="string", metavar="WC-ID to ass #default = machineSettings.options.bank, help = "HERT WC-ID to use for batch job.") -parser.add_option( "--threaded", action="store_true", dest="threaded", - help = "Run threaded tests. NOTE ATS must run these on login node (no salloc). ") - parser.add_option( "--nogpu", action="store_true", dest="nogpu", help = "For blueos. Filters out gpu test. Used in conjunction with threaded option.") @@ -814,18 +811,6 @@ ezatsArgs= ['addOp', 'batch', 'interactive', 'name', 'allocTime', 'atsExe', 'mac # Add glue arg to pass unique file system test path to ats toAdd= """ --glue='testpath=str("%s")' """ % options.testpath -# Add threaded arg to filter for threaded or non threaded tests. Otherwise all versions will run, and threaded will be run incorrectly -if options.threaded: - if options.nogpu: - toAdd += """ --filter="'nt' in locals()" --filter="'ngpu' not in locals()" """ - elif options.gpuonly: - toAdd += """ --filter="'nt' in locals()" --filter="'ngpu' in locals()" """ - # This version only works if we're using os.system - execv call fails because filter is split incorrectly - else: - toAdd += """ --filter="'nt' in locals()" """ -else: - toAdd += """ --filter="'nt' not in locals()" """ - if options.sanitize: toAdd += """ --filter="sanitize==1" """ @@ -982,7 +967,7 @@ else: + finalCommandToRun # + " -p " + machineSettings.options.partition + " " # Threaded tests under ats should NOT use salloc - elif not options.threaded and 'blue' not in os.environ['SYS_TYPE']: + elif 'blue' not in os.environ['SYS_TYPE']: finalCommandToRun= "salloc --exclusive " \ + " " + allocTime \ + HERT_WC_ID \ diff --git a/scripts/performance/performance.py.in b/scripts/performance/performance.py.in new file mode 100644 index 000000000..b337e5042 --- /dev/null +++ b/scripts/performance/performance.py.in @@ -0,0 +1,18 @@ +#!/user/bin/env python3 + +import sys, os +caliper_loc = "@CONFIG_CALIPER_DIR@" +sys.path.append(os.path.join(caliper_loc, "lib64/caliper")) + +import caliperreader as cr + +# Put some filler functions here +def compare_times(manager): + filtered = [test for test in manager.testlist if test.status is PASSED] + for t in filtered: + print(t) + +onExit(compare_times) +glue(keep=True) +source("functional/Hydro/Noh/Noh-cylindrical-2d.py") + diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 9488e55ad..8695500ea 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -14,7 +14,7 @@ class Spheral(CachedCMakePackage, CudaPackage): git = "https://github.com/llnl/spheral.git" tags = ['radiuss', 'simulations', 'hydrodynamics'] - maintainers = ['mdavis36','jmikeowen'] + maintainers = ['mdavis36','jmikeowen','owen32'] # ------------------------------------------------------------------------- # VERSIONS @@ -43,29 +43,32 @@ class Spheral(CachedCMakePackage, CudaPackage): depends_on('qhull@2020.2 +pic', type='build') depends_on('m-aneos@1.0') depends_on('eigen@3.4.0', type='build') - depends_on('hdf5@1.8.19 ~mpi +hl', type='build', when='~mpi') - depends_on('hdf5@1.8.19 +mpi +hl', type='build', when='+mpi') + depends_on('hdf5@1.8.19 +hl', type='build') depends_on('silo@4.10.2 +hdf5', type='build') # Zlib fix has been merged into conduit, using develop until next release. - depends_on('conduit@0.9.1 +shared +mpi +hdf5~hdf5_compat -test ~parmetis', type='build', when='+mpi') - depends_on('conduit@0.9.1 +shared ~mpi +hdf5~hdf5_compat -test ~parmetis', type='build', when='~mpi') - depends_on('conduit@0.9.1 +shared +mpi +hdf5 -test ~parmetis', type='build', when='+mpi^hdf5@1.8.0:1.8') - depends_on('conduit@0.9.1 +shared ~mpi +hdf5 -test ~parmetis', type='build', when='~mpi^hdf5@1.8.0:1.8') - - depends_on('raja@2024.02.0 +cuda cuda_arch=70', when='+cuda') - depends_on('umpire +cuda cuda_arch=70', when='+cuda') - - depends_on('raja@2024.02.0 ~cuda', when='~cuda') - depends_on('umpire ~cuda', when='~cuda') - - depends_on('axom@0.9.0 ~shared +cuda +mpi +hdf5 -lua -examples -python -fortran', type='build', when='+mpi+cuda') - depends_on('axom@0.9.0 ~shared +cuda ~mpi +hdf5 -lua -examples -python -fortran', type='build', when='~mpi+cuda') - depends_on('axom@0.9.0 ~shared ~cuda +mpi +hdf5 -lua -examples -python -fortran', type='build', when='+mpi~cuda') - depends_on('axom@0.9.0 ~shared ~cuda ~mpi +hdf5 -lua -examples -python -fortran', type='build', when='~mpi~cuda') - - depends_on('caliper@2.8.0 ~shared ~adiak ~libdw ~papi ~libunwind +pic', type='build') + depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') + depends_on('conduit +hdf5', type='build', when='^hdf5@1.8.0:1.8') + depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom +shared', when='~cuda', type='build') + depends_on('axom ~shared', when='+cuda', type='build') + depends_on('caliper@2.11 ~shared +adiak +gotcha ~libdw ~papi ~libunwind +pic', type='build') + mpi_tpl_list = ["hdf5", "conduit", "axom", "caliper", "adiak~shared"] + for ctpl in mpi_tpl_list: + for mpiv in ["+mpi", "~mpi"]: + depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}") + + depends_on("raja@2024.02.0", type="build") + cuda_tpl_list = ["raja", "umpire", "axom"] + with when("+cuda"): + depends_on('caliper ~cuda', type="build") + for ctpl in cuda_tpl_list: + for val in CudaPackage.cuda_arch_values: + depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") + with when("~cuda"): + for ctpl in cuda_tpl_list: + depends_on(f"{ctpl} ~cuda", type='build') depends_on('opensubdiv@3.4.3', type='build') depends_on('polytope@0.7.3 +python', type='build') @@ -163,6 +166,8 @@ def initconfig_package_entries(self): # TPL locations entries.append(cmake_cache_path('caliper_DIR', spec['caliper'].prefix)) + entries.append(cmake_cache_path('adiak_DIR', spec['adiak'].prefix)) + entries.append(cmake_cache_path('python_DIR', spec['python'].prefix)) entries.append(cmake_cache_path('boost_DIR', spec['boost'].prefix)) diff --git a/scripts/spheral-setup-venv.in b/scripts/spheral-setup-venv.in index 05363f62f..cf17c0bce 100644 --- a/scripts/spheral-setup-venv.in +++ b/scripts/spheral-setup-venv.in @@ -28,6 +28,6 @@ cp --symbolic-link scripts/lcatstest.sh spheral-lcatstest &> /dev/null cd - > /dev/null echo "Byte-compiling packages in install path ..." -@CMAKE_INSTALL_PREFIX@/spheral -m compileall @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@ +@CMAKE_INSTALL_PREFIX@/spheral -m compileall -q @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@ echo "Done." diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac648aaff..2298e9393 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,16 +80,25 @@ foreach(_package ${_packages}) add_subdirectory(${_package}) endforeach() -if(NOT ENABLE_DEV_BUILD) - # Retrieve the global list populated in spheral_obj_add_library - get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) - +# For dev builds, we must call LLNLSpheralInstallObjs.cmake +# to ensure LLNLSpheral libraries are added to the Spheral_CXX +# target +if(ENABLE_DEV_BUILD) + # This calls LLNLSpheralInstallObjs.cmake + if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) + include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) + endif() +else() set(CXX_sources spheralCXX.cc) - # Must use quotes when passing lists as inputs for functions - spheral_add_cxx_library(CXX "${SPHERAL_OBJ_LIBS}") endif() +# Retrieve the global list populated in spheral_obj_add_library +get_property(SPHERAL_OBJ_LIBS GLOBAL PROPERTY SPHERAL_OBJ_LIBS) +# Must use quotes when passing lists as inputs for functions +spheral_add_cxx_library(CXX "${SPHERAL_OBJ_LIBS}") # This calls LLNLSpheralInstallObjs.cmake -if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) - include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) +if(NOT ENABLE_DEV_BUILD) + if(EXISTS ${EXTERNAL_SPHERAL_OBJ_CMAKE}) + include(${EXTERNAL_SPHERAL_OBJ_CMAKE}) + endif() endif() diff --git a/src/CRKSPH/CRKSPHHydroBase.cc b/src/CRKSPH/CRKSPHHydroBase.cc index bbdcc0ae5..12f3394ae 100644 --- a/src/CRKSPH/CRKSPHHydroBase.cc +++ b/src/CRKSPH/CRKSPHHydroBase.cc @@ -31,7 +31,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/CRKSPHHydroBaseRZ.cc b/src/CRKSPH/CRKSPHHydroBaseRZ.cc index c032358b6..b9cc6e79a 100644 --- a/src/CRKSPH/CRKSPHHydroBaseRZ.cc +++ b/src/CRKSPH/CRKSPHHydroBaseRZ.cc @@ -29,7 +29,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/CRKSPHVariant.cc b/src/CRKSPH/CRKSPHVariant.cc index da5dad87f..32c15f995 100644 --- a/src/CRKSPH/CRKSPHVariant.cc +++ b/src/CRKSPH/CRKSPHVariant.cc @@ -48,7 +48,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/newtonRaphson.hh" #include "Utilities/SpheralFunctions.hh" diff --git a/src/CRKSPH/SolidCRKSPHHydroBase.cc b/src/CRKSPH/SolidCRKSPHHydroBase.cc index 25fa74825..8f4bf7539 100644 --- a/src/CRKSPH/SolidCRKSPHHydroBase.cc +++ b/src/CRKSPH/SolidCRKSPHHydroBase.cc @@ -32,7 +32,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "SolidMaterial/SolidEquationOfState.hh" diff --git a/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc b/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc index 2a281f2a6..01f5774b3 100644 --- a/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc +++ b/src/CRKSPH/SolidCRKSPHHydroBaseRZ.cc @@ -33,7 +33,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/NodeCoupling.hh" #include "SolidMaterial/SolidEquationOfState.hh" diff --git a/src/DEM/DEMBase.cc b/src/DEM/DEMBase.cc index 03867b6e2..23b579f94 100644 --- a/src/DEM/DEMBase.cc +++ b/src/DEM/DEMBase.cc @@ -25,7 +25,6 @@ #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/registerWithRedistribution.hh" diff --git a/src/Distributed/Communicator.cc b/src/Distributed/Communicator.cc index 96aa5a236..b913cb4bc 100644 --- a/src/Distributed/Communicator.cc +++ b/src/Distributed/Communicator.cc @@ -13,12 +13,11 @@ namespace Spheral { // Default constructor (private). //------------------------------------------------------------------------------ Communicator:: -Communicator() +Communicator() { #ifdef USE_MPI - : mCommunicator() { mCommunicator = MPI_COMM_WORLD; #else -{ + mCommunicator = 0; #endif } @@ -29,4 +28,31 @@ Communicator:: ~Communicator() { } +//------------------------------------------------------------------------------ +// Public routines +//------------------------------------------------------------------------------ + +MPI_Comm* Communicator::comm_ptr() { +#ifdef USE_MPI + return &(instance().mCommunicator); +#else + return nullptr; +#endif +} + +void Communicator::finalize() { +#ifdef USE_MPI + int finalized = 0; + MPI_Finalized(&finalized); + if (finalized != 0) { + int finalize = MPI_Finalize(); + if (finalize != 0) { + char* string = nullptr; + int resultlen = 0; + MPI_Error_string(finalize, string, &resultlen); + VERIFY2(finalize, string); + } + } +#endif +} } diff --git a/src/Distributed/Communicator.hh b/src/Distributed/Communicator.hh index 11a03fb13..f9c2a6808 100644 --- a/src/Distributed/Communicator.hh +++ b/src/Distributed/Communicator.hh @@ -9,6 +9,9 @@ #ifdef USE_MPI #include +#include "Utilities/DBC.hh" +#else +typedef int MPI_Comm; #endif namespace Spheral { @@ -21,19 +24,14 @@ public: static Communicator& instance() { static Communicator theInstance; return theInstance; } // Access the communicator. -#ifdef USE_MPI static MPI_Comm& communicator() { return instance().mCommunicator; } static void communicator(MPI_Comm& comm) { instance().mCommunicator = comm; } -#else - static int communicator() { return 0; } - static void communicator(int&) {} -#endif + static MPI_Comm* comm_ptr(); + static void finalize(); private: //------------------------===== Private Interface =====----------------------// -#ifdef USE_MPI MPI_Comm mCommunicator; -#endif // No public constructors, destructor, or assignment. Communicator(); diff --git a/src/Distributed/mpi_mpi4py.py b/src/Distributed/mpi_mpi4py.py index e1692e256..5f99f1755 100644 --- a/src/Distributed/mpi_mpi4py.py +++ b/src/Distributed/mpi_mpi4py.py @@ -11,6 +11,7 @@ # as supported, but seem to be broken. import mpi4py mpi4py.rc.recv_mprobe = False +mpi4py.rc.finalize = False # Now go on as usual... from mpi4py import MPI diff --git a/src/FSISPH/SolidFSISPHHydroBase.cc b/src/FSISPH/SolidFSISPHHydroBase.cc index c004cc14b..5da90aa22 100644 --- a/src/FSISPH/SolidFSISPHHydroBase.cc +++ b/src/FSISPH/SolidFSISPHHydroBase.cc @@ -44,7 +44,6 @@ #include "Field/NodeIterators.hh" #include "Boundary/Boundary.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/timingUtilities.hh" #include "Utilities/safeInv.hh" #include "Utilities/Timer.hh" #include "Utilities/globalBoundingVolumes.hh" diff --git a/src/PYB11/Utilities/Adiak.py b/src/PYB11/Utilities/Adiak.py new file mode 100644 index 000000000..9bfd5cb05 --- /dev/null +++ b/src/PYB11/Utilities/Adiak.py @@ -0,0 +1,42 @@ +#------------------------------------------------------------------------------- +# Adiak utilities +#------------------------------------------------------------------------------- +from PYB11Generator import * + +# adiak::init() is called automatically when this module is loaded +# adiak::fini() is called automatically when this module is destroyed + +@PYB11cppname("adiak::fini") +def adiak_fini(): + "Finish Adiak" + return "void" + +@PYB11cppname("adiak::collect_all") +def adiak_collect_all(): + "Add some default Adiak metadata" + return "void" + +adiak_categories = PYB11enum(("unset", "all", "general", "performance", "control"), + doc="Enum of Adiak categories") + +@PYB11cppname("adiak::value") +@PYB11template("ValueType") +def adiak_value(name = "std::string", + value = "%(ValueType)s", + category = ("int", + "adiak_categories::general"), + subcategory = ("std::string", '""')): + "Set a single value in Adiak with a given name" + return "bool" + +@PYB11cppname("adiak::value") +@PYB11pyname("adiak_value") +@PYB11template("ValueType") +def adiak_value2(name = "std::string", + value = "%(ValueType)s", + value2 = "%(ValueType)s", + category = ("int", + "adiak_categories::general"), + subcategory = ("std::string", '""')): + "Set a pair of values in Adiak with a given name" + return "bool" diff --git a/src/PYB11/Utilities/TimerMgr.py b/src/PYB11/Utilities/TimerMgr.py new file mode 100644 index 000000000..8158110c5 --- /dev/null +++ b/src/PYB11/Utilities/TimerMgr.py @@ -0,0 +1,68 @@ +#------------------------------------------------------------------------------- +# TimerMgr class +#------------------------------------------------------------------------------- +from PYB11Generator import * + +@PYB11singleton +class TimerMgr: + + "Singleton wrapper for CaliperManager. Accesses a C++ singleton object." + + @PYB11static + def add(self, config_str = "std::string"): + "Add a Caliper configuration" + return "void" + + @PYB11static + def load(self, config_json = "std::string"): + "Load a json file containing Caliper configurations" + return "void" + + @PYB11static + def default_start(self, testname = "std::string"): + "Set the spot Caliper configuration and start the manager" + return "void" + + @PYB11static + def start(self): + "Start the Caliper configuration manager" + return "void" + + @PYB11static + def stop(self): + "Stop the Caliper configuration manager" + return "void" + + @PYB11static + def fini(self): + "Flush the Caliper configuration manager" + return "void" + + @PYB11static + def get_config(self): + "Return the current Caliper configuration" + return "std::string" + + @PYB11static + def get_filename(self): + "Return current Caliper filename, if set" + return "std::string" + + @PYB11static + def timers_usable(self): + "Return whether the code has been compiled with timers turned on" + return "bool" + @PYB11static + def timer_start(self, region_name = "std::string"): + "Start custom region Caliper timer, must have corresponding timer_end call" + return "void" + + @PYB11static + def timer_end(self, region_name = "std::string"): + "End custom region Caliper timer" + return "void" + + @PYB11static + def is_started(self): + "Check if ConfigManager has been started" + return "bool" diff --git a/src/PYB11/Utilities/Utilities_PYB11.py b/src/PYB11/Utilities/Utilities_PYB11.py index 45ca9ae63..feba85364 100644 --- a/src/PYB11/Utilities/Utilities_PYB11.py +++ b/src/PYB11/Utilities/Utilities_PYB11.py @@ -56,13 +56,60 @@ '"Utilities/BiQuadraticInterpolator.hh"', '"Utilities/BiCubicInterpolator.hh"', '"Utilities/uniform_random.hh"', + '"Utilities/Timer.hh"', + '"Distributed/Communicator.hh"', + '"adiak.hpp"', ''] #------------------------------------------------------------------------------- # Preamble #------------------------------------------------------------------------------- PYB11preamble += """ +namespace Spheral { + +inline void spheral_adiak_init() { + adiak::init((void*) Communicator::comm_ptr()); + // Always collect some curated default adiak information + adiak::adiakversion(); + adiak::user(); + adiak::uid(); + adiak::launchdate(); + adiak::workdir(); + adiak::hostname(); + adiak::clustername(); + adiak::walltime(); + adiak::cputime(); + adiak::jobsize(); + adiak::numhosts(); + adiak::hostlist(); + adiak::mpi_library_version(); +} + +enum adiak_categories { +unset = 0, +all, +general, +performance, +control +}; +} +""" +PYB11modulepreamble = """ +TIME_BEGIN("main"); +Spheral::spheral_adiak_init(); + +// Call these routines when module is exited +auto atexit = py::module_::import("atexit"); +atexit.attr("register")(py::cpp_function([]() { + TIME_END("main"); + adiak::fini(); + if (Spheral::TimerMgr::is_started()) { + Spheral::TimerMgr::fini(); + } else { + Communicator::finalize(); + } +})); """ #------------------------------------------------------------------------------- @@ -91,6 +138,8 @@ def setGlobalFlags(): from BiCubicInterpolator import * from uniform_random import * from BuildData import * +from Adiak import * +from TimerMgr import * ScalarScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "double")) ScalarPairScalarFunctor = PYB11TemplateClass(SpheralFunctor, template_parameters=("double", "std::pair")) @@ -747,3 +796,14 @@ def clippedVolume(poly = "const Dim<3>::FacetedVolume&", planes = "const std::vector>>&"): "Return the volume of the clipped region." return "double" + +#............................................................................... +for (value, label) in (("int", "Int"), + ("unsigned", "Unsigned"), + ("long", "Long"), + ("double", "Scalar"), + ("std::string", "String")): + exec(""" +adiak_value%(label)s = PYB11TemplateFunction(adiak_value, "%(value)s") +adiak_value2%(label)s = PYB11TemplateFunction(adiak_value2, "%(value)s", pyname="adiak_value%(label)s") +""" % {"label" : label, "value" : value}) diff --git a/src/SimulationControl/SpheralOptionParser.py b/src/SimulationControl/SpheralOptionParser.py index 473447c01..7f45e26c0 100644 --- a/src/SimulationControl/SpheralOptionParser.py +++ b/src/SimulationControl/SpheralOptionParser.py @@ -1,52 +1,91 @@ #------------------------------------------------------------------------------- -# Create a standard and hopefully convenient command line parser for Spheral +# Create a standard and hopefully convenient command line parser for Spheral # scripts. #------------------------------------------------------------------------------- -import optparse +import argparse, mpi from SpheralCompiledPackages import * from SpheralTestUtilities import globalFrame +from SpheralUtilities import TimerMgr def commandLine(**options): # Build a command line parser with the keyword arguments passed to us. - parser = optparse.OptionParser() + parser = argparse.ArgumentParser() for key in options: - parser.add_option("--" + key, - dest = key, - default = options[key]) - - # Add the universal options suppoted by all Spheral++ scripts. - parser.add_option("-v", "--verbose", - action = "store_true", - dest = "__verbose", - default = False, - help = "Verbose output -- print all options that were set.") + parser.add_argument("--" + key, + dest = key, + default = options[key]) + # Add the universal options supported by all Spheral++ scripts. + parser.add_argument("-v", "--verbose", + action = "store_true", + dest = "verbose", + default = False, + help = "Verbose output -- print all options that were set.") + parser.add_argument("--caliperConfig", default="", type=str) + parser.add_argument("--caliperFilename", default="", type=str) + parser.add_argument("--caliperConfigJSON", default="", type=str) # Evaluate the command line. - opts, args = parser.parse_args() + args = parser.parse_args() + arg_dict = vars(args) + + if (not TimerMgr.timers_usable()): + if (args.caliperConfig or args.caliperFilename or args.caliperConfigJSON): + print("WARNING: Caliper command line inputs provided for "+\ + "non-timer install. Reconfigure the install with "+\ + "-DENABLE_TIMER=ON to be able to use Caliper timers.") # Verbose output? - if opts.__verbose: + if args.verbose: print("All parameters set:") - for key in options: - val = eval("opts.%s" % key) - if val != options[key]: - print(" * ", key, " = ", val) - else: - print(" ", key, " = ", val) - + for key, val in arg_dict.items(): + if key in options: + if val != options[key]: + print(" * ", key, " = ", val) + else: + print(" ", key, " = ", val) + if (args.caliperConfig): + print(" * caliperConfig = ", args.caliperConfig) + if (args.caliperFilename): + print(" * caliperFilename = ", args.caliperFilename) + if (args.caliperConfigJSON): + print(" * caliperConfigJSON = ", args.caliperConfigJSON) # Set all the variables. gd = globalFrame().f_globals - for key in options: - val = eval("opts.%s" % key) - - # The following bit of goofiness is necessary because optparse returns most - # arguments as a string, but we need the actual types here. - if type(val) != type(options[key]): - val = eval(val, gd) - + for key, val in arg_dict.items(): + if key in options: + if (type(val) != type(options[key])): + val = eval(val, gd) gd[key] = val + # Initialize timers + InitTimers(args.caliperConfig, args.caliperFilename, args.caliperConfigJSON) + return +def InitTimers(caliper_config, filename, caliper_json): + if(caliper_json): + TimerMgr.load(caliper_json) + if(not caliper_config): + raise RuntimeError("SpheralOptionParser: specifying a configuration file without using one of the configurations means no timers are started") + off_tests = ["none", "off", "disable", "disabled", "0"] + if (caliper_config.lower() in off_tests): + return + elif (caliper_config): + TimerMgr.add(caliper_config) + TimerMgr.start() + else: + import os, sys + if (filename): + testname = filename + else: + from datetime import datetime + # Append the current day and time to the filename + unique_digits = datetime.now().strftime("_%Y_%m_%d_%H%M%S_%f") + # Name file based on name of python file being run + testname = os.path.splitext(os.path.basename(sys.argv[0]))[0] + testname += unique_digits + ".cali" + TimerMgr.default_start(testname) + adiak_valueInt("threads_per_rank", omp_get_num_threads()) + adiak_valueInt("num_ranks", mpi.procs) return diff --git a/src/SimulationControl/SpheralVisitDump.py b/src/SimulationControl/SpheralVisitDump.py index 9e772feed..16751280b 100644 --- a/src/SimulationControl/SpheralVisitDump.py +++ b/src/SimulationControl/SpheralVisitDump.py @@ -365,8 +365,7 @@ def _dumpField(self, field, file): # Private method to process a Field name into a Visit palatable label #--------------------------------------------------------------------------- def _processName(self, name): - import string - return string.replace(str(name), " ", "_") + return name.replace(" ", "_") #--------------------------------------------------------------------------- # Private method to write an atomic DataType element to a file. diff --git a/src/Utilities/Timer.hh b/src/Utilities/Timer.hh index 93e965cea..4ba514b59 100644 --- a/src/Utilities/Timer.hh +++ b/src/Utilities/Timer.hh @@ -1,15 +1,113 @@ +//---------------------------------Spheral++----------------------------------// +// Timer macros and Caliper ConfigManger singleton +// +//----------------------------------------------------------------------------// +#ifndef __Spheral_Timer__ +#define __Spheral_Timer__ // If TIMER is defined then we want timer functionality #ifdef TIMER #include "caliper/cali.h" +#include "caliper/cali-manager.h" #define TIME_FUNCTION CALI_CXX_MARK_FUNCTION #define TIME_BEGIN(regionName) CALI_MARK_BEGIN(regionName) #define TIME_END(regionName) CALI_MARK_END(regionName) #else // TIMER -// Stub TIME macros, when TIME is zero +// Stub TIME macros, when TIMER is off + #define TIME_FUNCTION #define TIME_BEGIN(regionName) #define TIME_END(regionName) -#endif // TIMER \ No newline at end of file +#endif // TIMER +// Note: This class is initialized in +// SimulationControl/SpheralOptionParser.py +namespace Spheral { +class TimerMgr { +private: + TimerMgr() = default; + ~TimerMgr() { } + TimerMgr(const TimerMgr&) = delete; + TimerMgr& operator=(const TimerMgr&) = delete; + bool started = false; + std::string caliperFilename = ""; + std::string caliperConfig = ""; +public: + static TimerMgr& instance() { + static TimerMgr theInstance; + return theInstance; + } + static void timer_start(std::string regionName) { + TIME_BEGIN(regionName.c_str()); + } + static void timer_end(std::string regionName) { + TIME_END(regionName.c_str()); + } + static bool is_started() { + return instance().started; + } + static std::string get_config() { + return instance().caliperConfig; + } + static std::string get_filename() { + return instance().caliperFilename; + } +#ifdef TIMER +private: + cali::ConfigManager cali_mgr; +public: + static bool timers_usable() { + return true; + } + static void add(std::string config_str) { + bool test = instance().cali_mgr.add(config_str.c_str()); + VERIFY2(test, instance().cali_mgr.error_msg()); + instance().caliperConfig += config_str; + } + static void load(std::string config_json) { + instance().cali_mgr.load(config_json.c_str()); + VERIFY2(!instance().cali_mgr.error(), instance().cali_mgr.error_msg()); + } + static void default_start(std::string testname) { + if (!testname.empty()) { + std::string default_config = "spot,output=" + testname; + instance().caliperFilename = testname; + add(default_config); + start(); + } else if (Spheral::Process::getRank() == 0) { + std::cout << "WARNING: Caliper test name is empty, " + << "no Caliper configuration started" << std::endl; + } + } + static void start() { + instance().cali_mgr.start(); + instance().started = true; + } + static void stop() { + instance().cali_mgr.stop(); + } + static void fini() { + instance().cali_mgr.flush(); + instance().started = false; + } +#else + static bool timers_usable() { + return false; + } + static void load(std::string) { + } + static void default_start(std::string) { + } + static void add(std::string) { + } + static void start() { + } + static void stop() { + } + static void fini() { + } +#endif +}; +} +#endif diff --git a/tests/functional/Hydro/Noh/Noh-planar-1d.py b/tests/functional/Hydro/Noh/Noh-planar-1d.py index 9e5ac1cd0..2e9ac0831 100644 --- a/tests/functional/Hydro/Noh/Noh-planar-1d.py +++ b/tests/functional/Hydro/Noh/Noh-planar-1d.py @@ -17,8 +17,8 @@ # #ATS:t10 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre' --restartStep 20 --restartFileConstructor SidreFileIO", label="Planar Noh problem -- 1-D (serial) with Sidre") #ATS:t11 = testif(t10, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", label="Planar Noh problem -- 1-D (serial) RESTART CHECK with Sidre") -#ATS:t12 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre-parrallel' --restartStep 20 --restartFileConstructor SidreFileIO", np=2, label="Planar Noh problem -- 1-D (parallel) with Sidre") -#ATS:t13 = testif(t12, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre-parrallel' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", np=2, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre") +#ATS:t12 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-sidre-parallel' --restartStep 20 --restartFileConstructor SidreFileIO", np=2, label="Planar Noh problem -- 1-D (parallel) with Sidre") +#ATS:t13 = testif(t12, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-sidre-parallel' --restartStep 20 --restartFileConstructor SidreFileIO --restoreCycle 20 --steps 20 --checkRestart True", np=2, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre") #ATS:t14 = test( SELF, "--graphics None --clearDirectories True --checkError True --dataDir 'dumps-planar-spio' --restartStep 20 --restartFileConstructor SidreFileIO --SPIOFileCountPerTimeslice 1", np=6, label="Planar Noh problem -- 1-D (parallel) with Sidre (SPIO check)") #ATS:t15 = testif(t14, SELF, "--graphics None --clearDirectories False --checkError False --dataDir 'dumps-planar-spio' --restartStep 20 --restartFileConstructor SidreFileIO --SPIOFileCountPerTimeslice 1 --restoreCycle 20 --steps 20 --checkRestart True", np=6, label="Planar Noh problem -- 1-D (parallel) RESTART CHECK with Sidre (SPIO check)") # diff --git a/tests/integration.ats b/tests/integration.ats index 429e04ba8..21573b92a 100644 --- a/tests/integration.ats +++ b/tests/integration.ats @@ -71,6 +71,7 @@ source("unit/Utilities/testCubicHermiteInterpolator.py") source("unit/Utilities/testBiLinearInterpolator.py") source("unit/Utilities/testBiQuadraticInterpolator.py") source("unit/Utilities/testBiCubicInterpolator.py") +source("unit/Utilities/testTimers.py") # Mesh tests. source("unit/Mesh/testLineMesh.py") diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt new file mode 100644 index 000000000..9182c51f9 --- /dev/null +++ b/tests/unit/CMakeLists.txt @@ -0,0 +1,6 @@ +add_subdirectory(CXXTests) + +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/Utilities/testTimers.py.in" + "${SPHERAL_TEST_INSTALL_PREFIX}/tests/unit/Utilities/testTimers.py" + ) diff --git a/tests/unit/Utilities/testTimers.py.in b/tests/unit/Utilities/testTimers.py.in new file mode 100644 index 000000000..53fb21506 --- /dev/null +++ b/tests/unit/Utilities/testTimers.py.in @@ -0,0 +1,84 @@ +# +# +#ATS:test(SELF, "--caliperFilename 'timer_test_1.cali'", label="Timer test 1", np=8) +#ATS:test(SELF, "--caliperConfig 'None'", label="Timer test 2", np=8) +#ATS:test(SELF, "--caliperFilename 'timer_test_3.cali'", label="Timer test 3", np=1) +# + +import Spheral +from SpheralTestUtilities import * +from SpheralOptionParser import * +from SpheralUtilities import TimerMgr +from SpheralUtilities import * +import mpi + +import sys, os, time + +commandLine() + +# Remove cali files from previous test runs +caliper_file = TimerMgr.get_filename() +if (os.path.exists(caliper_file)): + if (mpi.rank == 0): + os.remove(caliper_file) + +do_timers = False +if (TimerMgr.is_started()): + do_timers = True +test_dict_0 = {"perf_test": "weak_scaling"} +adiak_valueString("perf_test", test_dict_0["perf_test"], + adiak_categories.performance) +# Caliperreader reads everything as strings for some terrible reason +# So the test have to be hacked up + +# Correct method: +# test_dict_1 = {"rank_count": mpi.procs} +# adiak_valueInt("rank_count", test_dict_1["rank_count"]) + +# Hacked method to have tests pass with caliperreader: +test_dict_1 = {"rank_count": str(mpi.procs)} +adiak_valueString("rank_count", test_dict_1["rank_count"]) + +test_dicts = [test_dict_0, test_dict_1] +run_count = 8 +sleep_time = 1.E-4 +fake_timer_name = "test_timer" + +for i in range(run_count): + TimerMgr.timer_start(fake_timer_name) + time.sleep(sleep_time) + TimerMgr.timer_end(fake_timer_name) +if (do_timers and TimerMgr.get_filename()): + adiak_fini() + TimerMgr.fini() + mpi.barrier() + caliper_loc = "@CONFIG_CALIPER_DIR@" + sys.path.append(os.path.join(caliper_loc, "lib64/caliper")) + import caliperreader as cr + if (not os.path.exists(caliper_file)): + raise ValueError("Caliper file not found") + r = cr.CaliperReader() + r.read(caliper_file) + records = r.records + found_errors = 0 + # Test for timer name + if (fake_timer_name in records[1]['region']): + print(f"Found {fake_timer_name} timer") + else: + found_errors += 1 + # Test for function count + count_val = int(eval(records[1]["avg#sum#rc.count"])) + if (count_val == run_count): + print("Run count in Caliper file is correct") + else: + found_errors += 1 + # Test for adiak values + for td in test_dicts: + if (td.items() <= r.globals.items()): + print(f"Found {td.items()}") + else: + found_errors += 1 + if (found_errors > 0): + raise ValueError("Caliper file not correct") + else: + print("No errors found for TimerMgr")