Utilize MFEM's petsc/ slepc wrappers

azure-pipelines.yml

@@ -6,8 +6,8 @@
 #
 
 variables:
-  Clang_14_ImageName: 'seracllnl/tpls:clang-14_05-31-24_19h-38m'
-  GCC_13_ImageName: 'seracllnl/tpls:gcc-13_05-31-24_19h-38m'
+  Clang_14_ImageName: 'seracllnl/tpls:clang-14_06-27-24_19h-46m'
+  GCC_13_ImageName: 'seracllnl/tpls:gcc-13_06-27-24_19h-46m'
 
 jobs:
 - job: Build_and_Test

cmake/SeracConfigHeader.cmake

@@ -36,7 +36,7 @@ message(STATUS "Configuring Serac version ${SERAC_VERSION_FULL}")
 #------------------------------------------------------------------------------
 # Create variable for every TPL
 #------------------------------------------------------------------------------
-set(TPL_DEPS ADIAK AXOM CAMP CONDUIT CUDA FMT HDF5 LUA MFEM MPI TRIBOL CALIPER PETSC RAJA STRUMPACK SUNDIALS UMPIRE)
+set(TPL_DEPS ADIAK AXOM CAMP CONDUIT CUDA FMT HDF5 LUA MFEM MPI TRIBOL CALIPER RAJA STRUMPACK SUNDIALS UMPIRE)
 foreach(dep ${TPL_DEPS})
     if( ${dep}_FOUND OR ENABLE_${dep} )
         set(SERAC_USE_${dep} TRUE)
@@ -114,8 +114,6 @@ install(
   FILES 
     ${CMAKE_CURRENT_BINARY_DIR}/serac-config.cmake
     ${CMAKE_CURRENT_BINARY_DIR}/serac-config-version.cmake
-    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/thirdparty/FindPETSc.cmake
-    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/thirdparty/FindSLEPc.cmake
   DESTINATION 
     ${SERAC_INSTALL_CMAKE_MODULE_DIR}
 )

cmake/serac-config.cmake.in

@@ -37,9 +37,7 @@ if(NOT SERAC_FOUND)
   set(SERAC_USE_HDF5           @SERAC_USE_HDF5@)
   set(SERAC_USE_MFEM           @SERAC_USE_MFEM@)
   set(SERAC_USE_MPI            @SERAC_USE_MPI@)
-  set(SERAC_USE_PETSC          @SERAC_USE_PETSC@)
   set(SERAC_USE_RAJA           @SERAC_USE_RAJA@)
-  set(SERAC_USE_SLEPC          @SERAC_USE_SLEPC@)
   set(SERAC_USE_STRUMPACK      @SERAC_USE_STRUMPACK@)
   set(SERAC_USE_SUNDIALS       @SERAC_USE_SUNDIALS@)
   set(SERAC_USE_TRIBOL         @SERAC_USE_TRIBOL@)
@@ -52,9 +50,7 @@ if(NOT SERAC_FOUND)
   set(SERAC_CONDUIT_DIR        "@CONDUIT_DIR@")
   set(SERAC_HDF5_DIR           "@HDF5_DIR@")
   set(SERAC_MFEM_DIR           "@MFEM_DIR@")
-  set(SERAC_PETSC_DIR          "@PETSC_DIR@")
   set(SERAC_RAJA_DIR           "@RAJA_DIR@")
-  set(SERAC_SLEPC_DIR          "@SLEPC_DIR@")
   set(SERAC_STRUMPACK_DIR      "@STRUMPACK_DIR@")
   set(SERAC_SUNDIALS_DIR       "@SUNDIALS_DIR@")
   set(SERAC_TRIBOL_DIR         "@TRIBOL_DIR@")
@@ -67,7 +63,7 @@ if(NOT SERAC_FOUND)
   endif()
 
   # Set to real variable unless user overrode it
-  foreach(dep ADIAK AXOM CAMP CALIPER CONDUIT HDF5 MFEM PETSC RAJA STRUMPACK SUNDIALS TRIBOL UMPIRE)
+  foreach(dep ADIAK AXOM CAMP CALIPER CONDUIT HDF5 MFEM RAJA STRUMPACK SUNDIALS TRIBOL UMPIRE)
     if (NOT ${dep}_DIR)
       set(${dep}_DIR "${SERAC_${dep}_DIR}")
     endif()
@@ -155,16 +151,6 @@ if(NOT SERAC_FOUND)
     find_dependency(tribol REQUIRED PATHS "${TRIBOL_DIR}/lib/cmake")
   endif()
 
-  # PETSc
-  if(SERAC_USE_PETSC)
-    include("${CMAKE_CURRENT_LIST_DIR}/FindPETSc.cmake")
-  endif()
-
-  # SLEPc
-  if(SERAC_USE_SLEPC)
-    include("${CMAKE_CURRENT_LIST_DIR}/FindSLEPc.cmake")
-  endif()
-
   # SUNDIALS
   if(SERAC_USE_SUNDIALS)
     find_dependency(SUNDIALS REQUIRED NO_DEFAULT_PATH PATHS "${SUNDIALS_DIR}")

cmake/thirdparty/FindARPACK.cmake

@@ -0,0 +1,45 @@
+# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and
+# other Serac Project Developers. See the top-level LICENSE file for
+# details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+#
+# Defines the following:
+# ARPACK_FOUND
+# ARPACK_INCLUDE_DIRS
+# ARPACK_LIBRARIES
+
+# Set ARPACK_DIR to specify the prefix of ARPACK installation
+
+if (NOT EXISTS "${ARPACK_DIR}")
+    message(FATAL_ERROR "Given ARPACK_DIR does not exist: ${ARPACK_DIR}")
+endif()
+
+if (NOT IS_DIRECTORY "${ARPACK_DIR}")
+    message(FATAL_ERROR "Given ARPACK_DIR is not a directory: ${ARPACK_DIR}")
+endif()
+
+# Find include dirs
+find_path(ARPACK_INCLUDE_DIRS
+  NAMES arpackdef.h
+  PATHS ${ARPACK_DIR}/include/arpack-ng
+  NO_DEFAULT_PATH
+  NO_CMAKE_ENVIRONMENT_PATH
+  NO_CMAKE_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  NO_CMAKE_SYSTEM_PATH
+)
+
+# Find libraries
+find_library(ARPACK_LIBRARIES
+  NAMES parpack
+  PATHS ${ARPACK_DIR}/lib
+  NO_DEFAULT_PATH
+  NO_CMAKE_ENVIRONMENT_PATH
+  NO_CMAKE_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  NO_CMAKE_SYSTEM_PATH
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ARPACK DEFAULT_MSG ARPACK_LIBRARIES ARPACK_INCLUDE_DIRS)

cmake/thirdparty/SetupSeracThirdParty.cmake

@@ -136,6 +136,51 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
     endif()
     message(STATUS "Sundials support is ${SERAC_USE_SUNDIALS}")
 
+    #------------------------------------------------------------------------------
+    # PETSc
+    #------------------------------------------------------------------------------
+    if (PETSC_DIR)
+        serac_assert_is_directory(DIR_VARIABLE PETSC_DIR)
+        # NOTE: PETSc is built and used through MFEM
+        set(SERAC_USE_PETSC ON CACHE BOOL "")
+
+        # Note: MFEM *does not* set PETSC_FOUND itself, likely because we skip petsc build tests
+        set(PETSC_FOUND TRUE)
+    else()
+        set(SERAC_USE_PETSC OFF CACHE BOOL "")
+        set(PETSC_FOUND FALSE)
+    endif()
+    message(STATUS "PETSc support is ${SERAC_USE_PETSC}")
+
+    #------------------------------------------------------------------------------
+    # SLEPc
+    #------------------------------------------------------------------------------
+    if (SLEPC_DIR AND SERAC_USE_PETSC)
+        serac_assert_is_directory(DIR_VARIABLE SLEPC_DIR)
+        # NOTE: SLEPc is built and used through MFEM
+        set(SERAC_USE_SLEPC ON CACHE BOOL "")
+
+        # Note: MFEM sets SLEPC_FOUND itself
+        if (NOT SERAC_ENABLE_CODEVELOP)
+            set(SLEPC_FOUND TRUE)
+        endif()
+    else()
+        set(SERAC_USE_SLEPC OFF CACHE BOOL "")
+        set(SLEPC_FOUND FALSE)
+    endif()
+    message(STATUS "SLEPc support is ${SERAC_USE_SLEPC}")
+
+    #------------------------------------------------------------------------------
+    # ARPACK
+    #------------------------------------------------------------------------------
+    if (ARPACK_DIR AND SERAC_USE_SLEPC)
+        serac_assert_is_directory(DIR_VARIABLE ARPACK_DIR)
+        include(${CMAKE_CURRENT_LIST_DIR}/FindARPACK.cmake)
+    else()
+        set(ARPACK_FOUND FALSE)
+    endif()
+    message(STATUS "ARPACK support is ${ARPACK_FOUND}")
+
     #------------------------------------------------------------------------------
     # MFEM
     #------------------------------------------------------------------------------
@@ -182,8 +227,21 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
             set(ParMETIS_DIR ${PARMETIS_DIR} CACHE PATH "")
         endif()
         set(MFEM_USE_OPENMP ${ENABLE_OPENMP} CACHE BOOL "")
-        set(MFEM_USE_PETSC OFF CACHE BOOL "")
-        set(MFEM_USE_SLEPC OFF CACHE BOOL "")
+
+        if(PETSC_DIR)
+            set(MFEM_USE_PETSC ON CACHE BOOL "")
+            set(PETSC_ARCH "" CACHE STRING "")
+            set(PETSC_EXECUTABLE_RUNS "ON" CACHE BOOL "")
+            if(SLEPC_DIR)
+                set(MFEM_USE_SLEPC ON CACHE BOOL "")
+                set(SLEPC_ARCH "" CACHE STRING "")
+                set(SLEPC_VERSION_OK "TRUE" CACHE BOOL "")
+            endif()
+        else()
+            set(MFEM_USE_PETSC OFF CACHE BOOL "")
+            set(MFEM_USE_SLEPC OFF CACHE BOOL "")
+        endif()
+
         set(MFEM_USE_RAJA OFF CACHE BOOL "")
         set(MFEM_USE_SUNDIALS ${SERAC_USE_SUNDIALS} CACHE BOOL "")
         if(SUPERLUDIST_DIR)
@@ -267,38 +325,6 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         set(MFEM_BUILT_WITH_CMAKE TRUE)
     endif()
 
-    #------------------------------------------------------------------------------
-    # PETSc
-    #------------------------------------------------------------------------------
-    if(PETSC_DIR)
-        serac_assert_is_directory(DIR_VARIABLE PETSC_DIR)
-        include(${CMAKE_CURRENT_LIST_DIR}/FindPETSc.cmake)
-        serac_assert_find_succeeded(PROJECT_NAME PETSc
-                                    TARGET       PkgConfig::PETSC
-                                    DIR_VARIABLE PETSC_DIR)
-        message(STATUS "PETSc support is ON")
-        set(PETSC_FOUND TRUE)
-    else()
-        message(STATUS "PETSc support is OFF")
-        set(PETSC_FOUND FALSE)
-    endif()
-
-    #------------------------------------------------------------------------------
-    # SLEPC
-    #------------------------------------------------------------------------------
-    if(SLEPC_DIR)
-        serac_assert_is_directory(DIR_VARIABLE SLEPC_DIR)
-        include(${CMAKE_CURRENT_LIST_DIR}/FindSLEPc.cmake)
-        serac_assert_find_succeeded(PROJECT_NAME SLEPC
-                                    TARGET PkgConfig::SLEPC
-                                    DIR_VARIABLE SLEPC_DIR)
-        message(STATUS "SLEPc support is ON")
-        set(SLEPC_FOUND TRUE)
-    else()
-        message(STATUS "SLEPc support is OFF")
-        set(SLEPC_FOUND FALSE)
-    endif()
-
     #------------------------------------------------------------------------------
     # Axom
     #------------------------------------------------------------------------------
@@ -489,14 +515,14 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
     # MPI library/directory but in some cases Spack cannot determine
     # the correct MPI lib directory. This guards against that.
     # https://github.com/spack/spack/issues/24685
+    set(_mfem_targets
+        mfem
+        axom::mfem
+        tribol::mfem)
     if(STRUMPACK_DIR)
         list(GET MPI_C_LIBRARIES 0 _first_mpi_lib)
         get_filename_component(_mpi_lib_dir ${_first_mpi_lib} DIRECTORY)
 
-        set(_mfem_targets
-            mfem
-            axom::mfem
-            tribol::mfem)
         foreach(_target ${_mfem_targets})
             if(TARGET ${_target})
                 message(STATUS "Adding MPI link directory to target [${_target}]")
@@ -505,6 +531,19 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         endforeach()
     endif()
 
+    # Add missing ARPACK flags needed by SLEPc by injecting them into the MFEM targets.
+    # https://github.com/mfem/mfem/issues/4364
+    if (ARPACK_FOUND)
+        foreach(_target ${_mfem_targets})
+            if(TARGET ${_target})
+                message(STATUS "Adding arpack libraries and include dirs to target [${_target}]")
+                target_include_directories(${_target} INTERFACE ${ARPACK_INCLUDE_DIRS})
+                target_link_libraries(${_target} INTERFACE ${ARPACK_LIBRARIES})
+            endif()
+        endforeach()
+    endif()
+    unset(_mfem_targets)
+
     # Restore cleared Adiak/Caliper directories, reason at top of file.
     set(ADIAK_DIR ${_adiak_dir} CACHE PATH "" FORCE)
     set(CALIPER_DIR ${_caliper_dir} CACHE PATH "" FORCE)