From 55816b5c1009bb5635e457b21b37958e85886210 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Tue, 12 Nov 2024 10:46:47 -0800 Subject: [PATCH 1/3] fixes for builds with no raja --- src/serac/infrastructure/CMakeLists.txt | 2 +- src/serac/infrastructure/debug_print.hpp | 8 +++++ .../functional/domain_integral_kernels.hpp | 2 ++ .../functional/element_restriction.cpp | 36 +++++++++++++++---- .../functional/element_restriction.hpp | 4 +++ 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/serac/infrastructure/CMakeLists.txt b/src/serac/infrastructure/CMakeLists.txt index dc88eed574..53e2fa324a 100644 --- a/src/serac/infrastructure/CMakeLists.txt +++ b/src/serac/infrastructure/CMakeLists.txt @@ -41,7 +41,7 @@ set(infrastructure_sources terminator.cpp ) -set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends}) +set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends}) blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND) blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING) list(APPEND infrastructure_depends blt::mpi) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 485f058afa..09c28f3580 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,7 +74,11 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template +#ifdef SERAC_USE_RAJA void write_to_file(axom::Array arr, std::string filename) +#else +void write_to_file(axom::Array arr, std::string filename) +#endif { std::ofstream outfile(filename); @@ -97,7 +101,11 @@ void write_to_file(axom::Array arr, std::string f * @param filename the name of the output file */ template +#ifdef SERAC_USE_RAJA void write_to_file(axom::Array arr, std::string filename) +#else +void write_to_file(axom::Array arr, std::string filename) +#endif { std::ofstream outfile(filename); diff --git a/src/serac/numerics/functional/domain_integral_kernels.hpp b/src/serac/numerics/functional/domain_integral_kernels.hpp index c218025fbe..b6043464c9 100644 --- a/src/serac/numerics/functional/domain_integral_kernels.hpp +++ b/src/serac/numerics/functional/domain_integral_kernels.hpp @@ -10,7 +10,9 @@ #include "serac/numerics/functional/quadrature_data.hpp" #include "serac/numerics/functional/function_signature.hpp" #include "serac/numerics/functional/differentiate_wrt.hpp" +#ifdef SERAC_USE_RAJA #include "RAJA/RAJA.hpp" +#endif #include #include diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 9068da4056..7d3a4fa243 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,8 +213,12 @@ std::vector > geom_local_face_dofs(int p) return output; } -axom::Array GetElementRestriction(const mfem::FiniteElementSpace* fes, - mfem::Geometry::Type geom) +#ifdef SERAC_USE_RAJA +axom::Array +#else +axom::Array +#endif +GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -267,17 +271,29 @@ axom::Array GetElementRestriction(const mfem::F } if (n == 0) { +#ifdef SERAC_USE_RAJA return axom::Array{}; +#else + return axom::Array{}; +#endif } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; + uint64_t dofs_per_elem = elem_dofs.size() / n; +#ifdef SERAC_USE_RAJA axom::Array output(n, dofs_per_elem); +#else + axom::Array output(n, dofs_per_elem); +#endif std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetFaceDofs(const mfem::FiniteElementSpace* fes, - mfem::Geometry::Type face_geom, FaceType type) +#ifdef SERAC_USE_RAJA +axom::Array +#else +axom::Array +#endif +GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -378,10 +394,18 @@ axom::Array GetFaceDofs(const mfem::FiniteEleme delete face_to_elem; if (n == 0) { +#ifdef SERAC_USE_RAJA return axom::Array{}; +#else + return axom::Array{}; +#endif } else { - uint64_t dofs_per_face = face_dofs.size() / n; + uint64_t dofs_per_face = face_dofs.size() / n; +#ifdef SERAC_USE_RAJA axom::Array output(n, dofs_per_face); +#else + axom::Array output(n, dofs_per_face); +#endif std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index 7f12dca598..10aa3ecb97 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,7 +195,11 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space +#ifdef SERAC_USE_RAJA axom::Array dof_info; +#else + axom::Array dof_info; +#endif /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering; From 12debca4f424cc70625cbb141afd468423e645a8 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Tue, 12 Nov 2024 11:02:35 -0800 Subject: [PATCH 2/3] change raja to umpire where appropriate --- src/serac/infrastructure/debug_print.hpp | 4 ++-- .../numerics/functional/element_restriction.cpp | 12 ++++++------ .../numerics/functional/element_restriction.hpp | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 09c28f3580..99daf67129 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,7 +74,7 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE void write_to_file(axom::Array arr, std::string filename) #else void write_to_file(axom::Array arr, std::string filename) @@ -101,7 +101,7 @@ void write_to_file(axom::Array arr, std::string filename) * @param filename the name of the output file */ template -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE void write_to_file(axom::Array arr, std::string filename) #else void write_to_file(axom::Array arr, std::string filename) diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 7d3a4fa243..03ee48a97e 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,7 +213,7 @@ std::vector > geom_local_face_dofs(int p) return output; } -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array #else axom::Array @@ -271,14 +271,14 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } if (n == 0) { -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE return axom::Array{}; #else return axom::Array{}; #endif } else { uint64_t dofs_per_elem = elem_dofs.size() / n; -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array output(n, dofs_per_elem); #else axom::Array output(n, dofs_per_elem); @@ -288,7 +288,7 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } } -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array #else axom::Array @@ -394,14 +394,14 @@ GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, delete face_to_elem; if (n == 0) { -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE return axom::Array{}; #else return axom::Array{}; #endif } else { uint64_t dofs_per_face = face_dofs.size() / n; -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array output(n, dofs_per_face); #else axom::Array output(n, dofs_per_face); diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index 10aa3ecb97..daeb5220b3 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,7 +195,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space -#ifdef SERAC_USE_RAJA +#ifdef SERAC_USE_UMPIRE axom::Array dof_info; #else axom::Array dof_info; From a07d20cfb76638a8bd805c184d13d0e64d5c7491 Mon Sep 17 00:00:00 2001 From: "Eric B. Chin" Date: Thu, 14 Nov 2024 09:10:47 -0800 Subject: [PATCH 3/3] use dynamic memory for all --- src/serac/infrastructure/debug_print.hpp | 8 ----- .../functional/element_restriction.cpp | 34 +++---------------- .../functional/element_restriction.hpp | 4 --- 3 files changed, 4 insertions(+), 42 deletions(-) diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 99daf67129..9d77c0ee65 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -74,11 +74,7 @@ std::ostream& operator<<(std::ostream& out, serac::SignedIndex i) * @param filename the name of the output file */ template -#ifdef SERAC_USE_UMPIRE -void write_to_file(axom::Array arr, std::string filename) -#else void write_to_file(axom::Array arr, std::string filename) -#endif { std::ofstream outfile(filename); @@ -101,11 +97,7 @@ void write_to_file(axom::Array arr, std::string filename) * @param filename the name of the output file */ template -#ifdef SERAC_USE_UMPIRE -void write_to_file(axom::Array arr, std::string filename) -#else void write_to_file(axom::Array arr, std::string filename) -#endif { std::ofstream outfile(filename); diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index 03ee48a97e..58f776bb28 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -213,12 +213,7 @@ std::vector > geom_local_face_dofs(int p) return output; } -#ifdef SERAC_USE_UMPIRE -axom::Array -#else -axom::Array -#endif -GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) +axom::Array GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -271,29 +266,16 @@ GetElementRestriction(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type } if (n == 0) { -#ifdef SERAC_USE_UMPIRE - return axom::Array{}; -#else return axom::Array{}; -#endif } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; -#ifdef SERAC_USE_UMPIRE - axom::Array output(n, dofs_per_elem); -#else + uint64_t dofs_per_elem = elem_dofs.size() / n; axom::Array output(n, dofs_per_elem); -#endif std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -#ifdef SERAC_USE_UMPIRE -axom::Array -#else -axom::Array -#endif -GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) +axom::Array GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -394,18 +376,10 @@ GetFaceDofs(const mfem::FiniteElementSpace* fes, mfem::Geometry::Type face_geom, delete face_to_elem; if (n == 0) { -#ifdef SERAC_USE_UMPIRE - return axom::Array{}; -#else return axom::Array{}; -#endif } else { - uint64_t dofs_per_face = face_dofs.size() / n; -#ifdef SERAC_USE_UMPIRE - axom::Array output(n, dofs_per_face); -#else + uint64_t dofs_per_face = face_dofs.size() / n; axom::Array output(n, dofs_per_face); -#endif std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index daeb5220b3..3dc1e6a3a3 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -195,11 +195,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space -#ifdef SERAC_USE_UMPIRE - axom::Array dof_info; -#else axom::Array dof_info; -#endif /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering;