Merge branch 'develop' into tupek/contact_adjoint

.gitlab-ci.yml

@@ -6,6 +6,7 @@
 variables:
   #TODO: uncomment this when everyone has service user access across all the machines
   #LLNL_SERVICE_USER: asmith
+  GIT_STRATEGY: clone
   GIT_SUBMODULE_STRATEGY: recursive
   PROJECT_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
   BUILD_ROOT: ${CI_PROJECT_DIR}

.gitlab/build_toss4.yml

@@ -61,11 +61,11 @@ toss4-gcc_10_3_1-src-no-tribol:
     ALLOC_TIME: "30"
   extends: .src_build_on_toss4
 
-toss4-gcc_10_3_1-src-no-sundials:
+toss4-gcc_10_3_1-src-no-optional-solvers:
   variables:
     COMPILER: "[email protected]"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -USUNDIALS_DIR"
+    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -USUNDIALS_DIR -UPETSC_DIR"
     ALLOC_NODES: "1"
     ALLOC_TIME: "20"
   extends: .src_build_on_toss4

scripts/spack/specs.json

@@ -18,8 +18,5 @@
       "[email protected]+devtools+profiling" ],
 
     "blueos_3_ppc64le_ib_p9":
-    [ "[email protected]+devtools+cuda+raja+umpire+profiling~strumpack cuda_arch=70" ],
-
-    "darwin-x86_64":
-    [ "[email protected]+devtools" ]
+    [ "[email protected]+devtools+cuda+raja+umpire+profiling~strumpack cuda_arch=70" ]
 }

src/docs/sphinx/dev_guide/profiling.rst

@@ -27,7 +27,7 @@ Introduction to SPOT
 --------------------
 
 `SPOT <https://software.llnl.gov/news/2021/01/07/spot-new>`_ is a framework developed at
-LLNL for vizualizing performance data.  SPOT is an external tool and does not need to be
+LLNL for visualizing performance data.  SPOT is an external tool and does not need to be
 linked into Serac.
 
 TPL Build Instructions
@@ -121,9 +121,21 @@ with benchmarking enabled (off by default). Then, run the build target ``run_ben
   cd <serac build location>
   make -j
   make run_benchmarks
-  find . -name "*.cali" -print0 | xargs -0 mv -t .
   pwd
 
 This will run all of Serac's benchmarks multiple times with varying MPI task counts, and generate a Caliper file for
-each benchmark run. The ``find`` command afterwards ensures all Caliper files are moved to the same directory. Now, you
-can visualize the results with SPOT, entering the path printed from ``pwd``.
+each benchmark run at ``PROJECT_BINARY_DIR``. Now, you can visualize the results with SPOT, entering the path printed
+from ``pwd``.
+
+Visualizing Benchmarks using SPOT
+---------------------------------
+
+If you have access to LC, you can go to the following website and enter a directory in CZ/ RZ that contains Caliper
+files:
+
+- `SPOT CZ <https://lc.llnl.gov/spot2>`_
+- `SPOT RZ <https://rzlc.llnl.gov/spot2>`_
+
+.. note::
+  There is a bug in SPOT where if you remove Caliper files from a directory, they still show up on SPOT - if you've
+  visualized them previously. The current workaround is by removing the ``llnl.gov`` site cache manually.

src/docs/sphinx/dev_guide/testing.rst

@@ -11,7 +11,7 @@ Testing
 
 Serac has two levels of tests, unit and integration. Unit tests are used to test
 individual components of code, such as a class or function.  While integration tests
-are for testing the code as a whole. For example, testing the `serac` driver with
+are for testing the code as a whole. For example, testing the ``serac`` driver with
 an input file against blessed answers.
 
 Unit Tests
@@ -41,7 +41,7 @@ Requirements:
       >>> socket.gethostname().rstrip('1234567890')
       >>> exit()
 
-  Currently, there are configuration json files for Toss3 and BlueOS which can be
+  Currently, there are configuration json files for TOSS4 and BlueOS which can be
   used as reference.
 
 #. **Build the code.**
@@ -54,7 +54,7 @@ Requirements:
      # BlueOS
      $ lalloc 2 ./ats.sh
      
-     # Toss3
+     # TOSS4
      $ salloc -N2 ./ats.sh
      
      # Personal Machine (currently runs subset of tests)
@@ -72,6 +72,22 @@ Requirements:
 
    ATS also outputs both a ``.log`` and ``.log.err`` for each test and checker that is run.
 
+#. **Rebaseline tests (as needed).**
+   If tolerances to tests need to be updated, first ensure you've generated new tolerances by running the integration
+   tests like mentioned above. Then, use the ``-b`` option::
+
+     # Single baseline
+     $ ./ats.sh -b dyn_solve_serial
+
+     # Comma-separated baselines
+     $ ./ats.sh -b dyn_solve_serial,dyn_solve_parallel
+
+     # All baselines
+     $ ./ats.sh -b all
+
+   This will update the json files located in the `serac_tests <https://github.com/LLNL/serac_tests>`_ submodule. To
+   avoid Caliper files from additionally being generated, configure with ``-DENABLE_BENCHMARKS=OFF``.
+
 
 Installing ATS
 --------------

src/docs/sphinx/quickstart.rst

@@ -78,7 +78,7 @@ For example on **Ubuntu 20.04**:
    python3 scripts/uberenv/uberenv.py --project-json=scripts/spack/devtools.json --spack-env-file=scripts/spack/configs/linux_ubuntu_20/spack.yaml --prefix=../path/to/install
 
 Unlike Serac's library dependencies, our developer tools can be built with any compiler because
-they are not linked into the serac executable.  We recommend GCC 8 because we have tested that they all
+they are not linked into the serac executable.  We recommend GCC 10 because we have tested that they all
 build with that compiler.
 
 Building Serac's Dependencies via Spack/uberenv
@@ -97,7 +97,7 @@ This has been encapsulated using `Uberenv <https://github.com/LLNL/uberenv>`_. U
 doing the following:
 
 * Pulls a blessed version of Spack locally
-* If you are on a known operating system (like TOSS3), we have defined Spack configuration files
+* If you are on a known operating system (like TOSS4), we have defined Spack configuration files
   to keep Spack from building the world
 * Installs our Spack packages into the local Spack
 * Simplifies whole dependency build into one command
@@ -120,18 +120,18 @@ Serac.
 
 .. note::
   On LC machines, it is good practice to do the build step in parallel on a compute node.
-  Here is an example command: ``salloc -ppdebug -N1-1 python3 scripts/uberenv/uberenv.py``
+  Here is an example command: ``salloc -ppdebug -N1 python3 scripts/uberenv/uberenv.py``
 
 Unless otherwise specified Spack will default to a compiler.  This is generally not a good idea when
 developing large codes. To specify which compiler to use add the compiler specification to the ``--spec`` Uberenv
-command line option. On TOSS3, we recommend and have tested ``--spec=%clang@10.0.0``.  More compiler specs
+command line option. On TOSS4, we recommend and have tested ``--spec=%clang@14.0.6``.  More compiler specs
 can be found in the Spack compiler files in our repository:
 ``scripts/spack/configs/<platform>/spack.yaml``.
 
 We currently regularly test the following Spack configuration files:
 
-* Linux Ubuntu 20.04 (via Windows WSL 2)
-* TOSS 3 (On Ruby at LC)
+* Linux Ubuntu 22.04 (via Azure)
+* TOSS4 (On Ruby at LC)
 * BlueOS (On Lassen at LC)
 
 To install Serac on a new platform, it is a good idea to start with a known Spack environments file, or ``spack.yaml`` file,
@@ -144,19 +144,19 @@ in the `Spack docs <https://spack.readthedocs.io/en/latest/configuration.html>`_
    If you do not have a ``spack.yaml`` already, you can leave off that command line option from ``uberenv`` and
    Spack will generate a new one for you. Uberenv will copy it where you ran your uberenv command for future use.
 .. note::
-   A newer vesion of cmake (>=3.20) and llvm (>=14) may be required.
+   A newer version of cmake (>=3.20) and llvm (>=14) may be required.
 
 
 Some helpful uberenv options include :
 
 * ``--spec=" build_type=Debug"`` (build the MFEM and Hypre libraries with debug symbols)
 * ``--spec=+profiling`` (build the Adiak and Caliper libraries)
 * ``--spec=+devtools`` (also build the devtools with one command)
-* ``--spec=%clang@10.0.0`` (build with a specific compiler as defined in the ``spack.yaml`` file)
+* ``--spec=%clang@14.0.6`` (build with a specific compiler as defined in the ``spack.yaml`` file)
 * ``--spack-env-file=<Path to Spack environment file>`` (use specific Spack environment configuration file)
 * ``--prefix=<Path>`` (required, build and install the dependencies in a particular location) - this *must be outside* of your local Serac repository
 
-The modifiers to the Spack specification ``spec`` can be chained together, e.g. ``--spec='%clang@10.0.0+devtools build_type=Debug'``.
+The modifiers to the Spack specification ``spec`` can be chained together, e.g. ``--spec='%clang@14.0.6+devtools build_type=Debug'``.
 
 If you already have a Spack instance from another project that you would like to reuse,
 you can do so by changing the uberenv command as follows:
@@ -169,7 +169,7 @@ Building Serac's Dependencies by Hand
 -------------------------------------
 
 To build Serac's dependencies by hand, use of a ``host-config`` CMake configuration file is
-stongly encouraged. A good place to start is by copying an existing host config in the
+strongly encouraged. A good place to start is by copying an existing host config in the
 ``host-config`` directory and modifying it according to your system setup.
 
 .. _build-label:

src/serac/infrastructure/accelerator.cpp

@@ -25,7 +25,7 @@ void initializeDevice()
 {
   SLIC_ERROR_ROOT_IF(device, "serac::accelerator::initializeDevice cannot be called more than once");
   device = std::make_unique<mfem::Device>();
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(SERAC_USE_CUDA_KERNEL_EVALUATION)
   device->Configure("cuda");
 #endif
 }

src/serac/infrastructure/tests/error_handling.cpp

@@ -57,39 +57,6 @@ TEST(ErrorHandling, BcOneComponentVectorCoefDofs)
   EXPECT_THROW(BoundaryCondition(coef, 0, *space, dofs), SlicErrorException);
 }
 
-TEST(ErrorHandling, BcRetrieveScalarCoef)
-{
-  auto mesh = mesh::refineAndDistribute(buildDiskMesh(10), 0, 0);
-
-  auto [space, coll] = serac::generateParFiniteElementSpace<H1<1>>(mesh.get());
-
-  auto              coef = std::make_shared<mfem::ConstantCoefficient>(1.0);
-  BoundaryCondition bc(coef, -1, *space, std::set<int>{});
-  EXPECT_NO_THROW(bc.scalarCoefficient());
-  EXPECT_THROW(bc.vectorCoefficient(), SlicErrorException);
-
-  const auto& const_bc = bc;
-  EXPECT_NO_THROW(const_bc.scalarCoefficient());
-  EXPECT_THROW(const_bc.vectorCoefficient(), SlicErrorException);
-}
-
-TEST(ErrorHandling, BcRetrieveVecCoef)
-{
-  auto mesh = mesh::refineAndDistribute(buildDiskMesh(10), 0, 0);
-
-  auto [space, coll] = serac::generateParFiniteElementSpace<H1<1>>(mesh.get());
-
-  mfem::Vector      vec;
-  auto              coef = std::make_shared<mfem::VectorConstantCoefficient>(vec);
-  BoundaryCondition bc(coef, {}, *space, std::set<int>{});
-  EXPECT_NO_THROW(bc.vectorCoefficient());
-  EXPECT_THROW(bc.scalarCoefficient(), SlicErrorException);
-
-  const auto& const_bc = bc;
-  EXPECT_NO_THROW(const_bc.vectorCoefficient());
-  EXPECT_THROW(const_bc.scalarCoefficient(), SlicErrorException);
-}
-
 TEST(ErrorHandling, InvalidCmdlineArg)
 {
   // The command is actually --input-file