Merge branch 'development' of github.com:erf-model/ERF into update_GABLS

ewquon · Nov 13, 2023 · 1576281 · 1576281
2 parents 46399d3 + fa25dc6
commit 1576281
Show file tree

Hide file tree

Showing 92 changed files with 3,000 additions and 2,002 deletions.
diff --git a/CHANGES b/CHANGES
@@ -1,3 +1,16 @@
+#23.11
+
+  -- AMReX submodule set to 23.11 release hash (ae7b64b)
+
+  -- Bug fix with vertical grid stretching and MOST (#1275)
+
+  -- Bug fix with negative K_turb below surface and MYNN2.5 PBL model (#1271)
+
+  -- Correction to equation of state with moisture (#1263)
+
+  -- Generalized multi-level sponge zones. This allows for nudging of fine solution
+     towards coarse solution with complex box arrays. (#1233)
+
 # 23.10
 
   -- First release since JOSS paper. Too many key ERF PRs to summarize.

diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -45,7 +45,6 @@ function(build_erf_lib erf_lib_name)
 
   if(ERF_ENABLE_NETCDF)
     target_sources(${erf_lib_name} PRIVATE
-                   ${SRC_DIR}/IO/NCBuildFABs.cpp
                    ${SRC_DIR}/IO/NCInterface.cpp
                    ${SRC_DIR}/IO/NCPlotFile.cpp
                    ${SRC_DIR}/IO/NCCheckpoint.cpp
@@ -79,6 +78,7 @@ function(build_erf_lib erf_lib_name)
                    ${SRC_DIR}/Radiation/Aero_rad_props.cpp
                    ${SRC_DIR}/Radiation/Optics.cpp
                    ${SRC_DIR}/Radiation/Radiation.cpp
+                   ${SRC_DIR}/Radiation/Albedo.cpp
                    ${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/examples/mo_load_coefficients.cpp
                    ${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp
                   )
@@ -113,6 +113,7 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/BoundaryConditions/BoundaryConditions_zvel.cpp
        ${SRC_DIR}/BoundaryConditions/BoundaryConditions_bndryreg.cpp
        ${SRC_DIR}/BoundaryConditions/BoundaryConditions_wrfbdy.cpp
+       ${SRC_DIR}/BoundaryConditions/BoundaryConditions_metgrid.cpp
        ${SRC_DIR}/BoundaryConditions/ERF_FillPatch.cpp
        ${SRC_DIR}/BoundaryConditions/ERF_FillPatcher.cpp
        ${SRC_DIR}/BoundaryConditions/ERF_PhysBCFunct.cpp
@@ -146,6 +147,7 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/TimeIntegration/ERF_TimeStep.cpp
        ${SRC_DIR}/TimeIntegration/ERF_advance_dycore.cpp
        ${SRC_DIR}/TimeIntegration/ERF_advance_microphysics.cpp
+       ${SRC_DIR}/TimeIntegration/ERF_advance_radiation.cpp
        ${SRC_DIR}/TimeIntegration/ERF_make_buoyancy.cpp
        ${SRC_DIR}/TimeIntegration/ERF_make_condensation_source.cpp
        ${SRC_DIR}/TimeIntegration/ERF_make_fast_coeffs.cpp

diff --git a/Docs/sphinx_doc/Inputs.rst b/Docs/sphinx_doc/Inputs.rst
@@ -860,7 +860,7 @@ List of Parameters
 | **erf.use_terrain**         | use terrain-fitted |  true / false      | false      |
 |                             | coordinates?       |                    |            |
 +-----------------------------+--------------------+--------------------+------------+
-| **erf.terrain_type**        | static or moving?  |  0 / 1             | 0          |
+| **erf.terrain_type**        | static or moving?  |  Static / Moving   | Static     |
 +-----------------------------+--------------------+--------------------+------------+
 | **erf.terrain_smoothing**   | specify terrain    | 0,                 | 0          |
 |                             | following          | 1,                 |            |

diff --git a/Docs/sphinx_doc/RegressionTests.rst b/Docs/sphinx_doc/RegressionTests.rst
@@ -45,7 +45,7 @@ The following problems are currently tested in the CI:
 | DensityCurrent_detJ2_MT       | 256 4 64 | Symmetry | Periodic | SlipWall   | None  | use_terrain = true    |
 |                               |          | Outflow  |          | SlipWall   |       | uses zlevels          |
 |                               |          | Outflow  |          | SlipWall   |       | detJ = 2 everywhere   |
-|                               |          |          |          |            |       | terrain_type = 1      |
+|                               |          |          |          |            |       | terrain_type = Moving |
 +-------------------------------+----------+----------+----------+------------+-------+-----------------------+
 | EkmanSpiral                   | 4 4 400  | Periodic | Periodic | NoSlipWall | Geo   | +Coriolis             |
 |                               |          |          |          | SlipWall   |       | +gravity              |

diff --git a/Docs/sphinx_doc/theory/WetEquations.rst b/Docs/sphinx_doc/theory/WetEquations.rst
@@ -38,7 +38,7 @@ The governing equations for this model are
   \frac{\partial \rho_d}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u})
 
   \frac{\partial (\rho_d \mathbf{u})}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u} \mathbf{u}) -
-          \frac{1}{1 + q_v + q_c}  \nabla p^\prime - \nabla \cdot \tau + \mathbf{F} + \delta_{i,3}\mathbf{B}
+          \frac{1}{1 + q_v + q_c} ( \nabla p^\prime  + \delta_{i,3}\mathbf{B} ) - \nabla \cdot \tau + \mathbf{F}
 
   \frac{\partial (\rho_d \theta_d)}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u} \theta_d)
                 + \nabla \cdot ( \rho_d \alpha_{T}\ \nabla \theta_d) + \frac{\theta_d L_v}{T_d C_{pd}} f_{cond}

diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt
@@ -24,4 +24,5 @@ else ()
   add_subdirectory(DevTests/MovingTerrain)
   add_subdirectory(DevTests/ParticlesOverWoA)
   add_subdirectory(DevTests/MiguelDev)
+  add_subdirectory(DevTests/MetGrid)
 endif()
diff --git a/Exec/DevTests/MetGrid/CMakeLists.txt b/Exec/DevTests/MetGrid/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(erf_exe_name erf_metgrid_dev)
+
+add_executable(${erf_exe_name} "")
+target_sources(${erf_exe_name}
+   PRIVATE
+     prob.cpp
+)
+
+target_include_directories(${erf_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
+build_erf_exe(${erf_exe_name})
diff --git a/Exec/DevTests/MetGrid/GNUmakefile b/Exec/DevTests/MetGrid/GNUmakefile
@@ -19,14 +19,19 @@ USE_HIP  = FALSE
 USE_SYCL = FALSE
 
 # Debugging
-DEBUG = FALSE
+#DEBUG = FALSE
 DEBUG = TRUE
 
 TEST = TRUE
 USE_ASSERTION = TRUE
 
 USE_NETCDF = TRUE
 
+#USE_MOISTURE = FALSE
+USE_MOISTURE = TRUE
+
+#USE_WARM_NO_PRECIP = TRUE
+
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DevTests/MetGrid/inputs b/Exec/DevTests/MetGrid/inputs
@@ -1,11 +1,13 @@
 # ------------------  INPUTS TO MAIN PROGRAM  -------------------
-max_step = 1
+max_step = 100
 
 amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1
 
 # PROBLEM SIZE & GEOMETRY
-geometry.prob_extent =  10600 5000 5000
-amr.n_cell           =   140 80 60
+geometry.prob_extent =  28000 16000 8000
+amr.n_cell           =   140 80 100
 
 geometry.is_periodic = 0 0 0
 
@@ -17,9 +19,10 @@ zlo.type = "NoSlipWall"
 zhi.type = "SlipWall"
 
 # TIME STEP CONTROL
-erf.fixed_dt           = 0.1  # fixed time step depending on grid resolution
+erf.fixed_dt           = 1.0  # fixed time step depending on grid resolution
 
 erf.use_terrain = true
+erf.terrain_smoothing = 2
 
 # DIAGNOSTICS & VERBOSITY
 erf.sum_interval   = -1       # timesteps between computing mass
@@ -36,22 +39,26 @@ erf.check_int       = 100        # number of timesteps between checkpoints
 
 # PLOTFILES
 erf.plot_file_1     = plt        # prefix of plotfile name
-erf.plot_int_1      = 10         # number of timesteps between plotfiles
-erf.plot_vars_1     = density rhoadv_0 x_velocity y_velocity z_velocity pressure temp theta z_phys
+erf.plot_int_1      = 1          # number of timesteps between plotfiles
+erf.plot_vars_1     = qv Rhoqv density dens_hse rhoadv_0 x_velocity y_velocity z_velocity pressure temp theta z_phys mapfac pres_hse pert_pres KE QKE
 
 # SOLVER CHOICE
-erf.alpha_T = 0.0
+erf.alpha_T = 1.0
 erf.alpha_C = 1.0
 erf.use_gravity = true
 
 erf.molec_diff_type = "None"
 erf.les_type        = "Smagorinsky"
+#erf.les_type        = "Deardorff"
 erf.Cs              = 0.1
 
+#erf.terrain_z_levels = 0 130 354 583 816 1054 1549 2068 2615 3193 3803 4450 5142 5892 6709 7603 8591 9702 10967 12442 14230 16610 18711 20752 22133 23960 26579 28493 31236 33699 36068 40000
+
 # INITIALIZATION WITH ATM DATA
-erf.init_type    = "metgrid"
-erf.nc_init_file_0 = "met_em_d01.nc"
-#erf.nc_bdy_file = ""
+erf.metgrid_bdy_width = 5
+erf.metgrid_bdy_set_width = 1
+erf.init_type      = "metgrid"
+erf.nc_init_file_0 = "met_em.d01.2022-06-18_00:00:00.nc" "met_em.d01.2022-06-18_06:00:00.nc" "met_em.d01.2022-06-18_12:00:00.nc" "met_em.d01.2022-06-18_18:00:00.nc"
 
 #There will be no OpenMP tiling
 fabarray.mfiter_tile_size = 1024 1024 1024
diff --git a/Exec/DevTests/MetGrid/prob.H b/Exec/DevTests/MetGrid/prob.H
@@ -12,7 +12,7 @@ struct ProbParm : ProbParmDefaults {
 class Problem : public ProblemBase
 {
 public:
-    Problem();
+    Problem(const amrex::Real* problo, const amrex::Real* probhi);
 
 protected:
     std::string name() override { return "Metgrid"; }

diff --git a/Exec/DevTests/MetGrid/prob.cpp b/Exec/DevTests/MetGrid/prob.cpp
@@ -8,5 +8,5 @@ amrex_probinit(const amrex_real* problo, const amrex_real* probhi)
     return std::make_unique<Problem>(problo, probhi);
 }
 
-Problem::Problem()
+Problem::Problem(const amrex::Real* problo, const amrex::Real* probhi)
 {}
diff --git a/Exec/DevTests/MovingTerrain/inputs b/Exec/DevTests/MovingTerrain/inputs
@@ -19,8 +19,8 @@ zlo.type = "SlipWall"
 zhi.type = "SlipWall"
 
 # TERRRAIN GRID TYPE
-erf.use_terrain = 1         # enable terrain stencils
-erf.terrain_type = 1        # moving terrain
+erf.use_terrain  = true     # enable terrain stencils
+erf.terrain_type = Moving   # moving terrain
 erf.terrain_smoothing = 2   # Sullivan 2004 approach
 
 # FOR NO SUBSTEPPING

diff --git a/Exec/LLJ/GNUmakefile b/Exec/LLJ/GNUmakefile
@@ -25,7 +25,7 @@ TEST = TRUE
 USE_ASSERTION = TRUE
 
 USE_NETCDF  = TRUE
-USE_HDF5    = TRUE
+#USE_HDF5    = TRUE
 
 # GNU Make
 Bpack := ./Make.package

diff --git a/Exec/RegTests/ScalarAdvDiff/prob.cpp b/Exec/RegTests/ScalarAdvDiff/prob.cpp
@@ -118,8 +118,9 @@ Problem::init_custom_pert(
         state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz, r0) / r0));
     } else if (parms.prob_type == 13) {
         const Real r0_z = parms.rad_0 * (prob_hi[2] - prob_lo[2]);
-        const Real r2d_xz = std::sqrt((x-xc)*(x-xc)/(r0*r0) + (z-zc)*(z-zc)/(r0_z*r0_z)); //ellipse for mapfac shear validation
-        state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz, r0_z) / r0_z));
+        //ellipse for mapfac shear validation
+        const Real r2d_xz_ell = std::sqrt((x-xc)*(x-xc)/(r0*r0) + (z-zc)*(z-zc)/(r0_z*r0_z));
+        state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz_ell, r0_z) / r0_z));
     } else if (parms.prob_type == 14) {
         state(i, j, k, RhoScalar_comp) = std::cos(PI*x);
     } else {

diff --git a/Exec/RegTests/WPS_Test/GNUmakefile b/Exec/RegTests/WPS_Test/GNUmakefile
@@ -27,7 +27,7 @@ USE_ASSERTION = TRUE
 USE_MOISTURE = TRUE
 
 USE_NETCDF  = TRUE
-USE_HDF5    = TRUE
+#USE_HDF5    = TRUE
 
 # GNU Make
 Bpack := ./Make.package

diff --git a/Source/Advection/Advection.H b/Source/Advection/Advection.H
@@ -28,21 +28,24 @@ void AdvectionSrcForRhoAndTheta (const amrex::Box& bx, const amrex::Box& valid_b
                                  const amrex::Array4<const amrex::Real>& mf_u,
                                  const amrex::Array4<const amrex::Real>& mf_v,
                                  const AdvType horiz_adv_type, const AdvType vert_adv_type,
-                                 const int use_terrain);
+                                 const bool use_terrain);
 
 /** Compute advection tendency for all scalars other than density and potential temperature */
-void AdvectionSrcForScalars (const amrex::Box& bx,
+void AdvectionSrcForScalars (int level, int finest_level,
+                             const amrex::MFIter& mfi,
+                             const amrex::Box& bx,
                              const int icomp, const int ncomp,
                              const amrex::Array4<const amrex::Real>& avg_xmom,
                              const amrex::Array4<const amrex::Real>& avg_ymom,
                              const amrex::Array4<const amrex::Real>& avg_zmom,
                              const amrex::Array4<const amrex::Real>& cell_prim,
                              const amrex::Array4<amrex::Real>& src,
                              const amrex::Array4<const amrex::Real>& detJ,
+                             const amrex::Real dt,
                              const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>& cellSize,
                              const amrex::Array4<const amrex::Real>& mf_m,
                              const AdvType horiz_adv_type, const AdvType vert_adv_type,
-                             const int use_terrain);
+                             const bool use_terrain, const bool is_two_way_coupling);
 
 /** Compute advection tendencies for all components of momentum */
 void AdvectionSrcForMom (const amrex::Box& bxx, const amrex::Box& bxy, const amrex::Box& bxz,
@@ -58,7 +61,7 @@ void AdvectionSrcForMom (const amrex::Box& bxx, const amrex::Box& bxy, const amr
                          const amrex::Array4<const amrex::Real>& mf_u,
                          const amrex::Array4<const amrex::Real>& mf_v,
                          const AdvType horiz_adv_type, const AdvType vert_adv_type,
-                         const int use_terrain, const int domhi_z);
+                         const bool use_terrain, const int domhi_z);
 
 AMREX_GPU_HOST_DEVICE
 AMREX_FORCE_INLINE

diff --git a/Source/Advection/AdvectionSrcForMom.cpp b/Source/Advection/AdvectionSrcForMom.cpp
@@ -51,7 +51,7 @@ AdvectionSrcForMom (const Box& bxx, const Box& bxy, const Box& bxz,
                     const Array4<const Real>& mf_v,
                     const AdvType horiz_adv_type,
                     const AdvType vert_adv_type,
-                    const int use_terrain,
+                    const bool use_terrain,
                     const int domhi_z)
 {
     BL_PROFILE_VAR("AdvectionSrcForMom", AdvectionSrcForMom);