diff --git a/CHANGES b/CHANGES index d12fabc89..76926febd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,16 @@ +#23.11 + + -- AMReX submodule set to 23.11 release hash (ae7b64b) + + -- Bug fix with vertical grid stretching and MOST (#1275) + + -- Bug fix with negative K_turb below surface and MYNN2.5 PBL model (#1271) + + -- Correction to equation of state with moisture (#1263) + + -- Generalized multi-level sponge zones. This allows for nudging of fine solution + towards coarse solution with complex box arrays. (#1233) + # 23.10 -- First release since JOSS paper. Too many key ERF PRs to summarize. diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake index 7f44ff5ff..b9219ad4b 100644 --- a/CMake/BuildERFExe.cmake +++ b/CMake/BuildERFExe.cmake @@ -45,7 +45,6 @@ function(build_erf_lib erf_lib_name) if(ERF_ENABLE_NETCDF) target_sources(${erf_lib_name} PRIVATE - ${SRC_DIR}/IO/NCBuildFABs.cpp ${SRC_DIR}/IO/NCInterface.cpp ${SRC_DIR}/IO/NCPlotFile.cpp ${SRC_DIR}/IO/NCCheckpoint.cpp @@ -79,6 +78,7 @@ function(build_erf_lib erf_lib_name) ${SRC_DIR}/Radiation/Aero_rad_props.cpp ${SRC_DIR}/Radiation/Optics.cpp ${SRC_DIR}/Radiation/Radiation.cpp + ${SRC_DIR}/Radiation/Albedo.cpp ${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/examples/mo_load_coefficients.cpp ${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp ) @@ -113,6 +113,7 @@ function(build_erf_lib erf_lib_name) ${SRC_DIR}/BoundaryConditions/BoundaryConditions_zvel.cpp ${SRC_DIR}/BoundaryConditions/BoundaryConditions_bndryreg.cpp ${SRC_DIR}/BoundaryConditions/BoundaryConditions_wrfbdy.cpp + ${SRC_DIR}/BoundaryConditions/BoundaryConditions_metgrid.cpp ${SRC_DIR}/BoundaryConditions/ERF_FillPatch.cpp ${SRC_DIR}/BoundaryConditions/ERF_FillPatcher.cpp ${SRC_DIR}/BoundaryConditions/ERF_PhysBCFunct.cpp @@ -146,6 +147,7 @@ function(build_erf_lib erf_lib_name) ${SRC_DIR}/TimeIntegration/ERF_TimeStep.cpp ${SRC_DIR}/TimeIntegration/ERF_advance_dycore.cpp ${SRC_DIR}/TimeIntegration/ERF_advance_microphysics.cpp + ${SRC_DIR}/TimeIntegration/ERF_advance_radiation.cpp ${SRC_DIR}/TimeIntegration/ERF_make_buoyancy.cpp ${SRC_DIR}/TimeIntegration/ERF_make_condensation_source.cpp ${SRC_DIR}/TimeIntegration/ERF_make_fast_coeffs.cpp diff --git a/Docs/sphinx_doc/Inputs.rst b/Docs/sphinx_doc/Inputs.rst index 672ef6a4c..faaf43adf 100644 --- a/Docs/sphinx_doc/Inputs.rst +++ b/Docs/sphinx_doc/Inputs.rst @@ -860,7 +860,7 @@ List of Parameters | **erf.use_terrain** | use terrain-fitted | true / false | false | | | coordinates? | | | +-----------------------------+--------------------+--------------------+------------+ -| **erf.terrain_type** | static or moving? | 0 / 1 | 0 | +| **erf.terrain_type** | static or moving? | Static / Moving | Static | +-----------------------------+--------------------+--------------------+------------+ | **erf.terrain_smoothing** | specify terrain | 0, | 0 | | | following | 1, | | diff --git a/Docs/sphinx_doc/RegressionTests.rst b/Docs/sphinx_doc/RegressionTests.rst index d5c858677..56266be2d 100644 --- a/Docs/sphinx_doc/RegressionTests.rst +++ b/Docs/sphinx_doc/RegressionTests.rst @@ -45,7 +45,7 @@ The following problems are currently tested in the CI: | DensityCurrent_detJ2_MT | 256 4 64 | Symmetry | Periodic | SlipWall | None | use_terrain = true | | | | Outflow | | SlipWall | | uses zlevels | | | | Outflow | | SlipWall | | detJ = 2 everywhere | -| | | | | | | terrain_type = 1 | +| | | | | | | terrain_type = Moving | +-------------------------------+----------+----------+----------+------------+-------+-----------------------+ | EkmanSpiral | 4 4 400 | Periodic | Periodic | NoSlipWall | Geo | +Coriolis | | | | | | SlipWall | | +gravity | diff --git a/Docs/sphinx_doc/theory/WetEquations.rst b/Docs/sphinx_doc/theory/WetEquations.rst index fc6c29d77..38288e484 100644 --- a/Docs/sphinx_doc/theory/WetEquations.rst +++ b/Docs/sphinx_doc/theory/WetEquations.rst @@ -38,7 +38,7 @@ The governing equations for this model are \frac{\partial \rho_d}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u}) \frac{\partial (\rho_d \mathbf{u})}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u} \mathbf{u}) - - \frac{1}{1 + q_v + q_c} \nabla p^\prime - \nabla \cdot \tau + \mathbf{F} + \delta_{i,3}\mathbf{B} + \frac{1}{1 + q_v + q_c} ( \nabla p^\prime + \delta_{i,3}\mathbf{B} ) - \nabla \cdot \tau + \mathbf{F} \frac{\partial (\rho_d \theta_d)}{\partial t} &= - \nabla \cdot (\rho_d \mathbf{u} \theta_d) + \nabla \cdot ( \rho_d \alpha_{T}\ \nabla \theta_d) + \frac{\theta_d L_v}{T_d C_{pd}} f_{cond} diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt index 6474d52a0..30ea921c9 100644 --- a/Exec/CMakeLists.txt +++ b/Exec/CMakeLists.txt @@ -24,4 +24,5 @@ else () add_subdirectory(DevTests/MovingTerrain) add_subdirectory(DevTests/ParticlesOverWoA) add_subdirectory(DevTests/MiguelDev) + add_subdirectory(DevTests/MetGrid) endif() diff --git a/Exec/DevTests/MetGrid/CMakeLists.txt b/Exec/DevTests/MetGrid/CMakeLists.txt new file mode 100644 index 000000000..621a7e0aa --- /dev/null +++ b/Exec/DevTests/MetGrid/CMakeLists.txt @@ -0,0 +1,12 @@ +set(erf_exe_name erf_metgrid_dev) + +add_executable(${erf_exe_name} "") +target_sources(${erf_exe_name} + PRIVATE + prob.cpp +) + +target_include_directories(${erf_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + +include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake) +build_erf_exe(${erf_exe_name}) diff --git a/Exec/DevTests/MetGrid/GNUmakefile b/Exec/DevTests/MetGrid/GNUmakefile index 838ce130e..0b412a6bb 100644 --- a/Exec/DevTests/MetGrid/GNUmakefile +++ b/Exec/DevTests/MetGrid/GNUmakefile @@ -19,7 +19,7 @@ USE_HIP = FALSE USE_SYCL = FALSE # Debugging -DEBUG = FALSE +#DEBUG = FALSE DEBUG = TRUE TEST = TRUE @@ -27,6 +27,11 @@ USE_ASSERTION = TRUE USE_NETCDF = TRUE +#USE_MOISTURE = FALSE +USE_MOISTURE = TRUE + +#USE_WARM_NO_PRECIP = TRUE + # GNU Make Bpack := ./Make.package Blocs := . diff --git a/Exec/DevTests/MetGrid/inputs b/Exec/DevTests/MetGrid/inputs index cbe1f1f85..2be9c6f2e 100644 --- a/Exec/DevTests/MetGrid/inputs +++ b/Exec/DevTests/MetGrid/inputs @@ -1,11 +1,13 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- -max_step = 1 +max_step = 100 amrex.fpe_trap_invalid = 1 +amrex.fpe_trap_zero = 1 +amrex.fpe_trap_overflow = 1 # PROBLEM SIZE & GEOMETRY -geometry.prob_extent = 10600 5000 5000 -amr.n_cell = 140 80 60 +geometry.prob_extent = 28000 16000 8000 +amr.n_cell = 140 80 100 geometry.is_periodic = 0 0 0 @@ -17,9 +19,10 @@ zlo.type = "NoSlipWall" zhi.type = "SlipWall" # TIME STEP CONTROL -erf.fixed_dt = 0.1 # fixed time step depending on grid resolution +erf.fixed_dt = 1.0 # fixed time step depending on grid resolution erf.use_terrain = true +erf.terrain_smoothing = 2 # DIAGNOSTICS & VERBOSITY erf.sum_interval = -1 # timesteps between computing mass @@ -36,22 +39,26 @@ erf.check_int = 100 # number of timesteps between checkpoints # PLOTFILES erf.plot_file_1 = plt # prefix of plotfile name -erf.plot_int_1 = 10 # number of timesteps between plotfiles -erf.plot_vars_1 = density rhoadv_0 x_velocity y_velocity z_velocity pressure temp theta z_phys +erf.plot_int_1 = 1 # number of timesteps between plotfiles +erf.plot_vars_1 = qv Rhoqv density dens_hse rhoadv_0 x_velocity y_velocity z_velocity pressure temp theta z_phys mapfac pres_hse pert_pres KE QKE # SOLVER CHOICE -erf.alpha_T = 0.0 +erf.alpha_T = 1.0 erf.alpha_C = 1.0 erf.use_gravity = true erf.molec_diff_type = "None" erf.les_type = "Smagorinsky" +#erf.les_type = "Deardorff" erf.Cs = 0.1 +#erf.terrain_z_levels = 0 130 354 583 816 1054 1549 2068 2615 3193 3803 4450 5142 5892 6709 7603 8591 9702 10967 12442 14230 16610 18711 20752 22133 23960 26579 28493 31236 33699 36068 40000 + # INITIALIZATION WITH ATM DATA -erf.init_type = "metgrid" -erf.nc_init_file_0 = "met_em_d01.nc" -#erf.nc_bdy_file = "" +erf.metgrid_bdy_width = 5 +erf.metgrid_bdy_set_width = 1 +erf.init_type = "metgrid" +erf.nc_init_file_0 = "met_em.d01.2022-06-18_00:00:00.nc" "met_em.d01.2022-06-18_06:00:00.nc" "met_em.d01.2022-06-18_12:00:00.nc" "met_em.d01.2022-06-18_18:00:00.nc" #There will be no OpenMP tiling fabarray.mfiter_tile_size = 1024 1024 1024 diff --git a/Exec/DevTests/MetGrid/prob.H b/Exec/DevTests/MetGrid/prob.H index 18d9e2892..6ad4ec3fe 100644 --- a/Exec/DevTests/MetGrid/prob.H +++ b/Exec/DevTests/MetGrid/prob.H @@ -12,7 +12,7 @@ struct ProbParm : ProbParmDefaults { class Problem : public ProblemBase { public: - Problem(); + Problem(const amrex::Real* problo, const amrex::Real* probhi); protected: std::string name() override { return "Metgrid"; } diff --git a/Exec/DevTests/MetGrid/prob.cpp b/Exec/DevTests/MetGrid/prob.cpp index 4d9c43e75..bedd22dbd 100644 --- a/Exec/DevTests/MetGrid/prob.cpp +++ b/Exec/DevTests/MetGrid/prob.cpp @@ -8,5 +8,5 @@ amrex_probinit(const amrex_real* problo, const amrex_real* probhi) return std::make_unique(problo, probhi); } -Problem::Problem() +Problem::Problem(const amrex::Real* problo, const amrex::Real* probhi) {} diff --git a/Exec/DevTests/MovingTerrain/inputs b/Exec/DevTests/MovingTerrain/inputs index 6c1e42970..8b0783fb4 100644 --- a/Exec/DevTests/MovingTerrain/inputs +++ b/Exec/DevTests/MovingTerrain/inputs @@ -19,8 +19,8 @@ zlo.type = "SlipWall" zhi.type = "SlipWall" # TERRRAIN GRID TYPE -erf.use_terrain = 1 # enable terrain stencils -erf.terrain_type = 1 # moving terrain +erf.use_terrain = true # enable terrain stencils +erf.terrain_type = Moving # moving terrain erf.terrain_smoothing = 2 # Sullivan 2004 approach # FOR NO SUBSTEPPING diff --git a/Exec/LLJ/GNUmakefile b/Exec/LLJ/GNUmakefile index 31c46b692..4a773e7b1 100644 --- a/Exec/LLJ/GNUmakefile +++ b/Exec/LLJ/GNUmakefile @@ -25,7 +25,7 @@ TEST = TRUE USE_ASSERTION = TRUE USE_NETCDF = TRUE -USE_HDF5 = TRUE +#USE_HDF5 = TRUE # GNU Make Bpack := ./Make.package diff --git a/Exec/RegTests/ScalarAdvDiff/prob.cpp b/Exec/RegTests/ScalarAdvDiff/prob.cpp index 643980677..a7d7dd560 100644 --- a/Exec/RegTests/ScalarAdvDiff/prob.cpp +++ b/Exec/RegTests/ScalarAdvDiff/prob.cpp @@ -118,8 +118,9 @@ Problem::init_custom_pert( state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz, r0) / r0)); } else if (parms.prob_type == 13) { const Real r0_z = parms.rad_0 * (prob_hi[2] - prob_lo[2]); - const Real r2d_xz = std::sqrt((x-xc)*(x-xc)/(r0*r0) + (z-zc)*(z-zc)/(r0_z*r0_z)); //ellipse for mapfac shear validation - state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz, r0_z) / r0_z)); + //ellipse for mapfac shear validation + const Real r2d_xz_ell = std::sqrt((x-xc)*(x-xc)/(r0*r0) + (z-zc)*(z-zc)/(r0_z*r0_z)); + state(i, j, k, RhoScalar_comp) = parms.A_0 * 0.25 * (1.0 + std::cos(PI * std::min(r2d_xz_ell, r0_z) / r0_z)); } else if (parms.prob_type == 14) { state(i, j, k, RhoScalar_comp) = std::cos(PI*x); } else { diff --git a/Exec/RegTests/WPS_Test/GNUmakefile b/Exec/RegTests/WPS_Test/GNUmakefile index ad0a36000..f83bb0ff5 100644 --- a/Exec/RegTests/WPS_Test/GNUmakefile +++ b/Exec/RegTests/WPS_Test/GNUmakefile @@ -27,7 +27,7 @@ USE_ASSERTION = TRUE USE_MOISTURE = TRUE USE_NETCDF = TRUE -USE_HDF5 = TRUE +#USE_HDF5 = TRUE # GNU Make Bpack := ./Make.package diff --git a/Source/Advection/Advection.H b/Source/Advection/Advection.H index 9b3121a57..1007f1f04 100644 --- a/Source/Advection/Advection.H +++ b/Source/Advection/Advection.H @@ -28,10 +28,12 @@ void AdvectionSrcForRhoAndTheta (const amrex::Box& bx, const amrex::Box& valid_b const amrex::Array4& mf_u, const amrex::Array4& mf_v, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain); + const bool use_terrain); /** Compute advection tendency for all scalars other than density and potential temperature */ -void AdvectionSrcForScalars (const amrex::Box& bx, +void AdvectionSrcForScalars (int level, int finest_level, + const amrex::MFIter& mfi, + const amrex::Box& bx, const int icomp, const int ncomp, const amrex::Array4& avg_xmom, const amrex::Array4& avg_ymom, @@ -39,10 +41,11 @@ void AdvectionSrcForScalars (const amrex::Box& bx, const amrex::Array4& cell_prim, const amrex::Array4& src, const amrex::Array4& detJ, + const amrex::Real dt, const amrex::GpuArray& cellSize, const amrex::Array4& mf_m, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain); + const bool use_terrain, const bool is_two_way_coupling); /** Compute advection tendencies for all components of momentum */ void AdvectionSrcForMom (const amrex::Box& bxx, const amrex::Box& bxy, const amrex::Box& bxz, @@ -58,7 +61,7 @@ void AdvectionSrcForMom (const amrex::Box& bxx, const amrex::Box& bxy, const amr const amrex::Array4& mf_u, const amrex::Array4& mf_v, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain, const int domhi_z); + const bool use_terrain, const int domhi_z); AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE diff --git a/Source/Advection/AdvectionSrcForMom.cpp b/Source/Advection/AdvectionSrcForMom.cpp index 2ca74102a..0af4657bd 100644 --- a/Source/Advection/AdvectionSrcForMom.cpp +++ b/Source/Advection/AdvectionSrcForMom.cpp @@ -51,7 +51,7 @@ AdvectionSrcForMom (const Box& bxx, const Box& bxy, const Box& bxz, const Array4& mf_v, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain, + const bool use_terrain, const int domhi_z) { BL_PROFILE_VAR("AdvectionSrcForMom", AdvectionSrcForMom); diff --git a/Source/Advection/AdvectionSrcForMom_N.H b/Source/Advection/AdvectionSrcForMom_N.H index a46807956..d20b504b6 100644 --- a/Source/Advection/AdvectionSrcForMom_N.H +++ b/Source/Advection/AdvectionSrcForMom_N.H @@ -42,22 +42,27 @@ AdvectionSrcForXMom_N (int i, int j, int k, amrex::Real interp_hi(0.), interp_lo(0.); rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) * mf_u_inv(i+1,j,0) + rho_u(i, j, k) * mf_u_inv(i,j,0)); - rho_u_avg_lo = 0.5 * (rho_u(i-1, j, k) * mf_u_inv(i-1,j,0) + rho_u(i, j, k) * mf_u_inv(i,j,0)); - interp_u_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + interp_u_h.InterpolateInX(i+1,j,k,0,interp_hi,rho_u_avg_hi); xflux_hi = rho_u_avg_hi * interp_hi; + + rho_u_avg_lo = 0.5 * (rho_u(i-1, j, k) * mf_u_inv(i-1,j,0) + rho_u(i, j, k) * mf_u_inv(i,j,0)); + interp_u_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); xflux_lo = rho_u_avg_lo * interp_lo; rho_v_avg_hi = 0.5 * (rho_v(i, j+1, k) * mf_v_inv(i,j+1,0) + rho_v(i-1, j+1, k) * mf_v_inv(i-1,j+1,0)); - rho_v_avg_lo = 0.5 * (rho_v(i, j , k) * mf_v_inv(i,j ,0) + rho_v(i-1, j , k) * mf_v_inv(i-1,j ,0)); - interp_u_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + interp_u_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); yflux_hi = rho_v_avg_hi * interp_hi; + + rho_v_avg_lo = 0.5 * (rho_v(i, j , k) * mf_v_inv(i,j ,0) + rho_v(i-1, j , k) * mf_v_inv(i-1,j ,0)); + interp_u_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); yflux_lo = rho_v_avg_lo * interp_lo; rho_w_avg_hi = 0.5 * (rho_w(i, j, k+1) + rho_w(i-1, j, k+1)); - rho_w_avg_lo = 0.5 * (rho_w(i, j, k ) + rho_w(i-1, j, k )); - interp_u_v.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi); - interp_u_v.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo); + interp_u_v.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi); zflux_hi = rho_w_avg_hi * interp_hi; + + rho_w_avg_lo = 0.5 * (rho_w(i, j, k ) + rho_w(i-1, j, k )); + interp_u_v.InterpolateInZ(i,j,k,0,interp_lo,rho_w_avg_lo); zflux_lo = rho_w_avg_lo * interp_lo; amrex::Real mfsq = 1 / (mf_u_inv(i,j,0) * mf_u_inv(i,j,0)); @@ -110,22 +115,27 @@ AdvectionSrcForYMom_N (int i, int j, int k, amrex::Real interp_hi(0.), interp_lo(0.); rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) * mf_u_inv(i+1,j ,0) + rho_u(i+1, j-1, k) * mf_u_inv(i+1,j-1,0)); - rho_u_avg_lo = 0.5 * (rho_u(i , j, k) * mf_u_inv(i ,j ,0) + rho_u(i , j-1, k) * mf_u_inv(i ,j-1,0)); - interp_v_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + interp_v_h.InterpolateInX(i+1,j,k,0,interp_hi,rho_u_avg_hi); xflux_hi = rho_u_avg_hi * interp_hi; + + rho_u_avg_lo = 0.5 * (rho_u(i , j, k) * mf_u_inv(i ,j ,0) + rho_u(i , j-1, k) * mf_u_inv(i ,j-1,0)); + interp_v_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); xflux_lo = rho_u_avg_lo * interp_lo; rho_v_avg_hi = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j+1, k) * mf_v_inv(i ,j+1,0)); - rho_v_avg_lo = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j-1, k) * mf_v_inv(i ,j-1,0)); - interp_v_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + interp_v_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); yflux_hi = rho_v_avg_hi * interp_hi; + + rho_v_avg_lo = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j-1, k) * mf_v_inv(i ,j-1,0)); + interp_v_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); yflux_lo = rho_v_avg_lo * interp_lo; rho_w_avg_hi = 0.5 * (rho_w(i, j, k+1) + rho_w(i, j-1, k+1)); - rho_w_avg_lo = 0.5 * (rho_w(i, j, k ) + rho_w(i, j-1, k )); - interp_v_v.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi); - interp_v_v.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo); + interp_v_v.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi); zflux_hi = rho_w_avg_hi * interp_hi; + + rho_w_avg_lo = 0.5 * (rho_w(i, j, k ) + rho_w(i, j-1, k )); + interp_v_v.InterpolateInZ(i,j,k ,0,interp_lo,rho_w_avg_lo); zflux_lo = rho_w_avg_lo * interp_lo; amrex::Real mfsq = 1 / (mf_v_inv(i,j,0) * mf_v_inv(i,j,0)); @@ -186,15 +196,19 @@ AdvectionSrcForZMom_N (int i, int j, int k, amrex::Real interp_hi(0.), interp_lo(0.); rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) + rho_u(i+1, j, k-1)) * mf_u_inv(i+1,j ,0); - rho_u_avg_lo = 0.5 * (rho_u(i , j, k) + rho_u(i , j, k-1)) * mf_u_inv(i ,j ,0); - interp_w_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + interp_w_h.InterpolateInX(i+1,j,k,0,interp_hi,rho_u_avg_hi); xflux_hi = rho_u_avg_hi * interp_hi; + + rho_u_avg_lo = 0.5 * (rho_u(i , j, k) + rho_u(i , j, k-1)) * mf_u_inv(i ,j ,0); + interp_w_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); xflux_lo = rho_u_avg_lo * interp_lo; rho_v_avg_hi = 0.5 * (rho_v(i, j+1, k) + rho_v(i, j+1, k-1)) * mf_v_inv(i ,j+1,0); - rho_v_avg_lo = 0.5 * (rho_v(i, j , k) + rho_v(i, j , k-1)) * mf_v_inv(i ,j ,0); - interp_w_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + interp_w_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); yflux_hi = rho_v_avg_hi * interp_hi; + + rho_v_avg_lo = 0.5 * (rho_v(i, j , k) + rho_v(i, j , k-1)) * mf_v_inv(i ,j ,0); + interp_w_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); yflux_lo = rho_v_avg_lo * interp_lo; // int l_spatial_order_hi = std::min(std::min(vert_spatial_order, 2*(domhi_z+1-k)), 2*(k+1)); @@ -208,15 +222,15 @@ AdvectionSrcForZMom_N (int i, int j, int k, rho_w_avg_hi = 0.5 * (rho_w(i,j,k) + rho_w(i,j,k+1)); if (k == domhi_z) { - interp_w_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi,AdvType::Centered_2nd); + interp_w_wall.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi,AdvType::Centered_2nd); } else if (k == domhi_z-1 || k == 1) { if (vert_adv_type != AdvType::Centered_2nd && vert_adv_type != AdvType::Upwind_3rd) { - interp_w_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi,AdvType::Centered_4th); + interp_w_wall.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi,AdvType::Centered_4th); } else { - interp_w_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi,vert_adv_type); + interp_w_wall.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi,vert_adv_type); } } else { - interp_w_v.InterpolateInZ_hi(i,j,k,0,interp_hi,rho_w_avg_hi); + interp_w_v.InterpolateInZ(i,j,k+1,0,interp_hi,rho_w_avg_hi); } zflux_hi = rho_w_avg_hi * interp_hi; } @@ -231,15 +245,15 @@ AdvectionSrcForZMom_N (int i, int j, int k, } else { rho_w_avg_lo = 0.5 * (rho_w(i,j,k) + rho_w(i,j,k-1)); if (k == 1) { - interp_w_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo,AdvType::Centered_2nd); + interp_w_wall.InterpolateInZ(i,j,k,0,interp_lo,rho_w_avg_lo,AdvType::Centered_2nd); } else if (k == 2 || k == domhi_z) { if (vert_adv_type != AdvType::Centered_2nd && vert_adv_type != AdvType::Upwind_3rd) { - interp_w_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo,AdvType::Centered_4th); + interp_w_wall.InterpolateInZ(i,j,k,0,interp_lo,rho_w_avg_lo,AdvType::Centered_4th); } else { - interp_w_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo,vert_adv_type); + interp_w_wall.InterpolateInZ(i,j,k,0,interp_lo,rho_w_avg_lo,vert_adv_type); } } else { - interp_w_v.InterpolateInZ_lo(i,j,k,0,interp_lo,rho_w_avg_lo); + interp_w_v.InterpolateInZ(i,j,k,0,interp_lo,rho_w_avg_lo); } zflux_lo = rho_w_avg_lo * interp_lo; } diff --git a/Source/Advection/AdvectionSrcForMom_T.H b/Source/Advection/AdvectionSrcForMom_T.H index a8b7ed79b..e61d22fce 100644 --- a/Source/Advection/AdvectionSrcForMom_T.H +++ b/Source/Advection/AdvectionSrcForMom_T.H @@ -47,9 +47,10 @@ AdvectionSrcForXMom_T (int i, int j, int k, // **************************************************************************************** rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) * mf_u_inv(i+1,j ,0) + rho_u(i, j, k) * mf_u_inv(i ,j ,0)); - rho_u_avg_lo = 0.5 * (rho_u(i-1, j, k) * mf_u_inv(i-1,j ,0) + rho_u(i, j, k) * mf_u_inv(i ,j ,0)); + interp_u_h.InterpolateInX(i,j,k+1,0,interp_hi,rho_u_avg_hi); - interp_u_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + rho_u_avg_lo = 0.5 * (rho_u(i-1, j, k) * mf_u_inv(i-1,j ,0) + rho_u(i, j, k) * mf_u_inv(i ,j ,0)); + interp_u_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); amrex::Real centFluxXXNext = rho_u_avg_hi * Compute_h_zeta_AtCellCenter(i ,j ,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real centFluxXXPrev = rho_u_avg_lo * Compute_h_zeta_AtCellCenter(i-1,j ,k ,cellSizeInv,z_nd) * interp_lo; @@ -58,9 +59,10 @@ AdvectionSrcForXMom_T (int i, int j, int k, // Y-fluxes (at edges in k-direction) // **************************************************************************************** rho_v_avg_hi = 0.5 * (rho_v(i, j+1, k) * mf_v_inv(i ,j+1,0) + rho_v(i-1, j+1, k) * mf_v_inv(i-1,j+1,0)); - rho_v_avg_lo = 0.5 * (rho_v(i, j , k) * mf_v_inv(i ,j ,0) + rho_v(i-1, j , k) * mf_v_inv(i-1,j ,0)); + interp_u_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); - interp_u_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + rho_v_avg_lo = 0.5 * (rho_v(i, j , k) * mf_v_inv(i ,j ,0) + rho_v(i-1, j , k) * mf_v_inv(i-1,j ,0)); + interp_u_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); amrex::Real edgeFluxXYNext = rho_v_avg_hi * Compute_h_zeta_AtEdgeCenterK(i ,j+1,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real edgeFluxXYPrev = rho_v_avg_lo * Compute_h_zeta_AtEdgeCenterK(i ,j ,k ,cellSizeInv,z_nd) * interp_lo; @@ -71,8 +73,8 @@ AdvectionSrcForXMom_T (int i, int j, int k, Omega_avg_hi = 0.5 * (Omega(i, j, k+1) + Omega(i-1, j, k+1)); Omega_avg_lo = 0.5 * (Omega(i, j, k ) + Omega(i-1, j, k )); - interp_u_v.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi); - interp_u_v.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo); + interp_u_v.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi); + interp_u_v.InterpolateInZ(i,j,k ,0,interp_lo,Omega_avg_lo); amrex::Real edgeFluxXZNext = Omega_avg_hi * interp_hi; amrex::Real edgeFluxXZPrev = Omega_avg_lo * interp_lo; @@ -133,9 +135,10 @@ AdvectionSrcForYMom_T (int i, int j, int k, // x-fluxes (at edges in k-direction) // **************************************************************************************** rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) * mf_u_inv(i+1,j ,0) + rho_u(i+1, j-1, k) * mf_u_inv(i+1,j-1,0)); - rho_u_avg_lo = 0.5 * (rho_u(i , j, k) * mf_u_inv(i ,j ,0) + rho_u(i , j-1, k) * mf_u_inv(i ,j-1,0)); + interp_v_h.InterpolateInX(i+1,j,k,0,interp_hi,rho_u_avg_hi); - interp_v_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + rho_u_avg_lo = 0.5 * (rho_u(i , j, k) * mf_u_inv(i ,j ,0) + rho_u(i , j-1, k) * mf_u_inv(i ,j-1,0)); + interp_v_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); amrex::Real edgeFluxYXNext = rho_u_avg_hi * Compute_h_zeta_AtEdgeCenterK(i+1,j ,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real edgeFluxYXPrev = rho_u_avg_lo * Compute_h_zeta_AtEdgeCenterK(i ,j ,k ,cellSizeInv,z_nd) * interp_lo; @@ -144,9 +147,10 @@ AdvectionSrcForYMom_T (int i, int j, int k, // y-fluxes (at cell centers) // **************************************************************************************** rho_v_avg_hi = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j+1, k) * mf_v_inv(i ,j+1,0)); - rho_v_avg_lo = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j-1, k) * mf_v_inv(i ,j-1,0)); + interp_v_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); - interp_v_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + rho_v_avg_lo = 0.5 * (rho_v(i, j, k) * mf_v_inv(i ,j ,0) + rho_v(i, j-1, k) * mf_v_inv(i ,j-1,0)); + interp_v_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); amrex::Real centFluxYYNext = rho_v_avg_hi * Compute_h_zeta_AtCellCenter(i ,j ,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real centFluxYYPrev = rho_v_avg_lo * Compute_h_zeta_AtCellCenter(i ,j-1,k ,cellSizeInv,z_nd) * interp_lo; @@ -157,8 +161,8 @@ AdvectionSrcForYMom_T (int i, int j, int k, Omega_avg_hi = 0.5 * (Omega(i, j, k+1) + Omega(i, j-1, k+1)); Omega_avg_lo = 0.5 * (Omega(i, j, k ) + Omega(i, j-1, k )); - interp_v_v.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi); - interp_v_v.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo); + interp_v_v.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi); + interp_v_v.InterpolateInZ(i,j,k ,0,interp_lo,Omega_avg_lo); amrex::Real edgeFluxYZNext = Omega_avg_hi * interp_hi; amrex::Real edgeFluxYZPrev = Omega_avg_lo * interp_lo; @@ -227,9 +231,10 @@ AdvectionSrcForZMom_T (int i, int j, int k, // x-fluxes (at edges in j-direction) // **************************************************************************************** rho_u_avg_hi = 0.5 * (rho_u(i+1, j, k) + rho_u(i+1, j, k-1)) * mf_u_inv(i+1,j ,0); - rho_u_avg_lo = 0.5 * (rho_u(i , j, k) + rho_u(i , j, k-1)) * mf_u_inv(i ,j ,0); + interp_omega_h.InterpolateInX(i+1,j,k,0,interp_hi,rho_u_avg_hi); - interp_omega_h.InterpolateInX(i,j,k,0,interp_hi,interp_lo,rho_u_avg_hi,rho_u_avg_lo); + rho_u_avg_lo = 0.5 * (rho_u(i , j, k) + rho_u(i , j, k-1)) * mf_u_inv(i ,j ,0); + interp_omega_h.InterpolateInX(i,j,k,0,interp_lo,rho_u_avg_lo); amrex::Real edgeFluxZXNext = rho_u_avg_hi * Compute_h_zeta_AtEdgeCenterJ(i+1,j ,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real edgeFluxZXPrev = rho_u_avg_lo * Compute_h_zeta_AtEdgeCenterJ(i ,j ,k ,cellSizeInv,z_nd) * interp_lo; @@ -238,9 +243,10 @@ AdvectionSrcForZMom_T (int i, int j, int k, // y-fluxes (at edges in i-direction) // **************************************************************************************** rho_v_avg_hi = 0.5 * (rho_v(i, j+1, k) + rho_v(i, j+1, k-1)) * mf_v_inv(i ,j+1,0); - rho_v_avg_lo = 0.5 * (rho_v(i, j , k) + rho_v(i, j , k-1)) * mf_v_inv(i ,j ,0); + interp_omega_h.InterpolateInY(i,j+1,k,0,interp_hi,rho_v_avg_hi); - interp_omega_h.InterpolateInY(i,j,k,0,interp_hi,interp_lo,rho_v_avg_hi,rho_v_avg_lo); + rho_v_avg_lo = 0.5 * (rho_v(i, j , k) + rho_v(i, j , k-1)) * mf_v_inv(i ,j ,0); + interp_omega_h.InterpolateInY(i,j,k,0,interp_lo,rho_v_avg_lo); amrex::Real edgeFluxZYNext = rho_v_avg_hi * Compute_h_zeta_AtEdgeCenterI(i ,j+1,k ,cellSizeInv,z_nd) * interp_hi; amrex::Real edgeFluxZYPrev = rho_v_avg_lo * Compute_h_zeta_AtEdgeCenterI(i ,j ,k ,cellSizeInv,z_nd) * interp_lo; @@ -260,15 +266,15 @@ AdvectionSrcForZMom_T (int i, int j, int k, centFluxZZNext *= w(i,j,k); } else { if (k == domhi_z) { - interp_omega_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi,AdvType::Centered_2nd); + interp_omega_wall.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi,AdvType::Centered_2nd); } else if (k == domhi_z-1 || k == 1) { if (vert_adv_type != AdvType::Centered_2nd && vert_adv_type != AdvType::Upwind_3rd) { - interp_omega_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi,AdvType::Centered_4th); + interp_omega_wall.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi,AdvType::Centered_4th); } else { - interp_omega_wall.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi,vert_adv_type); + interp_omega_wall.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi,vert_adv_type); } } else { - interp_omega_v.InterpolateInZ_hi(i,j,k,0,interp_hi,Omega_avg_hi); + interp_omega_v.InterpolateInZ(i,j,k+1,0,interp_hi,Omega_avg_hi); } centFluxZZNext *= interp_hi; } @@ -286,15 +292,15 @@ AdvectionSrcForZMom_T (int i, int j, int k, centFluxZZPrev *= w(i,j,k); } else { if (k == 1) { - interp_omega_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo,AdvType::Centered_2nd); + interp_omega_wall.InterpolateInZ(i,j,k,0,interp_lo,Omega_avg_lo,AdvType::Centered_2nd); } else if (k == 2 || k == domhi_z) { if (vert_adv_type != AdvType::Centered_2nd && vert_adv_type != AdvType::Upwind_3rd) { - interp_omega_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo,AdvType::Centered_4th); + interp_omega_wall.InterpolateInZ(i,j,k,0,interp_lo,Omega_avg_lo,AdvType::Centered_4th); } else { - interp_omega_wall.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo,vert_adv_type); + interp_omega_wall.InterpolateInZ(i,j,k,0,interp_lo,Omega_avg_lo,vert_adv_type); } } else { - interp_omega_v.InterpolateInZ_lo(i,j,k,0,interp_lo,Omega_avg_lo); + interp_omega_v.InterpolateInZ(i,j,k,0,interp_lo,Omega_avg_lo); } centFluxZZPrev *= interp_lo; } diff --git a/Source/Advection/AdvectionSrcForScalars.H b/Source/Advection/AdvectionSrcForScalars.H new file mode 100644 index 000000000..44e5865a3 --- /dev/null +++ b/Source/Advection/AdvectionSrcForScalars.H @@ -0,0 +1,101 @@ +#include +#include + +/** + * Wrapper function for computing the advective tendency w/ spatial order > 2. + */ +template +void +AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx, + const int& ncomp, const int& icomp, + const amrex::GpuArray, AMREX_SPACEDIM> flx_arr, + const amrex::Array4& cell_prim, + const amrex::Array4& avg_xmom, + const amrex::Array4& avg_ymom, + const amrex::Array4& avg_zmom) +{ + // Instantiate structs for vert/horiz interp + InterpType_H interp_prim_h(cell_prim); + InterpType_V interp_prim_v(cell_prim); + + const amrex::Box xbx = amrex::surroundingNodes(bx,0); + const amrex::Box ybx = amrex::surroundingNodes(bx,1); + const amrex::Box zbx = amrex::surroundingNodes(bx,2); + + amrex::ParallelFor(xbx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + const int cons_index = icomp + n; + const int prim_index = cons_index - 1; + + amrex::Real interpx(0.); + + interp_prim_h.InterpolateInX(i,j,k,prim_index,interpx,avg_xmom(i ,j ,k )); + (flx_arr[0])(i,j,k,n) = avg_xmom(i,j,k) * interpx; + }); + amrex::ParallelFor(ybx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + const int cons_index = icomp + n; + const int prim_index = cons_index - 1; + + amrex::Real interpy(0.); + interp_prim_h.InterpolateInY(i,j,k,prim_index,interpy,avg_ymom(i ,j ,k )); + + (flx_arr[1])(i,j,k,n) = avg_ymom(i,j,k) * interpy; + }); + amrex::ParallelFor(zbx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + const int cons_index = icomp + n; + const int prim_index = cons_index - 1; + + amrex::Real interpz(0.); + + interp_prim_v.InterpolateInZ(i,j,k,prim_index,interpz,avg_zmom(i ,j ,k )); + + (flx_arr[2])(i,j,k,n) = avg_zmom(i,j,k) * interpz; + }); +} + +/** + * Wrapper function for templating the vertical advective tendency w/ spatial order > 2. + */ +template +void +AdvectionSrcForScalarsVert_N(const amrex::Box& bx, + const int& ncomp, const int& icomp, + const amrex::GpuArray, AMREX_SPACEDIM> flx_arr, + const amrex::Array4& cell_prim, + const amrex::Array4& avg_xmom, + const amrex::Array4& avg_ymom, + const amrex::Array4& avg_zmom, + const AdvType vert_adv_type) +{ + switch(vert_adv_type) { + case AdvType::Centered_2nd: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, + flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Upwind_3rd: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, + flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Centered_4th: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, + flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Upwind_5th: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, + flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Centered_6th: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, + flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + default: + AMREX_ASSERT_WITH_MESSAGE(false, "Unknown vertical advection scheme!"); + } +} diff --git a/Source/Advection/AdvectionSrcForState.cpp b/Source/Advection/AdvectionSrcForState.cpp index ae260a59c..4be573815 100644 --- a/Source/Advection/AdvectionSrcForState.cpp +++ b/Source/Advection/AdvectionSrcForState.cpp @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include using namespace amrex; @@ -51,7 +53,7 @@ AdvectionSrcForRhoAndTheta (const Box& bx, const Box& valid_bx, const Array4& mf_v, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain) + const bool use_terrain) { BL_PROFILE_VAR("AdvectionSrcForRhoAndTheta", AdvectionSrcForRhoAndTheta); auto dxInv = cellSizeInv[0], dyInv = cellSizeInv[1], dzInv = cellSizeInv[2]; @@ -100,37 +102,43 @@ AdvectionSrcForRhoAndTheta (const Box& bx, const Box& valid_bx, }); // Template higher order methods } else { - if (horiz_adv_type == AdvType::Centered_2nd) { + switch(horiz_adv_type) { + case AdvType::Centered_2nd: AdvectionSrcForRhoThetaVert_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_3rd) { + break; + case AdvType::Upwind_3rd: AdvectionSrcForRhoThetaVert_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_4th) { + break; + case AdvType::Centered_4th: AdvectionSrcForRhoThetaVert_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_5th) { + break; + case AdvType::Upwind_5th: AdvectionSrcForRhoThetaVert_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_6th) { + break; + case AdvType::Centered_6th: AdvectionSrcForRhoThetaVert_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else { + break; + default: AMREX_ASSERT_WITH_MESSAGE(false, "Unknown advection scheme!"); } } @@ -188,37 +196,43 @@ AdvectionSrcForRhoAndTheta (const Box& bx, const Box& valid_bx, }); // Template higher order methods (horizontal first) } else { - if (horiz_adv_type == AdvType::Centered_2nd) { + switch(horiz_adv_type) { + case AdvType::Centered_2nd: AdvectionSrcForRhoThetaVert_T(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, z_nd, detJ, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_3rd) { + break; + case AdvType::Upwind_3rd: AdvectionSrcForRhoThetaVert_T(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, z_nd, detJ, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_4th) { + break; + case AdvType::Centered_4th: AdvectionSrcForRhoThetaVert_T(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, z_nd, detJ, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_5th) { + break; + case AdvType::Upwind_5th: AdvectionSrcForRhoThetaVert_T(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, z_nd, detJ, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_6th) { + break; + case AdvType::Centered_6th: AdvectionSrcForRhoThetaVert_T(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, Omega, avg_xmom, avg_ymom, avg_zmom, z_nd, detJ, cellSizeInv, mf_m, mf_u, mf_v, vert_adv_type); - } else { + break; + default: AMREX_ASSERT_WITH_MESSAGE(false, "Unknown advection scheme!"); } } @@ -248,98 +262,141 @@ AdvectionSrcForRhoAndTheta (const Box& bx, const Box& valid_bx, */ void -AdvectionSrcForScalars (const Box& bx, const int icomp, const int ncomp, +AdvectionSrcForScalars (int level, int finest_level, + const MFIter& mfi, + const Box& bx, const int icomp, const int ncomp, const Array4& avg_xmom, const Array4& avg_ymom, const Array4& avg_zmom, const Array4& cell_prim, const Array4& advectionSrc, const Array4& detJ, + const Real dt, const GpuArray& cellSizeInv, const Array4& mf_m, const AdvType horiz_adv_type, const AdvType vert_adv_type, - const int use_terrain) + const bool use_terrain, + const bool two_way_coupling) { BL_PROFILE_VAR("AdvectionSrcForScalars", AdvectionSrcForScalars); - auto dxInv = cellSizeInv[0], dyInv = cellSizeInv[1], dzInv = cellSizeInv[2]; + auto dxInv = cellSizeInv[0], dyInv = cellSizeInv[1], dzInv = cellSizeInv[2]; + + amrex::GpuArray flux; + for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { + const Box& efbx = amrex::surroundingNodes(bx, dir); + flux[dir].resize(efbx, ncomp, amrex::The_Async_Arena()); + flux[dir].setVal(0.); + } + + const GpuArray, AMREX_SPACEDIM> + flx_arr{{AMREX_D_DECL(flux[0].array(), flux[1].array(), flux[2].array())}}; + + const Box xbx = surroundingNodes(bx,0); + const Box ybx = surroundingNodes(bx,1); + const Box zbx = surroundingNodes(bx,2); // Inline with 2nd order for efficiency + // NOTE: we don't need to weight avg_xmom, avg_ymom, avg_zmom with terrain metrics + // because that was done when they were constructed in AdvectionSrcForRhoAndTheta if (horiz_adv_type == AdvType::Centered_2nd && vert_adv_type == AdvType::Centered_2nd) { - amrex::ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + amrex::ParallelFor(xbx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept { - Real invdetJ = (use_terrain) ? 1. / detJ(i,j,k) : 1.; - - // NOTE: we don't need to weight avg_xmom, avg_ymom, avg_zmom with terrain metrics - // because that was done when they were constructed in AdvectionSrcForRhoAndTheta - const int cons_index = icomp + n; const int prim_index = cons_index - 1; - - Real mfsq = mf_m(i,j,0) * mf_m(i,j,0); - - advectionSrc(i,j,k,cons_index) = - 0.5 * invdetJ * ( - ( avg_xmom(i+1,j,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i+1,j,k,prim_index)) - - avg_xmom(i ,j,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i-1,j,k,prim_index)) ) * dxInv * mfsq + - ( avg_ymom(i,j+1,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j+1,k,prim_index)) - - avg_ymom(i,j ,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j-1,k,prim_index)) ) * dyInv * mfsq + - ( avg_zmom(i,j,k+1) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j,k+1,prim_index)) - - avg_zmom(i,j,k ) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j,k-1,prim_index)) ) * dzInv); + (flx_arr[0])(i,j,k,n) = 0.5 * avg_xmom(i,j,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i-1,j,k,prim_index)); + }); + amrex::ParallelFor(ybx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + const int cons_index = icomp + n; + const int prim_index = cons_index - 1; + (flx_arr[1])(i,j,k,n) = 0.5 * avg_ymom(i,j,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j-1,k,prim_index)); + }); + amrex::ParallelFor(zbx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + const int cons_index = icomp + n; + const int prim_index = cons_index - 1; + (flx_arr[2])(i,j,k,n) = 0.5 * avg_zmom(i,j,k) * (cell_prim(i,j,k,prim_index) + cell_prim(i,j,k-1,prim_index)); }); + // Template higher order methods (horizontal first) } else { - if (horiz_adv_type == AdvType::Centered_2nd) { - AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_3rd) { - AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_4th) { - AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m, vert_adv_type); - } else if (horiz_adv_type == AdvType::Upwind_5th) { - AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m, vert_adv_type); - } else if (horiz_adv_type == AdvType::Centered_6th) { - AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m, vert_adv_type); - } else if (horiz_adv_type == AdvType::Weno_3) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (horiz_adv_type == AdvType::Weno_5) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (horiz_adv_type == AdvType::Weno_3Z) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (horiz_adv_type == AdvType::Weno_3MZQ) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (horiz_adv_type == AdvType::Weno_5Z) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else { + switch(horiz_adv_type) { + case AdvType::Centered_2nd: + AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom, vert_adv_type); + break; + case AdvType::Upwind_3rd: + AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom, vert_adv_type); + break; + case AdvType::Centered_4th: + AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom, vert_adv_type); + break; + case AdvType::Upwind_5th: + AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom, vert_adv_type); + break; + case AdvType::Centered_6th: + AdvectionSrcForScalarsVert_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom, vert_adv_type); + break; + case AdvType::Weno_3: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Weno_5: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Weno_3Z: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Weno_3MZQ: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + case AdvType::Weno_5Z: + AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, flx_arr, cell_prim, + avg_xmom, avg_ymom, avg_zmom); + break; + default: AMREX_ASSERT_WITH_MESSAGE(false, "Unknown advection scheme!"); } } + + amrex::ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + Real invdetJ = (use_terrain) ? 1. / detJ(i,j,k) : 1.; + + Real mfsq = mf_m(i,j,0) * mf_m(i,j,0); + + const int cons_index = icomp + n; + advectionSrc(i,j,k,cons_index) = - invdetJ * ( + ( (flx_arr[0])(i+1,j,k,n) - (flx_arr[0])(i ,j,k,n) ) * dxInv * mfsq + + ( (flx_arr[1])(i,j+1,k,n) - (flx_arr[1])(i,j ,k,n) ) * dyInv * mfsq + + ( (flx_arr[2])(i,j,k+1,n) - (flx_arr[2])(i,j,k ,n) ) * dzInv ); + }); + +#if 0 + Real fac_for_reflux = 1.0; + + std::array dxD = {{dx, dy, dz}}; + const Real* dxDp = &(dxD[0]); + + if (two_way_coupling) { + if (level < finest_level) { + getAdvFluxReg(level + 1).CrseAdd(mfi, + {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}}, + dxDp, fac_for_reflux*dt, amrex::RunOn::Device); + } + if (level > 0) { + getAdvFluxReg(level).FineAdd(mfi, + {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}}, + dxDp, fac_for_reflux*dt, amrex::RunOn::Device); + } + } // two-way coupling +#endif } diff --git a/Source/Advection/AdvectionSrcForState_N.H b/Source/Advection/AdvectionSrcForState_N.H index e594fa1af..3758df06f 100644 --- a/Source/Advection/AdvectionSrcForState_N.H +++ b/Source/Advection/AdvectionSrcForState_N.H @@ -58,11 +58,14 @@ AdvectionSrcForRhoThetaWrapper_N(const amrex::Box& bx, amrex::Real interpy_hi(0.), interpy_lo(0.); amrex::Real interpz_hi(0.), interpz_lo(0.); - interp_prim_h.InterpolateInX(i,j,k,prim_index,interpx_hi,interpx_lo,rho_u(i+1,j ,k ),rho_u(i ,j ,k )); - interp_prim_h.InterpolateInY(i,j,k,prim_index,interpy_hi,interpy_lo,rho_v(i ,j+1,k ),rho_v(i ,j ,k )); + interp_prim_h.InterpolateInX(i+1,j,k,prim_index,interpx_hi,rho_u(i+1,j ,k )); + interp_prim_h.InterpolateInX(i ,j,k,prim_index,interpx_lo,rho_u(i ,j ,k )); - interp_prim_v.InterpolateInZ_hi(i,j,k,prim_index,interpz_hi,rho_w(i ,j ,k+1)); - interp_prim_v.InterpolateInZ_lo(i,j,k,prim_index,interpz_lo,rho_w(i ,j ,k )); + interp_prim_h.InterpolateInY(i,j+1,k,prim_index,interpy_hi,rho_v(i ,j+1,k )); + interp_prim_h.InterpolateInY(i,j ,k,prim_index,interpy_lo,rho_v(i ,j ,k )); + + interp_prim_v.InterpolateInZ(i,j,k+1,prim_index,interpz_hi,rho_w(i ,j ,k+1)); + interp_prim_v.InterpolateInZ(i,j,k ,prim_index,interpz_lo,rho_w(i ,j ,k )); advectionSrc(i,j,k,1) = -( ( xflux_hi * interpx_hi - xflux_lo * interpx_lo ) * dxInv * mfsq + @@ -93,128 +96,39 @@ AdvectionSrcForRhoThetaVert_N(const amrex::Box& bx, const amrex::Array4& mf_v, const AdvType vert_adv_type) { - if (vert_adv_type == AdvType::Centered_2nd) { + switch(vert_adv_type) { + case AdvType::Centered_2nd: AdvectionSrcForRhoThetaWrapper_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, rho_w, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v); - } else if (vert_adv_type == AdvType::Upwind_3rd) { + break; + case AdvType::Upwind_3rd: AdvectionSrcForRhoThetaWrapper_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, rho_w, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v); - } else if (vert_adv_type == AdvType::Centered_4th) { + break; + case AdvType::Centered_4th: AdvectionSrcForRhoThetaWrapper_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, rho_w, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v); - } else if (vert_adv_type == AdvType::Upwind_5th) { + break; + case AdvType::Upwind_5th: AdvectionSrcForRhoThetaWrapper_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, rho_w, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v); - } else if (vert_adv_type == AdvType::Centered_6th) { + break; + case AdvType::Centered_6th: AdvectionSrcForRhoThetaWrapper_N(bx, vbx_hi, fac, advectionSrc, cell_prim, rho_u, rho_v, rho_w, avg_xmom, avg_ymom, avg_zmom, cellSizeInv, mf_m, mf_u, mf_v); - } else { + break; + default: AMREX_ASSERT_WITH_MESSAGE(false, "Unknown advection scheme!"); } } -/** - * Wrapper function for computing the advective tendency w/ spatial order > 2. - */ -template -void -AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx, - const int& ncomp, const int& icomp, const int& use_terrain, - const amrex::Array4< amrex::Real>& advectionSrc, - const amrex::Array4& cell_prim, - const amrex::Array4& avg_xmom, - const amrex::Array4& avg_ymom, - const amrex::Array4& avg_zmom, - const amrex::Array4& detJ, - const amrex::GpuArray& cellSizeInv, - const amrex::Array4& mf_m) -{ - // Instantiate structs for vert/horiz interp - InterpType_H interp_prim_h(cell_prim); - InterpType_V interp_prim_v(cell_prim); - - auto dxInv = cellSizeInv[0], dyInv = cellSizeInv[1], dzInv = cellSizeInv[2]; - amrex::ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept - { - amrex::Real invdetJ = (use_terrain) ? 1. / detJ(i,j,k) : 1.; - amrex::Real mfsq = mf_m(i,j,0) * mf_m(i,j,0); - - // NOTE: we don't need to weight avg_xmom, avg_ymom, avg_zmom with terrain metrics - // because that was done when they were constructed in AdvectionSrcForRhoAndTheta - - const int cons_index = icomp + n; - const int prim_index = cons_index - 1; - - amrex::Real interpx_hi(0.), interpx_lo(0.); - amrex::Real interpy_hi(0.), interpy_lo(0.); - amrex::Real interpz_hi(0.), interpz_lo(0.); - - interp_prim_h.InterpolateInX(i,j,k,prim_index,interpx_hi,interpx_lo,avg_xmom(i+1,j ,k ),avg_xmom(i ,j ,k )); - interp_prim_h.InterpolateInY(i,j,k,prim_index,interpy_hi,interpy_lo,avg_ymom(i ,j+1,k ),avg_ymom(i ,j ,k )); - - interp_prim_v.InterpolateInZ_hi(i,j,k,prim_index,interpz_hi,avg_zmom(i ,j ,k+1)); - interp_prim_v.InterpolateInZ_lo(i,j,k,prim_index,interpz_lo,avg_zmom(i ,j ,k )); - - advectionSrc(i,j,k,cons_index) = - invdetJ * ( - ( avg_xmom(i+1,j ,k ) * interpx_hi - avg_xmom(i ,j ,k ) * interpx_lo ) * dxInv * mfsq + - ( avg_ymom(i ,j+1,k ) * interpy_hi - avg_ymom(i ,j ,k ) * interpy_lo ) * dyInv * mfsq + - ( avg_zmom(i ,j ,k+1) * interpz_hi - avg_zmom(i ,j ,k ) * interpz_lo ) * dzInv); - }); -} - -/** - * Wrapper function for templating the vertical advective tendency w/ spatial order > 2. - */ -template -void -AdvectionSrcForScalarsVert_N(const amrex::Box& bx, - const int& ncomp, const int& icomp, const int& use_terrain, - const amrex::Array4< amrex::Real>& advectionSrc, - const amrex::Array4& cell_prim, - const amrex::Array4& avg_xmom, - const amrex::Array4& avg_ymom, - const amrex::Array4& avg_zmom, - const amrex::Array4& detJ, - const amrex::GpuArray& cellSizeInv, - const amrex::Array4& mf_m, - const AdvType vert_adv_type) -{ - if (vert_adv_type == AdvType::Centered_2nd) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (vert_adv_type == AdvType::Upwind_3rd) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (vert_adv_type == AdvType::Centered_4th) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (vert_adv_type == AdvType::Upwind_5th) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else if (vert_adv_type == AdvType::Centered_6th) { - AdvectionSrcForScalarsWrapper_N(bx, ncomp, icomp, - use_terrain, advectionSrc, cell_prim, - avg_xmom, avg_ymom, avg_zmom, detJ, - cellSizeInv, mf_m); - } else { - AMREX_ASSERT_WITH_MESSAGE(false, "Unknown advection scheme!"); - } -} diff --git a/Source/Advection/AdvectionSrcForState_T.H b/Source/Advection/AdvectionSrcForState_T.H index b04fb42b8..79ee4e0a9 100644 --- a/Source/Advection/AdvectionSrcForState_T.H +++ b/Source/Advection/AdvectionSrcForState_T.H @@ -73,11 +73,14 @@ AdvectionSrcForRhoThetaWrapper_T(const amrex::Box& bx, amrex::Real interpy_hi(0.), interpy_lo(0.); amrex::Real interpz_hi(0.), interpz_lo(0.); - interp_prim_h.InterpolateInX(i,j,k,prim_index,interpx_hi,interpx_lo,rho_u(i+1,j ,k ),rho_u(i ,j ,k )); - interp_prim_h.InterpolateInY(i,j,k,prim_index,interpy_hi,interpy_lo,rho_v(i ,j+1,k ),rho_v(i ,j ,k )); + interp_prim_h.InterpolateInX(i+1,j,k,prim_index,interpx_hi,rho_u(i+1,j ,k )); + interp_prim_h.InterpolateInX(i ,j,k,prim_index,interpx_lo,rho_u(i ,j ,k )); - interp_prim_v.InterpolateInZ_hi(i,j,k,prim_index,interpz_hi,Omega(i ,j ,k+1)); - interp_prim_v.InterpolateInZ_lo(i,j,k,prim_index,interpz_lo,Omega(i ,j ,k )); + interp_prim_h.InterpolateInY(i,j+1,k,prim_index,interpy_hi,rho_v(i ,j+1,k )); + interp_prim_h.InterpolateInY(i,j ,k,prim_index,interpy_lo,rho_v(i ,j ,k )); + + interp_prim_v.InterpolateInZ(i,j,k+1,prim_index,interpz_hi,Omega(i ,j ,k+1)); + interp_prim_v.InterpolateInZ(i,j,k ,prim_index,interpz_lo,Omega(i ,j ,k )); advectionSrc(i,j,k,1) = - invdetJ * ( ( xflux_hi * interpx_hi - xflux_lo * interpx_lo ) * dxInv * mfsq + diff --git a/Source/Advection/Make.package b/Source/Advection/Make.package index 4a77fb744..d2bcefa7f 100644 --- a/Source/Advection/Make.package +++ b/Source/Advection/Make.package @@ -6,4 +6,5 @@ CEXE_headers += AdvectionSrcForMom_N.H CEXE_headers += AdvectionSrcForMom_T.H CEXE_headers += AdvectionSrcForState_N.H CEXE_headers += AdvectionSrcForState_T.H +CEXE_headers += AdvectionSrcForScalars.H CEXE_headers += Interpolation.H diff --git a/Source/BoundaryConditions/ABLMost.H b/Source/BoundaryConditions/ABLMost.H index 9d6fd6bac..2b805a060 100644 --- a/Source/BoundaryConditions/ABLMost.H +++ b/Source/BoundaryConditions/ABLMost.H @@ -33,8 +33,14 @@ public: amrex::Vector>& vars_old, amrex::Vector>& Theta_prim, amrex::Vector>& z_phys_nd, - int ng_in) - : m_geom(geom), m_ma(geom,vars_old,Theta_prim,z_phys_nd) + amrex::Vector>>& sst_lev, + amrex::Vector>>& lmask_lev, + amrex::Real start_bdy_time = 0.0, + amrex::Real bdy_time_interval = 0.0) + : m_start_bdy_time(start_bdy_time), + m_bdy_time_interval(bdy_time_interval), + m_geom(geom), + m_ma(geom,vars_old,Theta_prim,z_phys_nd) { amrex::ParmParse pp("erf"); pp.query("most.z0", z0_const); @@ -86,6 +92,7 @@ public: amrex::Abort("Undefined MOST roughness type!"); } + // Size the MOST params for all levels int nlevs = m_geom.size(); z_0.resize(nlevs); u_star.resize(nlevs); @@ -93,11 +100,37 @@ public: t_surf.resize(nlevs); olen.resize(nlevs); - for (int lev = 0; lev < nlevs; lev++) - { - // Z0 heights + // Get pointers to SST and LANDMASK data + m_sst_lev.resize(nlevs); + m_lmask_lev.resize(nlevs); + for (int lev(0); lev(ba2d,dm,ncomp,ng); - u_star[lev]->setVal(1.E34); - - t_star[lev] = std::make_unique(ba2d,dm,ncomp,ng); - t_star[lev]->setVal(1.E34); - - olen[lev] = std::make_unique(ba2d,dm,ncomp,ng); - olen[lev]->setVal(1.E34); - - t_surf[lev] = std::make_unique(ba2d,dm,ncomp,ng); - if (erf_st) { - t_surf[lev]->setVal(surf_temp); - } else { - t_surf[lev]->setVal(0.0); - } + u_star[lev] = std::make_unique(ba2d,dm,ncomp,ng); + u_star[lev]->setVal(1.E34); + + t_star[lev] = std::make_unique(ba2d,dm,ncomp,ng); + t_star[lev]->setVal(1.E34); + + olen[lev] = std::make_unique(ba2d,dm,ncomp,ng); + olen[lev]->setVal(1.E34); + + t_surf[lev] = std::make_unique(ba2d,dm,ncomp,ng); + + if (m_sst_lev[lev][0]) { // Valid SST data at t==0 + theta_type = ThetaCalcType::SURFACE_TEMPERATURE; + amrex::MultiFab::Copy(*(t_surf[lev]), *(m_sst_lev[lev][0]), 0, 0, 1, ng); + } else if (erf_st) { // Constant temp + t_surf[lev]->setVal(surf_temp); + } else { + t_surf[lev]->setVal(0.0); } }// lev } void - update_surf_temp (amrex::Real cur_time) - { - if (surf_heating_rate != 0) { - int nlevs = m_geom.size(); - for (int lev = 0; lev < nlevs; lev++) - { - t_surf[lev]->setVal(surf_temp + surf_heating_rate*cur_time); - amrex::Print() << "Surface temp at t=" << cur_time - << ": " - << surf_temp + surf_heating_rate*cur_time - << std::endl; - } - } - } + update_fluxes (const int& lev, + const amrex::Real& time, + int max_iters = 25); + + template + void + compute_fluxes (const int& lev, + const int& max_iters, + const FluxIter& most_flux); void - impose_most_bcs (const int lev, + impose_most_bcs (const int& lev, const amrex::Vector& mfs, amrex::MultiFab* eddyDiffs, amrex::MultiFab* z_phys); template void - compute_most_bcs (const int lev, + compute_most_bcs (const int& lev, const amrex::Vector& mfs, amrex::MultiFab* eddyDiffs, amrex::MultiFab* z_phys, @@ -169,33 +187,42 @@ public: const FluxCalc& flux_comp); void - update_fluxes (int lev, - amrex::Real cur_time, - int max_iters = 25); + time_interp_tsurf(const int& lev, + const amrex::Real& time); - template void - compute_fluxes (const int& lev, - const int& max_iters, - const FluxIter& most_flux); + update_surf_temp (const amrex::Real& time) + { + if (surf_heating_rate != 0) { + int nlevs = m_geom.size(); + for (int lev = 0; lev < nlevs; lev++) + { + t_surf[lev]->setVal(surf_temp + surf_heating_rate*time); + amrex::Print() << "Surface temp at t=" << time + << ": " + << surf_temp + surf_heating_rate*time + << std::endl; + } + } + } void - update_mac_ptrs (int lev, + update_mac_ptrs (const int& lev, amrex::Vector>& vars_old, amrex::Vector>& Theta_prim) { m_ma.update_field_ptrs(lev,vars_old,Theta_prim); } const amrex::MultiFab* - get_u_star (int lev) { return u_star[lev].get(); } + get_u_star (const int& lev) { return u_star[lev].get(); } const amrex::MultiFab* - get_t_star (int lev) { return t_star[lev].get(); } + get_t_star (const int& lev) { return t_star[lev].get(); } const amrex::MultiFab* - get_olen (int lev) { return olen[lev].get(); } + get_olen (const int& lev) { return olen[lev].get(); } const amrex::MultiFab* - get_mac_avg (int lev, int comp) { return m_ma.get_average(lev,comp); } + get_mac_avg (const int& lev, int comp) { return m_ma.get_average(lev,comp); } enum struct FluxCalcType { MOENG = 0, ///< Moeng functional form @@ -225,6 +252,8 @@ private: amrex::Real surf_temp_flux{0}; amrex::Real cnk_a{0.0185}; amrex::Real depth{30.0}; + amrex::Real m_start_bdy_time; + amrex::Real m_bdy_time_interval; amrex::Vector m_geom; amrex::Vector z_0; @@ -233,6 +262,9 @@ private: amrex::Vector> t_star; amrex::Vector> olen; amrex::Vector> t_surf; + + amrex::Vector> m_sst_lev; + amrex::Vector> m_lmask_lev; }; #endif /* ABLMOST_H */ diff --git a/Source/BoundaryConditions/ABLMost.cpp b/Source/BoundaryConditions/ABLMost.cpp index c7daf5bbe..e088eeee3 100644 --- a/Source/BoundaryConditions/ABLMost.cpp +++ b/Source/BoundaryConditions/ABLMost.cpp @@ -10,10 +10,13 @@ using namespace amrex; * @param[in] max_iters maximum iterations to use */ void -ABLMost::update_fluxes (int lev, - Real cur_time, +ABLMost::update_fluxes (const int& lev, + const Real& time, int max_iters) { + // Update SST data if we have a valid pointer + if (m_sst_lev[lev][0]) time_interp_tsurf(lev, time); + // Compute plane averages for all vars (regardless of flux type) m_ma.compute_averages(lev); @@ -31,7 +34,7 @@ ABLMost::update_fluxes (int lev, compute_fluxes(lev, max_iters, most_flux); } } else if (theta_type == ThetaCalcType::SURFACE_TEMPERATURE) { - update_surf_temp(cur_time); + update_surf_temp(time); if (rough_type == RoughCalcType::CONSTANT) { surface_temp most_flux(m_ma.get_zref(), surf_temp_flux); compute_fluxes(lev, max_iters, most_flux); @@ -75,12 +78,9 @@ ABLMost::compute_fluxes (const int& lev, const auto *const tm_ptr = m_ma.get_average(lev,2); const auto *const umm_ptr = m_ma.get_average(lev,3); - // Ghost cells - amrex::IntVect ng = u_star[lev]->nGrowVect(); ng[2]=0; - for (MFIter mfi(*u_star[lev]); mfi.isValid(); ++mfi) { - amrex::Box gtbx = mfi.growntilebox(ng); + Box gtbx = mfi.growntilebox(); auto u_star_arr = u_star[lev]->array(mfi); auto t_star_arr = t_star[lev]->array(mfi); @@ -108,7 +108,7 @@ ABLMost::compute_fluxes (const int& lev, * @param[in] eddyDiffs Diffusion coefficients from turbulence model */ void -ABLMost::impose_most_bcs (const int lev, +ABLMost::impose_most_bcs (const int& lev, const Vector& mfs, MultiFab* eddyDiffs, MultiFab* z_phys) @@ -134,7 +134,7 @@ ABLMost::impose_most_bcs (const int lev, */ template void -ABLMost::compute_most_bcs (const int lev, +ABLMost::compute_most_bcs (const int& lev, const Vector& mfs, MultiFab* eddyDiffs, MultiFab* z_phys, @@ -218,3 +218,33 @@ ABLMost::compute_most_bcs (const int lev, } // var_idx } // mfiter } + +void +ABLMost::time_interp_tsurf(const int& lev, + const Real& time) +{ + // Time interpolation + Real dT = m_bdy_time_interval; + Real time_since_start = time - m_start_bdy_time; + int n_time = static_cast( time_since_start / dT); + amrex::Real alpha = (time_since_start - n_time * dT) / dT; + AMREX_ALWAYS_ASSERT( alpha >= 0. && alpha <= 1.0); + amrex::Real oma = 1.0 - alpha; + AMREX_ALWAYS_ASSERT( (n_time >= 0) && (n_time < (m_sst_lev[lev].size()-1))); + + // Populate t_surf + for (MFIter mfi(*t_surf[lev]); mfi.isValid(); ++mfi) + { + Box gtbx = mfi.growntilebox(); + + auto t_surf_arr = t_surf[lev]->array(mfi); + const auto sst_hi_arr = m_sst_lev[lev][n_time+1]->const_array(mfi); + const auto sst_lo_arr = m_sst_lev[lev][n_time ]->const_array(mfi); + + ParallelFor(gtbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + t_surf_arr(i,j,k) = oma * sst_lo_arr(i,j,k) + + alpha * sst_hi_arr(i,j,k); + }); + } +} diff --git a/Source/BoundaryConditions/BoundaryConditions_metgrid.cpp b/Source/BoundaryConditions/BoundaryConditions_metgrid.cpp new file mode 100644 index 000000000..37f4dc04f --- /dev/null +++ b/Source/BoundaryConditions/BoundaryConditions_metgrid.cpp @@ -0,0 +1,219 @@ +#include "ERF.H" +#include "Utils.H" + +using namespace amrex; + +#ifdef ERF_USE_NETCDF +/* + * Impose boundary conditions using data read in from met_em files + * + * @param[out] mfs Vector of MultiFabs to be filled + * @param[in] time time at which the data should be filled + */ + +void +ERF::fill_from_metgrid (const Vector& mfs, + const Real time, + bool cons_only, + int icomp_cons, + int ncomp_cons) +{ + int lev = 0; + + // Time interpolation + Real dT = bdy_time_interval; + Real time_since_start = time - start_bdy_time; + int n_time = static_cast( time_since_start / dT); + amrex::Real alpha = (time_since_start - n_time * dT) / dT; + AMREX_ALWAYS_ASSERT( alpha >= 0. && alpha <= 1.0); + amrex::Real oma = 1.0 - alpha; + + // Flags for read vars and index mapping + // See IndexDefines.H +#if defined(ERF_USE_MOISTURE) + // Cons includes [Rho RhoTheta RhoKE RhoQKE RhoScalar RhoQt RhoQp NumVars] + Vector cons_read = {1, 1, 0, 0, 0, 1, 0}; + Vector cons_map = {MetGridBdyVars::R, MetGridBdyVars::T, 0, 0, 0, MetGridBdyVars::QV, 0}; +#elif defined(ERF_USE_WARM_NO_PRECIP) + // Cons includes [Rho RhoTheta RhoKE RhoQKE RhoScalar RhoQv RhoQc NumVars] + Vector cons_read = {1, 1, 0, 0, 0, 1, 0}; + Vector cons_map = {MetGridBdyVars::R, MetGridBdyVars::T, 0, 0, 0, MetGridBdyVars::QV, 0}; +# else + // Cons includes [Rho RhoTheta RhoKE RhoQKE RhoScalar NumVars] + Vector cons_read = {1, 1, 0, 0, 0}; + Vector cons_map = {MetGridBdyVars::R, MetGridBdyVars::T, 0, 0, 0}; +#endif + + Vector> is_read; + is_read.push_back( cons_read ); + is_read.push_back( {1} ); // xvel + is_read.push_back( {1} ); // yvel + is_read.push_back( {0} ); // zvel + + Vector> ind_map; + ind_map.push_back( cons_map ); + ind_map.push_back( {MetGridBdyVars::U} ); // xvel + ind_map.push_back( {MetGridBdyVars::V} ); // yvel + ind_map.push_back( {0} ); // zvel + + // Nvars to loop over + Vector comp_var = {ncomp_cons, 1, 1, 1}; + + // End of vars loop + int var_idx_end = (cons_only) ? Vars::cons + 1 : Vars::NumTypes; + + // Loop over all variable types + for (int var_idx = Vars::cons; var_idx < var_idx_end; ++var_idx) + { + MultiFab& mf = *mfs[var_idx]; + + // + // Note that "domain" is mapped onto the type of box the data is in + // + Box domain = geom[lev].Domain(); + domain.convert(mf.boxArray().ixType()); + const auto& dom_lo = amrex::lbound(domain); + const auto& dom_hi = amrex::ubound(domain); + + // Offset only applys to cons (we may fill a subset of these vars) + int offset = (var_idx == Vars::cons) ? icomp_cons : 0; + + // Loop over each component + for (int comp_idx(0); comp_idx < comp_var[var_idx]; ++comp_idx) + { + int width = metgrid_bdy_set_width; + + // Variable can be read from met_em files + //------------------------------------ + if (is_read[var_idx][comp_idx]) + { + int ivar = ind_map[var_idx][comp_idx]; + IntVect ng_vect = mf.nGrowVect(); ng_vect[2] = 0; + +// if (ivar == MetGridBdyVars::U) { +// amrex::Print() << "fill_from_metgrid U var_idx=" << var_idx << " comp_idx=" << comp_idx << " ivar=" << ivar << std::endl; +// } else if (ivar == MetGridBdyVars::V) { +// amrex::Print() << "fill_from_metgrid V var_idx=" << var_idx << " comp_idx=" << comp_idx << " ivar=" << ivar << std::endl; +// } else if (ivar == MetGridBdyVars::R) { +// amrex::Print() << "fill_from_metgrid R var_idx=" << var_idx << " comp_idx=" << comp_idx << " ivar=" << ivar << std::endl; +// } else if (ivar == MetGridBdyVars::T) { +// amrex::Print() << "fill_from_metgrid T var_idx=" << var_idx << " comp_idx=" << comp_idx << " ivar=" << ivar << std::endl; +// } else if (ivar == MetGridBdyVars::QV) { +// amrex::Print() << "fill_from_metgrid QV var_idx=" << var_idx << " comp_idx=" << comp_idx << " ivar=" << ivar << std::endl; +// } else { +// amrex::Print() << "fill_from_metgrid UNKNOWN" << std::endl; +// } + + // We have data at fixed time intervals we will call dT + // Then to interpolate, given time, we can define n = (time/dT) + // and alpha = (time - n*dT) / dT, then we define the data at time + // as alpha * (data at time n+1) + (1 - alpha) * (data at time n) + const auto& bdatxlo_n = bdy_data_xlo[n_time ][ivar].const_array(); + const auto& bdatxlo_np1 = bdy_data_xlo[n_time+1][ivar].const_array(); + const auto& bdatxhi_n = bdy_data_xhi[n_time ][ivar].const_array(); + const auto& bdatxhi_np1 = bdy_data_xhi[n_time+1][ivar].const_array(); + const auto& bdatylo_n = bdy_data_ylo[n_time ][ivar].const_array(); + const auto& bdatylo_np1 = bdy_data_ylo[n_time+1][ivar].const_array(); + const auto& bdatyhi_n = bdy_data_yhi[n_time ][ivar].const_array(); + const auto& bdatyhi_np1 = bdy_data_yhi[n_time+1][ivar].const_array(); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(mf,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + // Grown tilebox so we fill exterior ghost cells as well + Box gbx = mfi.growntilebox(ng_vect); + const Array4& dest_arr = mf.array(mfi); + + // Call w/o interior ghost cells + Box bx_xlo, bx_xhi, bx_ylo, bx_yhi; + compute_interior_ghost_bxs_xy(gbx, domain, width, 0, + bx_xlo, bx_xhi, + bx_ylo, bx_yhi, ng_vect); + + // x-faces (includes exterior y ghost cells) + ParallelFor(bx_xlo, bx_xhi, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int ii = std::max(i , dom_lo.x); + int jj = std::max(j , dom_lo.y); + jj = std::min(jj, dom_hi.y); + dest_arr(i,j,k,comp_idx) = oma * bdatxlo_n (ii,jj,k,0) + + alpha * bdatxlo_np1(ii,jj,k,0); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int ii = std::min(i , dom_hi.x); + int jj = std::max(j , dom_lo.y); + jj = std::min(jj, dom_hi.y); + dest_arr(i,j,k,comp_idx) = oma * bdatxhi_n (ii,jj,k,0) + + alpha * bdatxhi_np1(ii,jj,k,0); + }); + + // y-faces (do not include exterior x ghost cells) + ParallelFor(bx_ylo, bx_yhi, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int jj = std::max(j , dom_lo.y); + dest_arr(i,j,k,comp_idx) = oma * bdatylo_n (i,jj,k,0) + + alpha * bdatylo_np1(i,jj,k,0); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int jj = std::min(j , dom_hi.y); + dest_arr(i,j,k,comp_idx) = oma * bdatyhi_n (i,jj,k,0) + + alpha * bdatyhi_np1(i,jj,k,0); + }); + } // mfi + + // Variable not read or computed from met_em files + //------------------------------------ + } else { + width = metgrid_bdy_width; + IntVect ng_vect = mf.nGrowVect(); ng_vect[2] = 0; + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(mf,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + // Grown tilebox so we fill exterior ghost cells as well + Box gbx = mfi.growntilebox(ng_vect); + const Array4& dest_arr = mf.array(mfi); + Box bx_xlo, bx_xhi, bx_ylo, bx_yhi; + compute_interior_ghost_bxs_xy(gbx, domain, width, 0, + bx_xlo, bx_xhi, + bx_ylo, bx_yhi, ng_vect); + + // x-faces (includes y ghost cells) + ParallelFor(bx_xlo, bx_xhi, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int jj = std::max(j , dom_lo.y); + jj = std::min(jj, dom_hi.y); + dest_arr(i,j,k,comp_idx) = dest_arr(dom_lo.x+width,jj,k,comp_idx); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + int jj = std::max(j , dom_lo.y); + jj = std::min(jj, dom_hi.y); + dest_arr(i,j,k,comp_idx) = dest_arr(dom_hi.x-width,jj,k,comp_idx); + }); + + // y-faces (does not include x ghost cells) + ParallelFor(bx_ylo, bx_yhi, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + dest_arr(i,j,k,comp_idx) = dest_arr(i,dom_lo.y+width,k,comp_idx); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + dest_arr(i,j,k,comp_idx) = dest_arr(i,dom_hi.y-width,k,comp_idx); + }); + } // mfi + } // is_read + } // comp + } // var +} +#endif diff --git a/Source/BoundaryConditions/BoundaryConditions_wrfbdy.cpp b/Source/BoundaryConditions/BoundaryConditions_wrfbdy.cpp index 6abb2221d..0153430ab 100644 --- a/Source/BoundaryConditions/BoundaryConditions_wrfbdy.cpp +++ b/Source/BoundaryConditions/BoundaryConditions_wrfbdy.cpp @@ -22,8 +22,8 @@ ERF::fill_from_wrfbdy (const Vector& mfs, // Time interpolation Real dT = bdy_time_interval; - Real time_since_start = (time - start_bdy_time) / 1.e10; - int n_time = static_cast( time_since_start / dT); + Real time_since_start = time - start_bdy_time; + int n_time = static_cast( time_since_start / dT); amrex::Real alpha = (time_since_start - n_time * dT) / dT; AMREX_ALWAYS_ASSERT( alpha >= 0. && alpha <= 1.0); amrex::Real oma = 1.0 - alpha; diff --git a/Source/BoundaryConditions/BoundaryConditions_zvel.cpp b/Source/BoundaryConditions/BoundaryConditions_zvel.cpp index b00ea1ba7..e320c6c42 100644 --- a/Source/BoundaryConditions/BoundaryConditions_zvel.cpp +++ b/Source/BoundaryConditions/BoundaryConditions_zvel.cpp @@ -159,7 +159,7 @@ void ERFPhysBCFunct::impose_vertical_zvel_bcs (const Array4& dest_arr, const Array4& z_phys_nd, const GpuArray dxInv, int bccomp_u, int bccomp_v, int bccomp_w, - int terrain_type) + TerrainType terrain_type) { BL_PROFILE_VAR("impose_vertical_zvel_bcs()",impose_vertical_zvel_bcs); const auto& dom_lo = amrex::lbound(domain); @@ -198,7 +198,7 @@ void ERFPhysBCFunct::impose_vertical_zvel_bcs (const Array4& dest_arr, const amrex::BCRec* bc_ptr_w = bcrs_w_d.data(); bool l_use_terrain = (m_z_phys_nd != nullptr); - bool l_moving_terrain = (terrain_type == 1); + bool l_moving_terrain = (terrain_type == TerrainType::Moving); GpuArray,AMREX_SPACEDIM+NVAR> l_bc_extdir_vals_d; diff --git a/Source/BoundaryConditions/ERF_FillPatch.cpp b/Source/BoundaryConditions/ERF_FillPatch.cpp index 7d2a4e4ed..d4c863c1e 100644 --- a/Source/BoundaryConditions/ERF_FillPatch.cpp +++ b/Source/BoundaryConditions/ERF_FillPatch.cpp @@ -73,7 +73,7 @@ ERF::FillPatch (int lev, Real time, const Vector& mfs, bool fillset) } // var_idx // Coarse-Fine set region - if (lev>0 && coupling_type=="OneWay" && cf_set_width>0 && fillset) { + if (lev>0 && solverChoice.coupling_type == CouplingType::OneWay && cf_set_width>0 && fillset) { FPr_c[lev-1].FillSet(*mfs[Vars::cons], time, null_bc, domain_bcs_type); FPr_u[lev-1].FillSet(*mfs[Vars::xvel], time, null_bc, domain_bcs_type); FPr_v[lev-1].FillSet(*mfs[Vars::yvel], time, null_bc, domain_bcs_type); @@ -92,7 +92,8 @@ ERF::FillPatch (int lev, Real time, const Vector& mfs, bool fillset) #ifdef ERF_USE_NETCDF // We call this here because it is an ERF routine - if (init_type=="real" && lev==0) fill_from_wrfbdy(mfs, time); + if (init_type == "real" && lev==0) fill_from_wrfbdy(mfs,time); + if (init_type == "metgrid" && lev==0) fill_from_metgrid(mfs,time); #endif if (m_r2d) fill_from_bndryregs(mfs,time); @@ -128,7 +129,7 @@ ERF::FillPatchMoistVars (int lev, MultiFab& mf) IntVect ngvect_cons = mf.nGrowVect(); IntVect ngvect_vels = {0,0,0}; - if (init_type != "real") { + if ((init_type != "real") and (init_type != "metgrid")) { (*physbcs[lev])({&mf},icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels,init_type,cons_only,bccomp_cons); } @@ -226,7 +227,7 @@ ERF::FillIntermediatePatch (int lev, Real time, } // var_idx // Coarse-Fine set region - if (lev>0 && coupling_type=="OneWay" && cf_set_width>0) { + if (lev>0 && solverChoice.coupling_type == CouplingType::OneWay && cf_set_width>0) { if (cons_only) { FPr_c[lev-1].FillSet(*mfs[Vars::cons], time, null_bc, domain_bcs_type); } else { @@ -248,7 +249,8 @@ ERF::FillIntermediatePatch (int lev, Real time, // --- i.e., cons_only and which cons indices (icomp_cons & ncomp_cons) // We call this here because it is an ERF routine - if (init_type=="real" && lev==0) fill_from_wrfbdy(mfs, time, cons_only, icomp_cons, ncomp_cons); + if (init_type == "real" && lev==0) fill_from_wrfbdy(mfs,time, cons_only, icomp_cons, ncomp_cons); + if (init_type == "metgrid" && lev==0) fill_from_metgrid(mfs,time, cons_only, icomp_cons, ncomp_cons); #endif if (m_r2d) fill_from_bndryregs(mfs,time); diff --git a/Source/BoundaryConditions/ERF_PhysBCFunct.H b/Source/BoundaryConditions/ERF_PhysBCFunct.H index 4c4a7ad55..c7790e0f1 100644 --- a/Source/BoundaryConditions/ERF_PhysBCFunct.H +++ b/Source/BoundaryConditions/ERF_PhysBCFunct.H @@ -25,7 +25,7 @@ public: ERFPhysBCFunct (const int lev, const amrex::Geometry& geom, const amrex::Vector& domain_bcs_type, const amrex::Gpu::DeviceVector& domain_bcs_type_d, - const int& terrain_type, + const TerrainType& terrain_type, amrex::Array,AMREX_SPACEDIM+NVAR> bc_extdir_vals, amrex::Array,AMREX_SPACEDIM+NVAR> bc_neumann_vals, std::unique_ptr& z_phys_nd, @@ -92,7 +92,7 @@ public: const amrex::Array4& z_nd, const amrex::GpuArray dxInv, int bccomp_u, int bccomp_v, int bccomp_w, - int terrain_type); + TerrainType terrain_type); void impose_lateral_cons_bcs (const amrex::Array4& dest_arr, const amrex::Box& bx, const amrex::Box& domain, @@ -108,7 +108,7 @@ private: amrex::Geometry m_geom; amrex::Vector m_domain_bcs_type; amrex::Gpu::DeviceVector m_domain_bcs_type_d; - int m_terrain_type; + TerrainType m_terrain_type; amrex::Array,AMREX_SPACEDIM+NVAR> m_bc_extdir_vals; amrex::Array,AMREX_SPACEDIM+NVAR> m_bc_neumann_vals; amrex::MultiFab* m_z_phys_nd; diff --git a/Source/BoundaryConditions/ERF_PhysBCFunct.cpp b/Source/BoundaryConditions/ERF_PhysBCFunct.cpp index ed3517ab2..eafea677a 100644 --- a/Source/BoundaryConditions/ERF_PhysBCFunct.cpp +++ b/Source/BoundaryConditions/ERF_PhysBCFunct.cpp @@ -82,7 +82,7 @@ void ERFPhysBCFunct::operator() (const Vector& mfs, int icomp_cons, i z_nd_arr = m_z_phys_nd->const_array(mfi); } - if (init_type != "real") + if ((init_type != "real") and (init_type != "metgrid")) { //! If there are cells not in the valid + periodic grown box @@ -109,7 +109,7 @@ void ERFPhysBCFunct::operator() (const Vector& mfs, int icomp_cons, i impose_lateral_zvel_bcs(velz_arr,velx_arr,vely_arr,zbx,domain,z_nd_arr,dxInv,BCVars::zvel_bc); } // !cons_only - } // init_type != "real" + } // init_type != "real" and init_type != "metgrid" // Every grid touches the bottom and top domain boundary so we call the vertical bcs // for every box -- and we need to call these even if init_type == real diff --git a/Source/BoundaryConditions/MOSTAverage.cpp b/Source/BoundaryConditions/MOSTAverage.cpp index 26b812428..a70b7a0ec 100644 --- a/Source/BoundaryConditions/MOSTAverage.cpp +++ b/Source/BoundaryConditions/MOSTAverage.cpp @@ -304,12 +304,13 @@ MOSTAverage::set_k_indices_T() auto k_arr = m_k_indx[lev]->array(mfi); ParallelFor(npbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { + k_arr(i,j,k) = 0; Real z_target = d_zref + z_phys_arr(i,j,k); for (int lk(0); lk<=kmax; ++lk) { Real z_lo = 0.25 * ( z_phys_arr(i,j ,lk ) + z_phys_arr(i+1,j ,lk ) - + z_phys_arr(i,j+1,lk ) + z_phys_arr(i+1,j+1,lk ) ); + + z_phys_arr(i,j+1,lk ) + z_phys_arr(i+1,j+1,lk ) ); Real z_hi = 0.25 * ( z_phys_arr(i,j ,lk+1) + z_phys_arr(i+1,j ,lk+1) - + z_phys_arr(i,j+1,lk+1) + z_phys_arr(i+1,j+1,lk+1) ); + + z_phys_arr(i,j+1,lk+1) + z_phys_arr(i+1,j+1,lk+1) ); if (z_target > z_lo && z_target < z_hi){ AMREX_ASSERT_WITH_MESSAGE(lk >= d_radius, "K index must be larger than averaging radius!"); @@ -376,9 +377,9 @@ MOSTAverage::set_norm_indices_T() Real z_target = delta_z + z_phys_arr(i,j,k); for (int lk(0); lk<=kmax; ++lk) { Real z_lo = 0.25 * ( z_phys_arr(i_new,j_new ,lk ) + z_phys_arr(i_new+1,j_new ,lk ) - + z_phys_arr(i_new,j_new+1,lk ) + z_phys_arr(i_new+1,j_new+1,lk ) ); + + z_phys_arr(i_new,j_new+1,lk ) + z_phys_arr(i_new+1,j_new+1,lk ) ); Real z_hi = 0.25 * ( z_phys_arr(i_new,j_new ,lk+1) + z_phys_arr(i_new+1,j_new ,lk+1) - + z_phys_arr(i_new,j_new+1,lk+1) + z_phys_arr(i_new+1,j_new+1,lk+1) ); + + z_phys_arr(i_new,j_new+1,lk+1) + z_phys_arr(i_new+1,j_new+1,lk+1) ); if (z_target > z_lo && z_target < z_hi){ AMREX_ASSERT_WITH_MESSAGE(lk >= d_radius, "K index must be larger than averaging radius!"); @@ -532,7 +533,7 @@ MOSTAverage::compute_plane_averages(int lev) // Peel back the level auto& fields = m_fields[lev]; auto& averages = m_averages[lev]; - const auto & geom = m_geom[lev]; + const auto & geom = m_geom[lev]; auto& z_phys = m_z_phys_nd[lev]; auto& x_pos = m_x_pos[lev]; diff --git a/Source/BoundaryConditions/MOSTStress.H b/Source/BoundaryConditions/MOSTStress.H index 1e4204192..c654546f0 100644 --- a/Source/BoundaryConditions/MOSTStress.H +++ b/Source/BoundaryConditions/MOSTStress.H @@ -149,7 +149,7 @@ struct adiabatic_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -202,7 +202,7 @@ struct adiabatic_mod_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -260,7 +260,7 @@ struct surface_flux private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -323,7 +323,7 @@ struct surface_flux_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -387,7 +387,7 @@ struct surface_flux_mod_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -447,7 +447,7 @@ struct surface_temp private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -512,7 +512,7 @@ struct surface_temp_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -578,7 +578,7 @@ struct surface_temp_mod_charnock private: most_data mdata; similarity_funs sfuns; - amrex::Real tol = 1.0e-5; + const amrex::Real tol = 1.0e-5; }; @@ -639,6 +639,7 @@ struct moeng_flux rho = cons_arr(ic,jc,zlo,Rho_comp); theta = cons_arr(ic,jc,zlo,RhoTheta_comp) / rho; eta = eta_arr(ie,je,zlo,EddyDiff::Theta_v); // == rho * alpha [kg/m^3 * m^2/s] + eta = amrex::max(eta,eps); amrex::Real theta_mean = tm_arr(ic,jc,zlo); amrex::Real wsp_mean = umm_arr(ic,jc,zlo); @@ -649,7 +650,7 @@ struct moeng_flux amrex::Real wsp = sqrt(velx*velx+vely*vely); amrex::Real num1 = (theta-theta_mean)*wsp_mean; amrex::Real num2 = (theta_mean-theta_surf)*wsp; - amrex::Real moflux = (std::abs(tstar) > EPS) ? + amrex::Real moflux = (std::abs(tstar) > eps) ? tstar*ustar*(num1+num2)/((theta_mean-theta_surf)*wsp_mean) : 0.0; amrex::Real deltaz = dz * (zlo - k); @@ -700,6 +701,7 @@ struct moeng_flux + cons_arr(ic ,jc,zlo,Rho_comp) ); eta = 0.5 *( eta_arr(ie-1,je,zlo,EddyDiff::Mom_v) + eta_arr(ie ,je,zlo,EddyDiff::Mom_v) ); + eta = amrex::max(eta,eps); amrex::Real umean = um_arr(i,j,zlo); amrex::Real wsp_mean = 0.5 * ( umm_arr(ic-1,jc,zlo) + umm_arr(ic,jc,zlo) ); @@ -763,6 +765,7 @@ struct moeng_flux + cons_arr(ic,jc ,zlo,Rho_comp) ); eta = 0.5*( eta_arr(ie,je-1,zlo,EddyDiff::Mom_v) + eta_arr(ie,je ,zlo,EddyDiff::Mom_v) ); + eta = amrex::max(eta,eps); amrex::Real vmean = vm_arr(i,j,zlo); amrex::Real wsp_mean = 0.5 * ( umm_arr(ic,jc-1,zlo) + umm_arr(ic,jc,zlo) ); @@ -784,7 +787,7 @@ struct moeng_flux private: int zlo; - static constexpr amrex::Real EPS = 1e-16; + const amrex::Real eps = 1e-16; }; @@ -833,6 +836,7 @@ struct donelan_flux rho = cons_arr(ic,jc,zlo,Rho_comp); theta = cons_arr(ic,jc,zlo,RhoTheta_comp) / rho; eta = eta_arr(ie,je,zlo,EddyDiff::Theta_v); // == rho * alpha [kg/m^3 * m^2/s] + eta = amrex::max(eta,eps); amrex::Real Cd = 0.0012; amrex::Real wsp_mean = umm_arr(ic,jc,zlo); @@ -888,6 +892,7 @@ struct donelan_flux + cons_arr(ic ,jc,zlo,Rho_comp) ); eta = 0.5 *( eta_arr(ie-1,je,zlo,EddyDiff::Mom_v) + eta_arr(ie ,je,zlo,EddyDiff::Mom_v) ); + eta = amrex::max(eta,eps); amrex::Real Cd = 0.001; const amrex::Real c = 7e-5; @@ -955,6 +960,7 @@ struct donelan_flux + cons_arr(ic,jc ,zlo,Rho_comp) ); eta = 0.5*( eta_arr(ie,je-1,zlo,EddyDiff::Mom_v) + eta_arr(ie,je ,zlo,EddyDiff::Mom_v) ); + eta = amrex::max(eta,eps); amrex::Real Cd = 0.001; const amrex::Real c = 7e-5; @@ -980,5 +986,6 @@ struct donelan_flux private: int zlo; + const amrex::Real eps = 1e-16; }; #endif diff --git a/Source/BoundaryConditions/Make.package b/Source/BoundaryConditions/Make.package index 56ffeafaf..b9bb5be57 100644 --- a/Source/BoundaryConditions/Make.package +++ b/Source/BoundaryConditions/Make.package @@ -4,6 +4,7 @@ CEXE_sources += BoundaryConditions_zvel.cpp CEXE_sources += BoundaryConditions_cons.cpp CEXE_sources += BoundaryConditions_bndryreg.cpp CEXE_sources += BoundaryConditions_wrfbdy.cpp +CEXE_sources += BoundaryConditions_metgrid.cpp CEXE_headers += ABLMost.H CEXE_sources += ABLMost.cpp diff --git a/Source/DataStructs/DataStruct.H b/Source/DataStructs/DataStruct.H index e8941c11e..abf87a276 100644 --- a/Source/DataStructs/DataStruct.H +++ b/Source/DataStructs/DataStruct.H @@ -20,6 +20,14 @@ enum struct ABLDriverType { None, PressureGradient, GeostrophicWind }; +enum struct CouplingType { + OneWay, TwoWay +}; + +enum struct TerrainType { + Static, Moving +}; + enum struct Coord { x, y, z }; @@ -59,11 +67,20 @@ struct SolverChoice { } // Is the terrain static or moving? - pp.query("terrain_type", terrain_type); + static std::string terrain_type_string = "Static"; + pp.query("terrain_type",terrain_type_string); + if (terrain_type_string == "Moving" || terrain_type_string == "moving") { + terrain_type = TerrainType::Moving; + } else if (terrain_type_string == "Static" || terrain_type_string == "static") { + terrain_type = TerrainType::Static; + } else { + amrex::Abort("terrain_type can be either Moving/moving or Static/static"); + } // Use lagged_delta_rt in the fast integrator? pp.query("use_lagged_delta_rt", use_lagged_delta_rt); - if (!use_lagged_delta_rt && !(terrain_type == 1)) { + + if (!use_lagged_delta_rt && !(terrain_type == TerrainType::Moving)) { amrex::Error("Can't turn off lagged_delta_rt when terrain not moving"); } @@ -171,6 +188,15 @@ struct SolverChoice { } } } + + // Which type of refinement + static std::string coupling_type_string = "OneWay"; + pp.query("coupling_type",coupling_type_string); + if (coupling_type_string == "TwoWay") { + coupling_type = CouplingType::TwoWay; + } else { + coupling_type = CouplingType::OneWay; + } } void display() @@ -183,6 +209,18 @@ struct SolverChoice { amrex::Print() << "use_rayleigh_damping : " << use_rayleigh_damping << std::endl; amrex::Print() << "use_gravity : " << use_gravity << std::endl; + if (coupling_type == CouplingType::TwoWay) { + amrex::Print() << "Using two-way coupling " << std::endl; + } else { + amrex::Print() << "Using one-way coupling " << std::endl; + } + + if (terrain_type == TerrainType::Static) { + amrex::Print() << "Using static terrain " << std::endl; + } else { + amrex::Print() << "Using moving terrain " << std::endl; + } + if (abl_driver_type == ABLDriverType::None) { amrex::Print() << "ABL Driver Type: " << "None" << std::endl; amrex::Print() << "No ABL driver selected " << std::endl; @@ -259,7 +297,6 @@ struct SolverChoice { bool test_mapfactor = false; bool use_terrain = false; - int terrain_type = 0; #ifdef ERF_USE_MOISTURE int buoyancy_type = 2; // uses Tprime #else @@ -306,6 +343,9 @@ struct SolverChoice { bool use_NumDiff{false}; amrex::Real NumDiffCoeff{0.}; + CouplingType coupling_type; + TerrainType terrain_type; + ABLDriverType abl_driver_type; amrex::GpuArray abl_pressure_grad; amrex::GpuArray abl_geo_forcing; diff --git a/Source/Diffusion/ComputeTurbulentViscosity.cpp b/Source/Diffusion/ComputeTurbulentViscosity.cpp index 4827610ff..2cc5d5946 100644 --- a/Source/Diffusion/ComputeTurbulentViscosity.cpp +++ b/Source/Diffusion/ComputeTurbulentViscosity.cpp @@ -61,7 +61,7 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu #endif for (amrex::MFIter mfi(eddyViscosity,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - // NOTE: This gets us the lateral ghost cells for lev>1; which + // NOTE: This gets us the lateral ghost cells for lev>0; which // have been filled from FP Two Levels. Box bxcc = mfi.growntilebox() & domain; @@ -164,7 +164,7 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu } } - // Extrapolate Kturb in extrap x/y, fill remaining elements + // Extrapolate Kturb in x/y, fill remaining elements (relevent to lev==0) //*********************************************************************************** int ngc(1); Real inv_Pr_t = turbChoice.Pr_t_inv; @@ -193,8 +193,8 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu for ( amrex::MFIter mfi(eddyViscosity,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { Box bxcc = mfi.tilebox(); - Box planex = bxcc; planex.setSmall(0, 1); planex.setBig(0, ngc); - Box planey = bxcc; planey.setSmall(1, 1); planey.setBig(1, ngc); + Box planex = bxcc; planex.setSmall(0, 1); planex.setBig(0, ngc); planex.grow(1,1); + Box planey = bxcc; planey.setSmall(1, 1); planey.setBig(1, ngc); planey.grow(0,1); int i_lo = bxcc.smallEnd(0); int i_hi = bxcc.bigEnd(0); int j_lo = bxcc.smallEnd(1); int j_hi = bxcc.bigEnd(1); bxcc.growLo(0,ngc); bxcc.growHi(0,ngc); @@ -202,33 +202,37 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu const Array4& mu_turb = eddyViscosity.array(mfi); - // Extrapolate outside the domain in lateral directions + // Extrapolate outside the domain in lateral directions (planex owns corner cells) if (i_lo == domain.smallEnd(0)) { amrex::ParallelFor(planex, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - mu_turb(i_lo-i, j, k, EddyDiff::Mom_h) = mu_turb(i_lo, j, k, EddyDiff::Mom_h); - mu_turb(i_lo-i, j, k, EddyDiff::Mom_v) = mu_turb(i_lo, j, k, EddyDiff::Mom_v); + int lj = amrex::min(amrex::max(j, domain.smallEnd(1)), domain.bigEnd(1)); + mu_turb(i_lo-i, j, k, EddyDiff::Mom_h) = mu_turb(i_lo, lj, k, EddyDiff::Mom_h); + mu_turb(i_lo-i, j, k, EddyDiff::Mom_v) = mu_turb(i_lo, lj, k, EddyDiff::Mom_v); }); } if (i_hi == domain.bigEnd(0)) { amrex::ParallelFor(planex, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - mu_turb(i_hi+i, j, k, EddyDiff::Mom_h) = mu_turb(i_hi, j, k, EddyDiff::Mom_h); - mu_turb(i_hi+i, j, k, EddyDiff::Mom_v) = mu_turb(i_hi, j, k, EddyDiff::Mom_v); + int lj = amrex::min(amrex::max(j, domain.smallEnd(1)), domain.bigEnd(1)); + mu_turb(i_hi+i, j, k, EddyDiff::Mom_h) = mu_turb(i_hi, lj, k, EddyDiff::Mom_h); + mu_turb(i_hi+i, j, k, EddyDiff::Mom_v) = mu_turb(i_hi, lj, k, EddyDiff::Mom_v); }); } if (j_lo == domain.smallEnd(1)) { amrex::ParallelFor(planey, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - mu_turb(i, j_lo-j, k, EddyDiff::Mom_h) = mu_turb(i, j_lo, k, EddyDiff::Mom_h); - mu_turb(i, j_lo-j, k, EddyDiff::Mom_v) = mu_turb(i, j_lo, k, EddyDiff::Mom_v); + int li = amrex::min(amrex::max(i, domain.smallEnd(0)), domain.bigEnd(0)); + mu_turb(i, j_lo-j, k, EddyDiff::Mom_h) = mu_turb(li, j_lo, k, EddyDiff::Mom_h); + mu_turb(i, j_lo-j, k, EddyDiff::Mom_v) = mu_turb(li, j_lo, k, EddyDiff::Mom_v); }); } if (j_hi == domain.bigEnd(1)) { amrex::ParallelFor(planey, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - mu_turb(i, j_hi+j, k, EddyDiff::Mom_h) = mu_turb(i, j_hi, k, EddyDiff::Mom_h); - mu_turb(i, j_hi+j, k, EddyDiff::Mom_v) = mu_turb(i, j_hi, k, EddyDiff::Mom_v); + int li = amrex::min(amrex::max(i, domain.smallEnd(0)), domain.bigEnd(0)); + mu_turb(i, j_hi+j, k, EddyDiff::Mom_h) = mu_turb(li, j_hi, k, EddyDiff::Mom_h); + mu_turb(i, j_hi+j, k, EddyDiff::Mom_v) = mu_turb(li, j_hi, k, EddyDiff::Mom_v); }); } @@ -336,9 +340,9 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu mu_turb(i, j, k_hi+k, indx_v) = mu_turb(i, j, k_hi, indx_v); }); break; - } - } - } + } + } + } } /** diff --git a/Source/EOS.H b/Source/EOS.H index cc4604878..2ce071025 100644 --- a/Source/EOS.H +++ b/Source/EOS.H @@ -7,7 +7,20 @@ #include /** - * Function to return temperature given density and potential temperatue + * Function to return potential temperature given pressure and temperature + * + * @params[in] pressure + * @params[in] temperature + * @params[in] rd0cp ratio of R_d to c_p +*/ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real getThgivenPandT(const amrex::Real T, const amrex::Real P, const amrex::Real rdOcp) +{ + return T*std::pow(p_0/P, rdOcp); +} + +/** + * Function to return temperature given density and potential temperature * * @params[in] rho density * @params[in] rhotheta density times potential temperature theta @@ -24,7 +37,7 @@ amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, } /** - * Function to return potential temperature given density and temperatue + * Function to return potential temperature given density and temperature * * @params[in] rho density * @params[in] T temperature diff --git a/Source/ERF.H b/Source/ERF.H index 340a4edbb..0f8adfc1f 100644 --- a/Source/ERF.H +++ b/Source/ERF.H @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #ifdef AMREX_MEM_PROFILING @@ -48,6 +48,10 @@ #include "Microphysics.H" #endif +#ifdef ERF_USE_RRTMGP +#include "Radiation.H" +#endif + #ifdef ERF_USE_NETCDF #include "NCWpsFile.H" #endif @@ -218,6 +222,12 @@ public: const amrex::Real& dt_advance); #endif +#if defined(ERF_USE_RRTMGP) + void advance_radiation (int lev, + amrex::MultiFab& cons_in, + const amrex::Real& dt_advance); +#endif + amrex::MultiFab& build_fine_mask (int lev); void MakeHorizontalAverages (); @@ -301,6 +311,11 @@ public: bool cons_only = false, int icomp_cons = 0, int ncomp_cons = NVAR); + void fill_from_metgrid (const amrex::Vector& mfs, + amrex::Real time, + bool cons_only = false, + int icomp_cons = 0, + int ncomp_cons = NVAR); #endif #ifdef ERF_USE_PARTICLES @@ -308,6 +323,11 @@ public: static amrex::Vector tracer_particle_varnames; #endif +#ifdef ERF_USE_NETCDF + void init_from_wrfinput (int lev); + void init_from_metgrid (int lev); +#endif // ERF_USE_NETCDF + private: /////////////////////////// @@ -340,11 +360,6 @@ private: void initialize_integrator (int lev, amrex::MultiFab& cons_mf, amrex::MultiFab& vel_mf); -#ifdef ERF_USE_NETCDF - void init_from_wrfinput (int lev); - void init_from_metgrid (int lev); -#endif // ERF_USE_NETCDF - // more flexible version of AverageDown() that lets you average down across multiple levels void AverageDownTo (int crse_lev); // NOLINT @@ -530,6 +545,10 @@ private: amrex::Vector qmoist; // This has 6 components: qv, qc, qi, qr, qs, qg #endif +#if defined(ERF_USE_RRTMGP) + Radiation rad; +#endif + // Fillpatcher classes for coarse-fine boundaries int cf_width{0}; int cf_set_width{0}; @@ -546,6 +565,10 @@ private: amrex::Vector> eddyDiffs_lev; amrex::Vector> SmnSmn_lev; + // Sea Surface Temps and Land Masks (lev, ntimes) + amrex::Vector>> sst_lev; + amrex::Vector>> lmask_lev; + // Other SFS terms amrex::Vector> SFS_hfx1_lev, SFS_hfx2_lev, SFS_hfx3_lev; amrex::Vector> SFS_diss_lev; @@ -568,13 +591,11 @@ private: amrex::Vector> mapfac_u; amrex::Vector> mapfac_v; - amrex::Vector> sst; - amrex::Vector base_state; amrex::Vector base_state_new; // array of flux registers - amrex::Vector flux_registers; + amrex::Vector advflux_reg; // A BCRec is essentially a 2*DIM integer array storing the boundary // condition type at each lo/hi walls in each direction. We have one BCRec @@ -680,9 +701,6 @@ private: // flag to turn tracer particle generation on/off static bool use_tracer_particles; - // mesh refinement - static std::string coupling_type; - // Diagnostic output interval static int sum_interval; static amrex::Real sum_per; @@ -701,6 +719,10 @@ private: int wrfbdy_width{0}; int wrfbdy_set_width{0}; + // NetCDF initialization (met_em) file + int metgrid_bdy_width{0}; + int metgrid_bdy_set_width{0}; + // Text input_sounding file static std::string input_sounding_file; @@ -820,10 +842,9 @@ private: } AMREX_FORCE_INLINE - amrex::FluxRegister& - get_flux_reg (int lev) + amrex::YAFluxRegister& getAdvFluxReg (int lev) { - return *flux_registers[lev]; + return *advflux_reg[lev]; } AMREX_FORCE_INLINE diff --git a/Source/ERF.cpp b/Source/ERF.cpp index ac2641b92..9dff3a68b 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -42,9 +42,6 @@ bool ERF::use_tracer_particles = false; amrex::Vector ERF::tracer_particle_varnames = {AMREX_D_DECL("xvel", "yvel", "zvel")}; #endif -// Type of mesh refinement algorithm -std::string ERF::coupling_type = "OneWay"; - // Dictate verbosity in screen output int ERF::verbose = 0; @@ -169,7 +166,7 @@ ERF::ERF () mri_integrator_mem.resize(nlevs_max); physbcs.resize(nlevs_max); - flux_registers.resize(nlevs_max); + advflux_reg.resize(nlevs_max); // Stresses Tau11_lev.resize(nlevs_max); Tau22_lev.resize(nlevs_max); Tau33_lev.resize(nlevs_max); @@ -181,6 +178,10 @@ ERF::ERF () eddyDiffs_lev.resize(nlevs_max); SmnSmn_lev.resize(nlevs_max); + // Sea surface temps + sst_lev.resize(nlevs_max); + lmask_lev.resize(nlevs_max); + // Metric terms z_phys_nd.resize(nlevs_max); z_phys_cc.resize(nlevs_max); @@ -315,12 +316,12 @@ ERF::post_timestep (int nstep, Real time, Real dt_lev0) { BL_PROFILE("ERF::post_timestep()"); - if (coupling_type == "TwoWay") + if (solverChoice.coupling_type == CouplingType::TwoWay) { for (int lev = finest_level-1; lev >= 0; lev--) { // This call refluxes from the lev/lev+1 interface onto lev - get_flux_reg(lev+1).Reflux(vars_new[lev][Vars::cons],1.0, 0, 0, NVAR, geom[lev]); + getAdvFluxReg(lev+1).Reflux(vars_new[lev][Vars::cons], 0, 0, NVAR); // We need to do this before anything else because refluxing changes the // values of coarse cells underneath fine grids with the assumption they'll @@ -367,7 +368,7 @@ ERF::post_timestep (int nstep, Real time, Real dt_lev0) } // Moving terrain - if ( solverChoice.use_terrain && (solverChoice.terrain_type == 1) ) + if ( solverChoice.use_terrain && (solverChoice.terrain_type == TerrainType::Moving) ) { for (int lev = finest_level; lev >= 0; lev--) { @@ -403,8 +404,8 @@ ERF::InitData () } } - if (!solverChoice.use_terrain && solverChoice.terrain_type != 0) { - amrex::Abort("We do not allow terrain_type != 0 with use_terrain = false"); + if (!solverChoice.use_terrain && solverChoice.terrain_type != TerrainType::Static) { + amrex::Abort("We do not allow non-static terrain_type with use_terrain = false"); } last_plot_file_step_1 = -1; @@ -485,7 +486,7 @@ ERF::InitData () } - if (coupling_type == "TwoWay") { + if (solverChoice.coupling_type == CouplingType::TwoWay) { AverageDown(); } @@ -525,11 +526,14 @@ ERF::InitData () } // Initialize flux registers (whether we start from scratch or restart) - if (coupling_type == "TwoWay") { - flux_registers[0] = 0; + if (solverChoice.coupling_type == CouplingType::TwoWay) { + advflux_reg[0] = nullptr; for (int lev = 1; lev <= finest_level; lev++) { - flux_registers[lev] = new FluxRegister(grids[lev], dmap[lev], ref_ratio[lev-1], lev, NVAR); + advflux_reg[lev] = new YAFluxRegister(grids[lev], grids[lev-1], + dmap[lev], dmap[lev-1], + geom[lev], geom[lev-1], + ref_ratio[lev-1], lev, NVAR); } } @@ -554,8 +558,12 @@ ERF::InitData () // WritePlotFile calls FillPatch in order to compute gradients if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST) { - int ng_for_most = ComputeGhostCells(solverChoice.advChoice,solverChoice.use_NumDiff)+1; - m_most = std::make_unique(geom,vars_old,Theta_prim,z_phys_nd,ng_for_most); + m_most = std::make_unique(geom, vars_old, Theta_prim, z_phys_nd, + sst_lev, lmask_lev +#ifdef ERF_USE_NETCDF + ,start_bdy_time, bdy_time_interval +#endif + ); // We now configure ABLMost params here so that we can print the averages at t=0 // Note we don't fill ghost cells here because this is just for diagnostics @@ -566,7 +574,7 @@ ERF::InitData () MultiFab::Copy( *Theta_prim[lev], S, Cons::RhoTheta, 0, 1, ng); MultiFab::Divide(*Theta_prim[lev], S, Cons::Rho , 0, 1, ng); m_most->update_mac_ptrs(lev, vars_new, Theta_prim); - m_most->update_fluxes(lev,t_new[lev]); + m_most->update_fluxes(lev, t_new[lev]); } } @@ -634,7 +642,7 @@ ERF::InitData () { // NOTE: we must set up the FillPatcher object before calling // FillPatch at a fine level - if (coupling_type=="OneWay" && cf_width>0 && lev>0) { + if (solverChoice.coupling_type== CouplingType::OneWay && cf_width>0 && lev>0) { Construct_ERFFillPatchers(lev); Register_ERFFillPatchers(lev); } @@ -649,7 +657,7 @@ ERF::InitData () base_state[lev].FillBoundary(geom[lev].periodicity()); // For moving terrain only - if (solverChoice.terrain_type > 0) { + if (solverChoice.terrain_type != TerrainType::Static) { MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,3,1); base_state_new[lev].FillBoundary(geom[lev].periodicity()); } @@ -936,10 +944,6 @@ ERF::ReadParameters () AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt > 0.); - // Mesh refinement - pp.query("coupling_type",coupling_type); - AMREX_ALWAYS_ASSERT( (coupling_type == "OneWay") || (coupling_type == "TwoWay") ); - // How to initialize pp.query("init_type",init_type); if (!init_type.empty() && @@ -953,12 +957,8 @@ ERF::ReadParameters () } // No moving terrain with init real - if (init_type == "real") { - int terr_type(0); - pp.query("terrain_type",terr_type); - if (terr_type) { - amrex::Abort("Moving terrain is not supported with init real"); - } + if (init_type == "real" && solverChoice.terrain_type != TerrainType::Static) { + amrex::Abort("Moving terrain is not supported with init real"); } // We use this to keep track of how many boxes we read in from WRF initialization @@ -1041,6 +1041,13 @@ ERF::ReadParameters () AMREX_ALWAYS_ASSERT(wrfbdy_set_width >= 0); AMREX_ALWAYS_ASSERT(wrfbdy_width >= wrfbdy_set_width); + // Query the set and total widths for metgrid_bdy interior ghost cells + pp.query("metgrid_bdy_width", metgrid_bdy_width); + pp.query("metgrid_bdy_set_width", metgrid_bdy_set_width); + AMREX_ALWAYS_ASSERT(metgrid_bdy_width >= 0); + AMREX_ALWAYS_ASSERT(metgrid_bdy_set_width >= 0); + AMREX_ALWAYS_ASSERT(metgrid_bdy_width >= metgrid_bdy_set_width); + // Query the set and total widths for crse-fine interior ghost cells pp.query("cf_width", cf_width); pp.query("cf_set_width", cf_set_width); @@ -1072,7 +1079,7 @@ ERF::MakeHorizontalAverages () int lev = 0; // First, average down all levels (if doing two-way coupling) - if (coupling_type == "TwoWay") { + if (solverChoice.coupling_type == CouplingType::TwoWay) { AverageDown(); } @@ -1204,7 +1211,7 @@ ERF::MakeDiagnosticAverage (Vector& h_havg, MultiFab& S, int n) void ERF::AverageDown () { - AMREX_ALWAYS_ASSERT(coupling_type == "TwoWay"); + AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay); for (int lev = finest_level-1; lev >= 0; --lev) { AverageDownTo(lev); @@ -1215,7 +1222,7 @@ ERF::AverageDown () void ERF::AverageDownTo (int crse_lev) // NOLINT { - AMREX_ALWAYS_ASSERT(coupling_type == "TwoWay"); + AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay); for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) { const BoxArray& ba(vars_new[crse_lev][var_idx].boxArray()); if (ba[0].type() == IntVect::TheZeroVector()) @@ -1376,7 +1383,7 @@ ERF::ERF (const amrex::RealBox& rb, int max_level_in, nbx = convert(domain_p[0],IntVect(0,0,1)); domain_p.push_back(nbx); - flux_registers.resize(nlevs_max); + advflux_reg.resize(nlevs_max); // Stresses Tau11_lev.resize(nlevs_max); Tau22_lev.resize(nlevs_max); Tau33_lev.resize(nlevs_max); @@ -1388,6 +1395,10 @@ ERF::ERF (const amrex::RealBox& rb, int max_level_in, eddyDiffs_lev.resize(nlevs_max); SmnSmn_lev.resize(nlevs_max); + // Sea surface temps + sst_lev.resize(nlevs_max); + lmask_lev.resize(nlevs_max); + // Metric terms z_phys_nd.resize(nlevs_max); z_phys_cc.resize(nlevs_max); diff --git a/Source/ERF_make_new_level.cpp b/Source/ERF_make_new_level.cpp index 2ada7d831..c94b4e635 100644 --- a/Source/ERF_make_new_level.cpp +++ b/Source/ERF_make_new_level.cpp @@ -105,7 +105,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, base_state[lev].define(ba,dm,3,1); base_state[lev].setVal(0.); - if (solverChoice.use_terrain && solverChoice.terrain_type > 0) { + if (solverChoice.use_terrain && solverChoice.terrain_type != TerrainType::Static) { base_state_new[lev].define(ba,dm,3,1); base_state_new[lev].setVal(0.); } @@ -114,7 +114,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, z_phys_cc[lev] = std::make_unique(ba,dm,1,1); detJ_cc[lev] = std::make_unique(ba,dm,1,1); - if (solverChoice.terrain_type > 0) { + if (solverChoice.terrain_type != TerrainType::Static) { detJ_cc_new[lev] = std::make_unique(ba,dm,1,1); detJ_cc_src[lev] = std::make_unique(ba,dm,1,1); z_t_rk[lev] = std::make_unique( convert(ba, IntVect(0,0,1)), dm, 1, 1 ); @@ -126,7 +126,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, // We need this to be one greater than the ghost cells to handle levels > 0 int ngrow = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 2; z_phys_nd[lev] = std::make_unique(ba_nd,dm,1,IntVect(ngrow,ngrow,1)); - if (solverChoice.terrain_type > 0) { + if (solverChoice.terrain_type != TerrainType::Static) { z_phys_nd_new[lev] = std::make_unique(ba_nd,dm,1,IntVect(ngrow,ngrow,1)); z_phys_nd_src[lev] = std::make_unique(ba_nd,dm,1,IntVect(ngrow,ngrow,1)); } @@ -145,6 +145,9 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, z_t_rk[lev] = nullptr; } + sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr; + lmask_lev[lev].resize(1); lmask_lev[lev][0] = nullptr; + // ******************************************************************************************** // Define Theta_prim storage if using MOST BC // ******************************************************************************************** @@ -167,7 +170,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, // since we may need to use the grid information before constructing // initial idealized data // ******************************************************************************************** - if (init_type == "real") { + if ((init_type == "real") || (init_type == "metgrid")) { // If called from restart, the data structures for terrain-related quantities // are built in the ReadCheckpoint routine. Otherwise we build them here. diff --git a/Source/IO/Make.package b/Source/IO/Make.package index e608b0756..079a2eb1c 100644 --- a/Source/IO/Make.package +++ b/Source/IO/Make.package @@ -15,7 +15,6 @@ ifeq ($(USE_NETCDF), TRUE) CEXE_sources += ReadFromWRFBdy.cpp CEXE_sources += ReadFromWRFInput.cpp CEXE_sources += ReadFromMetgrid.cpp - CEXE_sources += NCBuildFABs.cpp CEXE_sources += NCInterface.cpp CEXE_sources += NCPlotFile.cpp CEXE_sources += NCColumnFile.cpp diff --git a/Source/IO/NCBuildFABs.cpp b/Source/IO/NCBuildFABs.cpp deleted file mode 100644 index 048803254..000000000 --- a/Source/IO/NCBuildFABs.cpp +++ /dev/null @@ -1,152 +0,0 @@ -#include -#include -#include -#include - -#include "DataStruct.H" -#include "NCInterface.H" -#include "NCWpsFile.H" -#include "AMReX_FArrayBox.H" -#include "AMReX_IndexType.H" -#include "AMReX_Print.H" - -using namespace amrex; - -using RARRAY = NDArray; - -void -fill_fab_from_arrays(int iv, Vector& nc_arrays, - const std::string& var_name, - NC_Data_Dims_Type& NC_dim_type, - FArrayBox& temp); - -/** - * Function to read NetCDF variables and fill the corresponding Array4's - * - * @param fname Name of the NetCDF file to be read - * @param nc_var_names Variable names in the NetCDF file - * @param NC_dim_types NetCDF data dimension types - * @param fab_vars Fab data we are to fill - */ -void -BuildFABsFromNetCDFFile(const Box& domain, - const std::string &fname, - Vector nc_var_names, - Vector NC_dim_types, - Vector fab_vars) -{ - int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank - - amrex::Vector nc_arrays(nc_var_names.size()); - - if (amrex::ParallelDescriptor::IOProcessor()) - { - ReadNetCDFFile(fname, nc_var_names, nc_arrays); - } - - for (int iv = 0; iv < nc_var_names.size(); iv++) - { - amrex::FArrayBox tmp; - if (amrex::ParallelDescriptor::IOProcessor()) { - fill_fab_from_arrays(iv, nc_arrays, nc_var_names[iv], NC_dim_types[iv], tmp); - } - - Box box = tmp.box(); - int ncomp = tmp.nComp(); - ParallelDescriptor::Bcast(&box, 1, ioproc); - ParallelDescriptor::Bcast(&ncomp, 1, ioproc); - - if (!amrex::ParallelDescriptor::IOProcessor()) { -#ifdef AMREX_USE_GPU - tmp.resize(box,ncomp, The_Pinned_Arena()); -#else - tmp.resize(box,ncomp); -#endif - } - - ParallelDescriptor::Bcast(tmp.dataPtr(), tmp.size(), ioproc); - - // Shift box by the domain lower corner - Box fab_bx = tmp.box(); - Dim3 dom_lb = lbound(domain); - fab_bx += IntVect(dom_lb.x,dom_lb.y,dom_lb.z); - // fab_vars points to data on device - fab_vars[iv]->resize(fab_bx,1); -#ifdef AMREX_USE_GPU - Gpu::copy(Gpu::hostToDevice, tmp.dataPtr(), tmp.dataPtr() + tmp.size(), fab_vars[iv]->dataPtr()); -#else - // Provided by BaseFab inheritance through FArrayBox - fab_vars[iv]->copy(tmp,tmp.box(),0,fab_bx,0,1); -#endif - } -} - -/** - * Helper function for reading data from NetCDF file into a - * provided FAB. - * - * @param iv Index for which variable we are going to fill - * @param nc_arrays Arrays of data from NetCDF file - * @param var_name Variable name - * @param NC_dim_type Dimension type for the variable as stored in the NetCDF file - * @param temp FAB where we store the variable data from the NetCDF Arrays - */ -void -fill_fab_from_arrays(int iv, Vector& nc_arrays, - const std::string& var_name, - NC_Data_Dims_Type& NC_dim_type, - FArrayBox& temp) -{ - int ns1, ns2, ns3; - if (NC_dim_type == NC_Data_Dims_Type::Time_BT) { - ns1 = nc_arrays[iv].get_vshape()[1]; - ns2 = 1; - ns3 = 1; - // amrex::Print() << "TYPE BT " << ns1 << std::endl; - } else if (NC_dim_type == NC_Data_Dims_Type::Time_SN_WE) { - ns1 = 1; - ns2 = nc_arrays[iv].get_vshape()[1]; - ns3 = nc_arrays[iv].get_vshape()[2]; - // amrex::Print() << "TYPE SN WE " << ns2 << " " << ns3 << std::endl; - } else if (NC_dim_type == NC_Data_Dims_Type::Time_BT_SN_WE) { - ns1 = nc_arrays[iv].get_vshape()[1]; - ns2 = nc_arrays[iv].get_vshape()[2]; - ns3 = nc_arrays[iv].get_vshape()[3]; - // amrex::Print() << "TYPE BT SN WE " << ns1 << " " << ns2 << " " << ns3 << std::endl; - } else { - amrex::Abort("Dont know this NC_Data_Dims_Type"); - } - - // TODO: The box will only start at (0,0,0) at level 0 -- we need to generalize this - Box my_box(IntVect(0,0,0), IntVect(ns3-1,ns2-1,ns1-1)); - // amrex::Print() <<" MY BOX " << my_box << std::endl; - - if (var_name == "U" || var_name == "UU" || - var_name == "MACFAC_U" || var_name == "MAPFAC_UY") my_box.setType(amrex::IndexType(IntVect(1,0,0))); - if (var_name == "V" || var_name == "VV" || - var_name == "MACFAC_V" || var_name == "MAPFAC_VY") my_box.setType(amrex::IndexType(IntVect(0,1,0))); - if (var_name == "W" || var_name == "WW") my_box.setType(amrex::IndexType(IntVect(0,0,1))); - -#ifdef AMREX_USE_GPU - // Make sure temp lives on CPU since nc_arrays lives on CPU only - temp.resize(my_box,1,The_Pinned_Arena()); -#else - temp.resize(my_box,1); -#endif - Array4 fab_arr = temp.array(); - - int ioff = temp.box().smallEnd()[0]; - int joff = temp.box().smallEnd()[1]; - - auto num_pts = my_box.numPts(); - - // amrex::Print() <<" ns1 * ns2 * ns3 " << ns1 * ns2 * ns3 << std::endl; - // amrex::Print() <<" NUMPTS " << num_pts << std::endl; - - for (int n(0); n < num_pts; ++n) { - int k = n / (ns2*ns3); - int j = (n - k*(ns2*ns3)) / ns3 + joff; - int i = n - k*(ns2*ns3) - (j-joff) * ns3 + ioff; - fab_arr(i,j,k,0) = static_cast(*(nc_arrays[iv].get_data()+n)); - } -} diff --git a/Source/IO/NCInterface.H b/Source/IO/NCInterface.H index 687d44a2a..386bd79e0 100644 --- a/Source/IO/NCInterface.H +++ b/Source/IO/NCInterface.H @@ -58,120 +58,128 @@ struct NCVar const int varid; //! Name of this variable - [[nodiscard]] std::string name() const; + [[nodiscard]] std::string name () const; //! Number of array dimensions for this variable - [[nodiscard]] int ndim() const; + [[nodiscard]] int ndim () const; //! Shape of the array (size in each array dimension) - [[nodiscard]] std::vector shape() const; + [[nodiscard]] std::vector shape () const; //! Write out the entire variable - void put(const double*) const; - void put(const float*) const; - void put(const int*) const; + void put (const double*) const; + void put (const float*) const; + void put (const int*) const; //! Write out a slice of data void - put(const double*, - const std::vector&, - const std::vector&) const; + put (const double*, + const std::vector&, + const std::vector&) const; //! Write out a slice of data with with strides (see hyperslab definition in //! NetCDF) void - put(const double*, - const std::vector&, - const std::vector&, - const std::vector&) const; + put (const double*, + const std::vector&, + const std::vector&, + const std::vector&) const; //! Write out a slice of data void - put(const float*, - const std::vector&, - const std::vector&) const; + put (const float*, + const std::vector&, + const std::vector&) const; //! Write out a slice of data with with strides (see hyperslab definition in //! NetCDF) void - put(const float*, - const std::vector&, - const std::vector&, - const std::vector&) const; + put (const float*, + const std::vector&, + const std::vector&, + const std::vector&) const; + + void put (const int*, const std::vector&, const std::vector&) const; - void put(const int*, const std::vector&, const std::vector&) const; void - put(const int*, - const std::vector&, - const std::vector&, - const std::vector&) const; + put (const int*, + const std::vector&, + const std::vector&, + const std::vector&) const; + + void put (const char**, const std::vector&, const std::vector&) const; - void put(const char**, const std::vector&, const std::vector&) const; void - put( - const char** dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const; + put (const char** dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const; //! Read the entire variable from file - void get(double*) const; - void get(float*) const; - void get(int*) const; + void get (double*) const; + void get (float*) const; + void get (int*) const; //! Read a chunk of data from the file void - get(double*, const std::vector&, const std::vector&) const; + get (double*, const std::vector&, const std::vector&) const; //! Read a chunk of data with strides void - get(double*, - const std::vector&, - const std::vector&, - const std::vector&) const; + get (double*, + const std::vector&, + const std::vector&, + const std::vector&) const; //! Read a chunk of data from the file void - get(float*, const std::vector&, const std::vector&) const; + get (float*, const std::vector&, const std::vector&) const; //! Read a chunk of data with strides void - get(float*, - const std::vector&, - const std::vector&, - const std::vector&) const; + get (float*, + const std::vector&, + const std::vector&, + const std::vector&) const; void - get(int*, const std::vector&, const std::vector&) const; + get (int*, const std::vector&, const std::vector&) const; + void - get(int*, - const std::vector&, - const std::vector&, - const std::vector&) const; + get (int*, + const std::vector&, + const std::vector&, + const std::vector&) const; //! Read a chunk of data from the file void - get(char*, const std::vector&, const std::vector&) const; + get (char*, const std::vector&, const std::vector&) const; //! Read a chunk of data with strides void - get(char*, - const std::vector&, - const std::vector&, - const std::vector&) const; + get (char*, + const std::vector&, + const std::vector&, + const std::vector&) const; + + [[nodiscard]] bool has_attr (const std::string& name) const; + + void put_attr (const std::string& name, const std::string& value) const; - [[nodiscard]] bool has_attr(const std::string& name) const; - void put_attr(const std::string& name, const std::string& value) const; void - put_attr(const std::string& name, const std::vector& value) const; + put_attr (const std::string& name, const std::vector& value) const; + void - put_attr(const std::string& name, const std::vector& value) const; - void put_attr(const std::string& name, const std::vector& value) const; - - [[nodiscard]] std::string get_attr(const std::string& name) const; - void get_attr(const std::string& name, std::vector& value) const; - void get_attr(const std::string& name, std::vector& value) const; - void get_attr(const std::string& name, std::vector& value) const; - void par_access(int cmode) const; //Uncomment for parallel NetCDF + put_attr (const std::string& name, const std::vector& value) const; + + void put_attr (const std::string& name, const std::vector& value) const; + + [[nodiscard]] std::string get_attr (const std::string& name) const; + + void get_attr (const std::string& name, std::vector& value) const; + void get_attr (const std::string& name, std::vector& value) const; + void get_attr (const std::string& name, std::vector& value) const; + + void par_access (int cmode) const; //Uncomment for parallel NetCDF }; //! Representation of a NetCDF group @@ -182,99 +190,97 @@ public: const int ncid; //! Name of this group - [[nodiscard]] std::string name() const; + [[nodiscard]] std::string name () const; //! Full name for this group - [[nodiscard]] std::string full_name() const; + [[nodiscard]] std::string full_name () const; //! Number of sub-groups within this group - [[nodiscard]] int num_groups() const; + [[nodiscard]] int num_groups () const; //! Number of dimensions in this group - [[nodiscard]] int num_dimensions() const; + [[nodiscard]] int num_dimensions () const; //! Number of variables within this group - [[nodiscard]] int num_variables() const; + [[nodiscard]] int num_variables () const; //! Number of attributes within this group - [[nodiscard]] int num_attributes() const; + [[nodiscard]] int num_attributes () const; //! Check if a group exists - [[nodiscard]] bool has_group(const std::string&) const; + [[nodiscard]] bool has_group (const std::string&) const; //! Check if a dimension exists by name - [[nodiscard]] bool has_dim(const std::string&) const; + [[nodiscard]] bool has_dim (const std::string&) const; //! Check if a variable exists by name - [[nodiscard]] bool has_var(const std::string&) const; + [[nodiscard]] bool has_var (const std::string&) const; //! Check if an attribute exists - [[nodiscard]] bool has_attr(const std::string&) const; + [[nodiscard]] bool has_attr (const std::string&) const; /** Get the group by name * * Throws error if the group doesn't exist, use `has_group` to check */ - [[nodiscard]] NCGroup group(const std::string&) const; + [[nodiscard]] NCGroup group (const std::string&) const; //! Get the dimension instance by name - [[nodiscard]] NCDim dim(const std::string&) const; + [[nodiscard]] NCDim dim (const std::string&) const; //! Get the variable instance by name - [[nodiscard]] NCVar var(const std::string&) const; + [[nodiscard]] NCVar var (const std::string&) const; //! Define new group - [[nodiscard]] NCGroup def_group(const std::string&) const; + [[nodiscard]] NCGroup def_group (const std::string&) const; //! Define new dimension - [[nodiscard]] NCDim def_dim(const std::string&, size_t len) const; + [[nodiscard]] NCDim def_dim (const std::string&, size_t len) const; //! Define a scalar variable, i.e., 0-dimensional array - [[nodiscard]] NCVar def_scalar(const std::string& name, nc_type dtype) const; + [[nodiscard]] NCVar def_scalar (const std::string& name, nc_type dtype) const; //! Define an array - [[nodiscard]] NCVar def_array( - const std::string& name, - nc_type dtype, - const std::vector&) const; + [[nodiscard]] NCVar def_array (const std::string& name, + nc_type dtype, + const std::vector&) const; //! Define a variable (wrapper for def_array) - [[nodiscard]] NCVar def_var( - const std::string& name, - const nc_type dtype, - const std::vector& dnames) const + [[nodiscard]] NCVar def_var (const std::string& name, + const nc_type dtype, + const std::vector& dnames) const { return def_array(name, dtype, dnames); } - void put_attr(const std::string& name, const std::string& value) const; - void put_attr(const std::string& name, const std::vector& value) const; - void put_attr(const std::string& name, const std::vector& value) const; - void put_attr(const std::string& name, const std::vector& value) const; + void put_attr (const std::string& name, const std::string& value) const; + void put_attr (const std::string& name, const std::vector& value) const; + void put_attr (const std::string& name, const std::vector& value) const; + void put_attr (const std::string& name, const std::vector& value) const; - [[nodiscard]] std::string get_attr(const std::string& name) const; - void get_attr(const std::string& name, std::vector& value) const; - void get_attr(const std::string& name, std::vector& value) const; - void get_attr(const std::string& name, std::vector& value) const; + [[nodiscard]] std::string get_attr (const std::string& name) const; + void get_attr (const std::string& name, std::vector& value) const; + void get_attr (const std::string& name, std::vector& value) const; + void get_attr (const std::string& name, std::vector& value) const; //! Return a list of all groups defined in this group - [[nodiscard]] std::vector all_groups() const; + [[nodiscard]] std::vector all_groups () const; //! Return a list of all dimensions defined in this group - [[nodiscard]] std::vector all_dims() const; + [[nodiscard]] std::vector all_dims () const; //! Return a list of all variables defined in this group - [[nodiscard]] std::vector all_vars() const; + [[nodiscard]] std::vector all_vars () const; //! Enter definition mode (not needed for NetCDF4 format) - void enter_def_mode() const; + void enter_def_mode () const; //! Exit definition mode - void exit_def_mode() const; + void exit_def_mode () const; protected: - NCGroup(const int id) : ncid(id) {} - NCGroup(const int id, const NCGroup* /*par*/) : ncid(id) {} + NCGroup (const int id) : ncid(id) {} + NCGroup (const int id, const NCGroup* /*par*/) : ncid(id) {} }; /** Representation of a NetCDF file @@ -285,28 +291,26 @@ class NCFile : public NCGroup { public: static NCFile - create(const std::string& name, const int cmode = NC_CLOBBER | NC_NETCDF4); + create (const std::string& name, const int cmode = NC_CLOBBER | NC_NETCDF4); - static NCFile open(const std::string& name, const int cmode = NC_NOWRITE); + static NCFile open (const std::string& name, const int cmode = NC_NOWRITE); - static NCFile create_par( - const std::string& name, - const int cmode = NC_CLOBBER | NC_NETCDF4 | NC_MPIIO, - MPI_Comm comm = MPI_COMM_WORLD, - MPI_Info info = MPI_INFO_NULL); + static NCFile create_par (const std::string& name, + const int cmode = NC_CLOBBER | NC_NETCDF4 | NC_MPIIO, + MPI_Comm comm = MPI_COMM_WORLD, + MPI_Info info = MPI_INFO_NULL); - static NCFile open_par( - const std::string& name, - const int cmode = NC_NOWRITE, - MPI_Comm comm = MPI_COMM_WORLD, - MPI_Info info = MPI_INFO_NULL); + static NCFile open_par (const std::string& name, + const int cmode = NC_NOWRITE, + MPI_Comm comm = MPI_COMM_WORLD, + MPI_Info info = MPI_INFO_NULL); - ~NCFile(); + ~NCFile (); - void close(); + void close (); protected: - NCFile(const int id) : NCGroup(id), is_open{true} {} + NCFile (const int id) : NCGroup(id), is_open{true} {} bool is_open{false}; }; diff --git a/Source/IO/NCInterface.cpp b/Source/IO/NCInterface.cpp index fb61f1288..8a2000cb0 100644 --- a/Source/IO/NCInterface.cpp +++ b/Source/IO/NCInterface.cpp @@ -16,7 +16,7 @@ char recname[NC_MAX_NAME + 1]; * * @param ierr Error flag from NetCDF */ -void check_nc_error(int ierr) +void check_nc_error (int ierr) { if (ierr != NC_NOERR) { printf("\n%s\n\n", nc_strerror(ierr)); @@ -28,7 +28,7 @@ void check_nc_error(int ierr) /** * Error-checking wrapper for NetCDF function nc_inq_dimname */ -std::string NCDim::name() const +std::string NCDim::name () const { check_nc_error(nc_inq_dimname(ncid, dimid, recname)); return std::string(recname); @@ -37,7 +37,7 @@ std::string NCDim::name() const /** * Error-checking wrapper for NetCDF function nc_inq_dimlen */ -size_t NCDim::len() const +size_t NCDim::len () const { size_t dlen; check_nc_error(nc_inq_dimlen(ncid, dimid, &dlen)); @@ -47,7 +47,7 @@ size_t NCDim::len() const /** * Error-checking wrapper for NetCDF function nc_inq_varname */ -std::string NCVar::name() const +std::string NCVar::name () const { check_nc_error(nc_inq_varname(ncid, varid, recname)); return std::string(recname); @@ -56,7 +56,7 @@ std::string NCVar::name() const /** * Error-checking wrapper for NetCDF function nc_inq_varndims */ -int NCVar::ndim() const +int NCVar::ndim () const { int ndims; check_nc_error(nc_inq_varndims(ncid, varid, &ndims)); @@ -66,7 +66,7 @@ int NCVar::ndim() const /** * Error-checking function to get the length of each dimension from a NetCDF identity */ -std::vector NCVar::shape() const +std::vector NCVar::shape () const { int ndims = ndim(); std::vector dimids(ndims); @@ -86,7 +86,7 @@ std::vector NCVar::shape() const * * @param ptr Pointer to the data to put */ -void NCVar::put(const double* ptr) const +void NCVar::put (const double* ptr) const { check_nc_error(nc_put_var_double(ncid, varid, ptr)); } @@ -96,7 +96,7 @@ void NCVar::put(const double* ptr) const * * @param ptr Pointer to the data to put */ -void NCVar::put(const float* ptr) const +void NCVar::put (const float* ptr) const { check_nc_error(nc_put_var_float(ncid, varid, ptr)); } @@ -106,7 +106,7 @@ void NCVar::put(const float* ptr) const * * @param ptr Pointer to the data to put */ -void NCVar::put(const int* ptr) const +void NCVar::put (const int* ptr) const { check_nc_error(nc_put_var_int(ncid, varid, ptr)); } @@ -118,10 +118,9 @@ void NCVar::put(const int* ptr) const * @param start Starting indices * @param count Count sizes */ -void NCVar::put( - const double* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::put (const double* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_put_vara_double(ncid, varid, start.data(), count.data(), dptr)); @@ -135,11 +134,10 @@ void NCVar::put( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::put( - const double* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::put (const double* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_put_vars_double( ncid, varid, start.data(), count.data(), stride.data(), dptr)); @@ -152,10 +150,9 @@ void NCVar::put( * @param start Starting indices * @param count Count sizes */ -void NCVar::put( - const float* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::put (const float* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_put_vara_float(ncid, varid, start.data(), count.data(), dptr)); @@ -169,11 +166,10 @@ void NCVar::put( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::put( - const float* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::put (const float* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_put_vars_float( ncid, varid, start.data(), count.data(), stride.data(), dptr)); @@ -186,10 +182,9 @@ void NCVar::put( * @param start Starting indices * @param count Count sizes */ -void NCVar::put( - const int* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::put (const int* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_put_vara_int(ncid, varid, start.data(), count.data(), dptr)); @@ -203,11 +198,10 @@ void NCVar::put( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::put( - const int* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::put (const int* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_put_vars_int( ncid, varid, start.data(), count.data(), stride.data(), dptr)); @@ -220,10 +214,9 @@ void NCVar::put( * @param start Starting indices * @param count Count sizes */ -void NCVar::put( - const char** dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::put (const char** dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_put_vara_string(ncid, varid, start.data(), count.data(), dptr)); @@ -237,11 +230,10 @@ void NCVar::put( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::put( - const char** dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::put (const char** dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_put_vars_string( ncid, varid, start.data(), count.data(), stride.data(), dptr)); @@ -252,7 +244,7 @@ void NCVar::put( * * @param ptr Pointer to the data location we use to get */ -void NCVar::get(double* ptr) const +void NCVar::get (double* ptr) const { check_nc_error(nc_get_var_double(ncid, varid, ptr)); } @@ -262,7 +254,7 @@ void NCVar::get(double* ptr) const * * @param ptr Pointer to the data location we use to get */ -void NCVar::get(float* ptr) const +void NCVar::get (float* ptr) const { check_nc_error(nc_get_var_float(ncid, varid, ptr)); } @@ -272,7 +264,7 @@ void NCVar::get(float* ptr) const * * @param ptr Pointer to the data location we use to get */ -void NCVar::get(int* ptr) const +void NCVar::get (int* ptr) const { check_nc_error(nc_get_var_int(ncid, varid, ptr)); } @@ -284,10 +276,9 @@ void NCVar::get(int* ptr) const * @param start Starting indices * @param count Count sizes */ -void NCVar::get( - double* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::get (double* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_get_vara_double(ncid, varid, start.data(), count.data(), dptr)); @@ -301,11 +292,10 @@ void NCVar::get( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::get( - double* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::get (double* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_get_vars_double( ncid, varid, start.data(), count.data(), stride.data(), dptr)); @@ -318,10 +308,9 @@ void NCVar::get( * @param start Starting indices * @param count Count sizes */ -void NCVar::get( - float* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::get (float* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_get_vara_float(ncid, varid, start.data(), count.data(), dptr)); @@ -335,55 +324,50 @@ void NCVar::get( * @param count Count sizes * @param stride Stride length for the data */ -void NCVar::get( - float* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::get (float* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_get_vars_float( ncid, varid, start.data(), count.data(), stride.data(), dptr)); } -void NCVar::get( - int* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::get (int* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_get_vara_int(ncid, varid, start.data(), count.data(), dptr)); } -void NCVar::get( - int* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::get (int* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_get_vars_int( ncid, varid, start.data(), count.data(), stride.data(), dptr)); } -void NCVar::get( - char* dptr, - const std::vector& start, - const std::vector& count) const +void NCVar::get (char* dptr, + const std::vector& start, + const std::vector& count) const { check_nc_error( nc_get_vara_text(ncid, varid, start.data(), count.data(), dptr)); } -void NCVar::get( - char* dptr, - const std::vector& start, - const std::vector& count, - const std::vector& stride) const +void NCVar::get (char* dptr, + const std::vector& start, + const std::vector& count, + const std::vector& stride) const { check_nc_error(nc_get_vars_text( ncid, varid, start.data(), count.data(), stride.data(), dptr)); } -bool NCVar::has_attr(const std::string& name) const +bool NCVar::has_attr (const std::string& name) const { int ierr; size_t lenp; @@ -391,34 +375,31 @@ bool NCVar::has_attr(const std::string& name) const return (ierr == NC_NOERR); } -void NCVar::put_attr(const std::string& name, const std::string& value) const +void NCVar::put_attr (const std::string& name, const std::string& value) const { check_nc_error( nc_put_att_text(ncid, varid, name.data(), value.size(), value.data())); } -void NCVar::put_attr( - const std::string& name, const std::vector& value) const +void NCVar::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_double( ncid, varid, name.data(), NC_DOUBLE, value.size(), value.data())); } -void NCVar::put_attr( - const std::string& name, const std::vector& value) const +void NCVar::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_float( ncid, varid, name.data(), NC_FLOAT, value.size(), value.data())); } -void NCVar::put_attr( - const std::string& name, const std::vector& value) const +void NCVar::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_int( ncid, varid, name.data(), NC_INT, value.size(), value.data())); } -std::string NCVar::get_attr(const std::string& name) const +std::string NCVar::get_attr (const std::string& name) const { size_t lenp; std::vector aval; @@ -428,7 +409,7 @@ std::string NCVar::get_attr(const std::string& name) const return std::string{aval.begin(), aval.end()}; } -void NCVar::get_attr(const std::string& name, std::vector& values) const +void NCVar::get_attr (const std::string& name, std::vector& values) const { size_t lenp; check_nc_error(nc_inq_attlen(ncid, varid, name.data(), &lenp)); @@ -436,7 +417,7 @@ void NCVar::get_attr(const std::string& name, std::vector& values) const check_nc_error(nc_get_att_double(ncid, varid, name.data(), values.data())); } -void NCVar::get_attr(const std::string& name, std::vector& values) const +void NCVar::get_attr (const std::string& name, std::vector& values) const { size_t lenp; check_nc_error(nc_inq_attlen(ncid, varid, name.data(), &lenp)); @@ -444,7 +425,7 @@ void NCVar::get_attr(const std::string& name, std::vector& values) const check_nc_error(nc_get_att_float(ncid, varid, name.data(), values.data())); } -void NCVar::get_attr(const std::string& name, std::vector& values) const +void NCVar::get_attr (const std::string& name, std::vector& values) const { size_t lenp; check_nc_error(nc_inq_attlen(ncid, varid, name.data(), &lenp)); @@ -453,12 +434,12 @@ void NCVar::get_attr(const std::string& name, std::vector& values) const } //Uncomment for parallel NetCDF -void NCVar::par_access(const int cmode) const +void NCVar::par_access (const int cmode) const { check_nc_error(nc_var_par_access(ncid, varid, cmode)); } -std::string NCGroup::name() const +std::string NCGroup::name () const { size_t nlen; std::vector grpname; @@ -468,7 +449,7 @@ std::string NCGroup::name() const return std::string{grpname.begin(), grpname.end()}; } -std::string NCGroup::full_name() const +std::string NCGroup::full_name () const { size_t nlen; std::vector grpname; @@ -478,45 +459,44 @@ std::string NCGroup::full_name() const return std::string{grpname.begin(), grpname.end()}; } -NCGroup NCGroup::def_group(const std::string& name) const +NCGroup NCGroup::def_group (const std::string& name) const { int newid; check_nc_error(nc_def_grp(ncid, name.data(), &newid)); return NCGroup(newid, this); } -NCGroup NCGroup::group(const std::string& name) const +NCGroup NCGroup::group (const std::string& name) const { int newid; check_nc_error(nc_inq_ncid(ncid, name.data(), &newid)); return NCGroup(newid, this); } -NCDim NCGroup::dim(const std::string& name) const +NCDim NCGroup::dim (const std::string& name) const { int newid; check_nc_error(nc_inq_dimid(ncid, name.data(), &newid)); return NCDim{ncid, newid}; } -NCDim NCGroup::def_dim(const std::string& name, const size_t len) const +NCDim NCGroup::def_dim (const std::string& name, const size_t len) const { int newid; check_nc_error(nc_def_dim(ncid, name.data(), len, &newid)); return NCDim{ncid, newid}; } -NCVar NCGroup::def_scalar(const std::string& name, const nc_type dtype) const +NCVar NCGroup::def_scalar (const std::string& name, const nc_type dtype) const { int newid; check_nc_error(nc_def_var(ncid, name.data(), dtype, 0, nullptr, &newid)); return NCVar{ncid, newid}; } -NCVar NCGroup::def_array( - const std::string& name, - const nc_type dtype, - const std::vector& dnames) const +NCVar NCGroup::def_array (const std::string& name, + const nc_type dtype, + const std::vector& dnames) const { int newid; int ndims = dnames.size(); @@ -528,28 +508,28 @@ NCVar NCGroup::def_array( return NCVar{ncid, newid}; } -NCVar NCGroup::var(const std::string& name) const +NCVar NCGroup::var (const std::string& name) const { int varid; check_nc_error(nc_inq_varid(ncid, name.data(), &varid)); return NCVar{ncid, varid}; } -int NCGroup::num_groups() const +int NCGroup::num_groups () const { int ngrps; check_nc_error(nc_inq_grps(ncid, &ngrps, nullptr)); return ngrps; } -int NCGroup::num_dimensions() const +int NCGroup::num_dimensions () const { int ndims; check_nc_error(nc_inq(ncid, &ndims, nullptr, nullptr, nullptr)); return ndims; } -int NCGroup::num_attributes() const +int NCGroup::num_attributes () const { int nattrs; check_nc_error(nc_inq(ncid, nullptr, nullptr, &nattrs, nullptr)); @@ -563,25 +543,25 @@ int NCGroup::num_variables() const return nvars; } -bool NCGroup::has_group(const std::string& name) const +bool NCGroup::has_group (const std::string& name) const { int ierr = nc_inq_ncid(ncid, name.data(), nullptr); return (ierr == NC_NOERR); } -bool NCGroup::has_dim(const std::string& name) const +bool NCGroup::has_dim (const std::string& name) const { int ierr = nc_inq_dimid(ncid, name.data(), nullptr); return (ierr == NC_NOERR); } -bool NCGroup::has_var(const std::string& name) const +bool NCGroup::has_var (const std::string& name) const { int ierr = nc_inq_varid(ncid, name.data(), nullptr); return (ierr == NC_NOERR); } -bool NCGroup::has_attr(const std::string& name) const +bool NCGroup::has_attr (const std::string& name) const { int ierr; size_t lenp; @@ -589,34 +569,31 @@ bool NCGroup::has_attr(const std::string& name) const return (ierr == NC_NOERR); } -void NCGroup::put_attr(const std::string& name, const std::string& value) const +void NCGroup::put_attr (const std::string& name, const std::string& value) const { check_nc_error(nc_put_att_text( ncid, NC_GLOBAL, name.data(), value.size(), value.data())); } -void NCGroup::put_attr( - const std::string& name, const std::vector& value) const +void NCGroup::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_double( ncid, NC_GLOBAL, name.data(), NC_DOUBLE, value.size(), value.data())); } -void NCGroup::put_attr( - const std::string& name, const std::vector& value) const +void NCGroup::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_float( ncid, NC_GLOBAL, name.data(), NC_FLOAT, value.size(), value.data())); } -void NCGroup::put_attr( - const std::string& name, const std::vector& value) const +void NCGroup::put_attr (const std::string& name, const std::vector& value) const { check_nc_error(nc_put_att_int( ncid, NC_GLOBAL, name.data(), NC_INT, value.size(), value.data())); } -std::string NCGroup::get_attr(const std::string& name) const +std::string NCGroup::get_attr (const std::string& name) const { size_t lenp; std::vector aval; @@ -626,8 +603,7 @@ std::string NCGroup::get_attr(const std::string& name) const return std::string{aval.begin(), aval.end()}; } -void NCGroup::get_attr( - const std::string& name, std::vector& values) const +void NCGroup::get_attr (const std::string& name, std::vector& values) const { size_t lenp; check_nc_error(nc_inq_attlen(ncid, NC_GLOBAL, name.data(), &lenp)); @@ -636,8 +612,7 @@ void NCGroup::get_attr( nc_get_att_double(ncid, NC_GLOBAL, name.data(), values.data())); } -void NCGroup::get_attr( - const std::string& name, std::vector& values) const +void NCGroup::get_attr (const std::string& name, std::vector& values) const { size_t lenp; check_nc_error(nc_inq_attlen(ncid, NC_GLOBAL, name.data(), &lenp)); @@ -654,7 +629,7 @@ void NCGroup::get_attr(const std::string& name, std::vector& values) const check_nc_error(nc_get_att_int(ncid, NC_GLOBAL, name.data(), values.data())); } -std::vector NCGroup::all_groups() const +std::vector NCGroup::all_groups () const { std::vector grps; int ngrps = num_groups(); @@ -669,7 +644,7 @@ std::vector NCGroup::all_groups() const return grps; } -std::vector NCGroup::all_dims() const +std::vector NCGroup::all_dims () const { std::vector adims; int ndims = num_dimensions(); @@ -680,7 +655,7 @@ std::vector NCGroup::all_dims() const return adims; } -std::vector NCGroup::all_vars() const +std::vector NCGroup::all_vars () const { std::vector avars; int nvars = num_variables(); @@ -691,7 +666,7 @@ std::vector NCGroup::all_vars() const return avars; } -void NCGroup::enter_def_mode() const +void NCGroup::enter_def_mode () const { int ierr; ierr = nc_redef(ncid); @@ -702,24 +677,23 @@ void NCGroup::enter_def_mode() const check_nc_error(ierr); } -void NCGroup::exit_def_mode() const { check_nc_error(nc_enddef(ncid)); } +void NCGroup::exit_def_mode () const { check_nc_error(nc_enddef(ncid)); } -NCFile NCFile::create(const std::string& name, const int cmode) +NCFile NCFile::create (const std::string& name, const int cmode) { int ncid; check_nc_error(nc_create(name.data(), cmode, &ncid)); return NCFile(ncid); } -NCFile NCFile::open(const std::string& name, const int cmode) +NCFile NCFile::open (const std::string& name, const int cmode) { int ncid; check_nc_error(nc_open(name.data(), cmode, &ncid)); return NCFile(ncid); } //Uncomment for parallel NetCDF -NCFile NCFile::create_par( - const std::string& name, const int cmode, MPI_Comm comm, MPI_Info info) +NCFile NCFile::create_par (const std::string& name, const int cmode, MPI_Comm comm, MPI_Info info) { int ncid; check_nc_error(nc_create_par(name.data(), cmode, comm, info, &ncid)); @@ -727,20 +701,19 @@ NCFile NCFile::create_par( } //Uncomment for parallel NetCDF -NCFile NCFile::open_par( - const std::string& name, const int cmode, MPI_Comm comm, MPI_Info info) +NCFile NCFile::open_par (const std::string& name, const int cmode, MPI_Comm comm, MPI_Info info) { int ncid; check_nc_error(nc_open_par(name.data(), cmode, comm, info, &ncid)); return NCFile(ncid); } -NCFile::~NCFile() +NCFile::~NCFile () { if (is_open) check_nc_error(nc_close(ncid)); } -void NCFile::close() +void NCFile::close () { is_open = false; check_nc_error(nc_close(ncid)); diff --git a/Source/IO/NCMultiFabFile.cpp b/Source/IO/NCMultiFabFile.cpp index 61fce6227..3235380fe 100644 --- a/Source/IO/NCMultiFabFile.cpp +++ b/Source/IO/NCMultiFabFile.cpp @@ -20,10 +20,10 @@ using namespace amrex; void -ERF::ReadNCMultiFab(FabArray &mf, - const std::string &mf_name, - int /*coordinatorProc*/, - int /*allow_empty_mf*/) { +ERF::ReadNCMultiFab (FabArray &mf, + const std::string &mf_name, + int /*coordinatorProc*/, + int /*allow_empty_mf*/) { const std::string& FullPath = amrex::Concatenate(check_file,istep[0],5); diff --git a/Source/IO/NCPlotFile.cpp b/Source/IO/NCPlotFile.cpp index bbb094d71..807a1401f 100644 --- a/Source/IO/NCPlotFile.cpp +++ b/Source/IO/NCPlotFile.cpp @@ -19,10 +19,10 @@ using namespace amrex; void -ERF::writeNCPlotFile(int lev, int which_subdomain, const std::string& dir, - const Vector &plotMF, - const Vector &plot_var_names, - const Vector& level_steps, const Real time) const +ERF::writeNCPlotFile (int lev, int which_subdomain, const std::string& dir, + const Vector &plotMF, + const Vector &plot_var_names, + const Vector& level_steps, const Real time) const { // get the processor number int iproc = amrex::ParallelContext::MyProcAll(); @@ -132,6 +132,7 @@ ERF::writeNCPlotFile(int lev, int which_subdomain, const std::string& dir, ncf.put_attr("number_variables", std::vector{n_data_items}); ncf.put_attr("space_dimension", std::vector{AMREX_SPACEDIM}); ncf.put_attr("current_time", std::vector{time}); + ncf.put_attr("start_time", std::vector{start_bdy_time}); ncf.put_attr("CurrentLevel", std::vector{flev}); Real dx[AMREX_SPACEDIM]; diff --git a/Source/IO/NCWpsFile.H b/Source/IO/NCWpsFile.H index 097fa7502..5e4994d64 100644 --- a/Source/IO/NCWpsFile.H +++ b/Source/IO/NCWpsFile.H @@ -7,14 +7,19 @@ #include #include "AMReX_FArrayBox.H" +#include "AMReX_IArrayBox.H" #include "NCInterface.H" using PlaneVector = amrex::Vector; /* // Read from metgrid - NetCDF variables of dimensions Time_BT_SN_WE: "UU", "VV", "TT", "RH", "PRES" - NetCDF variables of dimensions Time_SN_WE : "HGT", "MAPFAC_U", "MAPFAC_V", "MAPFAC_M" + NetCDF variables of dimensions Time_BT_SN_WE: "UU", "VV", "TT", "RH", "PRES", "GHT" + NetCDF variables of dimensions Time_SN_WE : "HGT", "MAPFAC_U", "MAPFAC_V", "MAPFAC_M", "PSFC" + NetCDF global attributes of type int : "FLAG_PSFC", "FLAG_MAPFAC_U", "FLAG_MAPFAC_V", "FLAG_MAPFAC_M", + "FLAG_HGT_M", "WEST-EAST_GRID_DIMENSION", "SOUTH-NORTH_GRID_DIMENSION" + NetCDF global attributes of type string : "SIMULATION_START_DATE" + NetCDF global attributes of type real : "DX", "DY" // Read from wrfbdy NetCDF variables of dimensions Time_BdyWidth_BT_SN : "U_BXS", "U_BXE", "V_BXS", "V_BXE" etc. @@ -33,18 +38,6 @@ enum class NC_Data_Dims_Type { Time_BdyWidth_WE }; -void BuildFABsFromNetCDFFile(const amrex::Box& domain, - const std::string &fname, - amrex::Vector nc_var_names, - amrex::Vector NC_dim_types, - amrex::Vector fab_vars); - -int BuildFABsFromWRFBdyFile(const std::string &fname, - amrex::Vector>& bdy_data_xlo, - amrex::Vector>& bdy_data_xhi, - amrex::Vector>& bdy_data_ylo, - amrex::Vector>& bdy_data_yhi); - // // NDArray is the datatype designed to hold any data, including scalars, multidimensional // arrays, that read from the NetCDF file. @@ -65,10 +58,10 @@ struct NDArray } // default constructor - NDArray() : name{"null"}, ref_counted{1}, owned{false}, data{nullptr} {} + NDArray () : name{"null"}, ref_counted{1}, owned{false}, data{nullptr} {} // copy constructor - NDArray(const NDArray& array) { + NDArray (const NDArray& array) { name = array.name; shape = array.shape; data = array.data; @@ -87,29 +80,29 @@ struct NDArray } // destructor - ~NDArray() { + ~NDArray () { ref_counted.fetch_sub(1, std::memory_order_acq_rel); if (ref_counted == 1 && owned) delete[] data; } // get the data pointer - decltype(auto) get_data() { + decltype(auto) get_data () { ref_counted.fetch_add(1, std::memory_order_relaxed); return data; } // get the variable name - std::string get_vname() { + std::string get_vname () { return name; } // get the variable data shape - std::vector get_vshape() { + std::vector get_vshape () { return shape; } // return the total number of data - size_t ndim() { + size_t ndim () { size_t num = 1; int isize = static_cast(shape.size()); for (auto i=0; i vshape) { + void set_vshape (std::vector vshape) { shape = vshape; } @@ -129,9 +122,40 @@ struct NDArray DType* data; }; +int BuildFABsFromWRFBdyFile (const std::string &fname, + amrex::Vector>& bdy_data_xlo, + amrex::Vector>& bdy_data_xhi, + amrex::Vector>& bdy_data_ylo, + amrex::Vector>& bdy_data_yhi); + +AMREX_FORCE_INLINE +std::time_t +getEpochTime (const std::string& dateTime, const std::string& dateTimeFormat) +{ + // Create a stream which we will use to parse the string, + // which we provide to constructor of stream to fill the buffer. + std::istringstream ss{ dateTime }; + + // Create a tm object to store the parsed date and time. + std::tm tmTime; + memset(&tmTime, 0, sizeof(tmTime)); + + // Now we read from buffer using get_time manipulator + // and formatting the input appropriately. + strptime(dateTime.c_str(), dateTimeFormat.c_str(), &tmTime); + + // Convert the tm structure to time_t value and return. + // Here we use timegm since the output should be relative to UTC. + auto epoch = timegm(&tmTime); + // Print() << "Time Stamp: "<< std::put_time(&tmTime, "%c") + // << " , Epoch: " << epoch << std::endl; + + return epoch; +} + template -void ReadNetCDFFile(const std::string& fname, amrex::Vector names, - amrex::Vector >& arrays) +void ReadNetCDFFile (const std::string& fname, amrex::Vector names, + amrex::Vector >& arrays) { AMREX_ASSERT(arrays.size() == names.size()); @@ -160,7 +184,12 @@ void ReadNetCDFFile(const std::string& fname, amrex::Vector names, if (vname_to_read.substr(0,2) == "R_") { vname_to_read = names[n+4]; // This allows us to read "T" instead -- we will over-write this later } - // amrex::Print() << "About to read " << vname_to_read << " while filling the array for " << vname_to_write << std::endl; + + /* + amrex::AllPrint() << "About to read " << vname_to_read + << " while filling the array for " << vname_to_write << std::endl; + */ + std::vector shape = ncf.var(vname_to_read).shape(); arrays[n] = NDArray(vname_to_read,shape); DType* dataPtr = arrays[n].get_data(); @@ -180,4 +209,142 @@ void ReadNetCDFFile(const std::string& fname, amrex::Vector names, ncf.close(); } } + +/** + * Helper function for reading data from NetCDF file into a + * provided FAB. + * + * @param iv Index for which variable we are going to fill + * @param nc_arrays Arrays of data from NetCDF file + * @param var_name Variable name + * @param NC_dim_type Dimension type for the variable as stored in the NetCDF file + * @param temp FAB where we store the variable data from the NetCDF Arrays + */ +template +void +fill_fab_from_arrays (int iv, + amrex::Vector>& nc_arrays, + const std::string& var_name, + NC_Data_Dims_Type& NC_dim_type, + FAB& temp) +{ + int ns1, ns2, ns3; + if (NC_dim_type == NC_Data_Dims_Type::Time_BT) { + ns1 = nc_arrays[iv].get_vshape()[1]; + ns2 = 1; + ns3 = 1; + // amrex::Print() << "TYPE BT " << ns1 << std::endl; + } else if (NC_dim_type == NC_Data_Dims_Type::Time_SN_WE) { + ns1 = 1; + ns2 = nc_arrays[iv].get_vshape()[1]; + ns3 = nc_arrays[iv].get_vshape()[2]; + // amrex::Print() << "TYPE SN WE " << ns2 << " " << ns3 << std::endl; + } else if (NC_dim_type == NC_Data_Dims_Type::Time_BT_SN_WE) { + ns1 = nc_arrays[iv].get_vshape()[1]; + ns2 = nc_arrays[iv].get_vshape()[2]; + ns3 = nc_arrays[iv].get_vshape()[3]; + // amrex::Print() << "TYPE BT SN WE " << ns1 << " " << ns2 << " " << ns3 << std::endl; + } else { + amrex::Abort("Dont know this NC_Data_Dims_Type"); + } + + // TODO: The box will only start at (0,0,0) at level 0 -- we need to generalize this + amrex::Box my_box(amrex::IntVect(0,0,0), amrex::IntVect(ns3-1,ns2-1,ns1-1)); + // amrex::Print() <<" MY BOX " << my_box << std::endl; + + if (var_name == "U" || var_name == "UU" || + var_name == "MAPFAC_U" || var_name == "MAPFAC_UY") my_box.setType(amrex::IndexType(amrex::IntVect(1,0,0))); + if (var_name == "V" || var_name == "VV" || + var_name == "MAPFAC_V" || var_name == "MAPFAC_VY") my_box.setType(amrex::IndexType(amrex::IntVect(0,1,0))); + if (var_name == "W" || var_name == "WW") my_box.setType(amrex::IndexType(amrex::IntVect(0,0,1))); + +#ifdef AMREX_USE_GPU + // Make sure temp lives on CPU since nc_arrays lives on CPU only + temp.resize(my_box,1,amrex::The_Pinned_Arena()); +#else + temp.resize(my_box,1); +#endif + amrex::Array4 fab_arr = temp.array(); + + int ioff = temp.box().smallEnd()[0]; + int joff = temp.box().smallEnd()[1]; + + auto num_pts = my_box.numPts(); + + // amrex::Print() <<" ns1 * ns2 * ns3 " << ns1 * ns2 * ns3 << std::endl; + // amrex::Print() <<" NUMPTS " << num_pts << std::endl; + + for (int n(0); n < num_pts; ++n) { + int k = n / (ns2*ns3); + int j = (n - k*(ns2*ns3)) / ns3 + joff; + int i = n - k*(ns2*ns3) - (j-joff) * ns3 + ioff; + fab_arr(i,j,k,0) = static_cast(*(nc_arrays[iv].get_data()+n)); + } +} + +/** + * Function to read NetCDF variables and fill the corresponding Array4's + * + * @param fname Name of the NetCDF file to be read + * @param nc_var_names Variable names in the NetCDF file + * @param NC_dim_types NetCDF data dimension types + * @param fab_vars Fab data we are to fill + */ +template +void +BuildFABsFromNetCDFFile (const amrex::Box& domain, + const std::string &fname, + amrex::Vector nc_var_names, + amrex::Vector NC_dim_types, + amrex::Vector fab_vars) +{ + int ioproc = amrex::ParallelDescriptor::IOProcessorNumber(); // I/O rank + + amrex::Vector> nc_arrays(nc_var_names.size()); + + if (amrex::ParallelDescriptor::IOProcessor()) + { + ReadNetCDFFile(fname, nc_var_names, nc_arrays); + } + + for (int iv = 0; iv < nc_var_names.size(); iv++) + { + FAB tmp; + if (amrex::ParallelDescriptor::IOProcessor()) { + fill_fab_from_arrays(iv, nc_arrays, nc_var_names[iv], NC_dim_types[iv], tmp); + } + + int ncomp = tmp.nComp(); + amrex::Box box = tmp.box(); + + amrex::ParallelDescriptor::Bcast(&box, 1, ioproc); + amrex::ParallelDescriptor::Bcast(&ncomp, 1, ioproc); + + if (!amrex::ParallelDescriptor::IOProcessor()) { +#ifdef AMREX_USE_GPU + tmp.resize(box,ncomp,amrex::The_Pinned_Arena()); +#else + tmp.resize(box,ncomp); +#endif + } + + amrex::ParallelDescriptor::Bcast(tmp.dataPtr(), tmp.size(), ioproc); + + // Shift box by the domain lower corner + amrex::Box fab_bx = tmp.box(); + amrex::Dim3 dom_lb = lbound(domain); + fab_bx += amrex::IntVect(dom_lb.x,dom_lb.y,dom_lb.z); + // fab_vars points to data on device + fab_vars[iv]->resize(fab_bx,1); +#ifdef AMREX_USE_GPU + amrex::Gpu::copy(amrex::Gpu::hostToDevice, + tmp.dataPtr(), tmp.dataPtr() + tmp.size(), + fab_vars[iv]->dataPtr()); +#else + // Provided by BaseFab inheritance through FArrayBox + fab_vars[iv]->copy(tmp,tmp.box(),0,fab_bx,0,1); +#endif + } +} + #endif diff --git a/Source/IO/Plotfile.cpp b/Source/IO/Plotfile.cpp index 5763a585a..41729e723 100644 --- a/Source/IO/Plotfile.cpp +++ b/Source/IO/Plotfile.cpp @@ -8,7 +8,7 @@ using namespace amrex; template -bool containerHasElement(const V& iterable, const T& query) { +bool containerHasElement (const V& iterable, const T& query) { return std::find(iterable.begin(), iterable.end(), query) != iterable.end(); } @@ -78,7 +78,7 @@ ERF::setPlotVariables (const std::string& pp_plot_var_names, Vector // set plotfile variable names Vector -ERF::PlotFileVarNames ( Vector plot_var_names ) +ERF::PlotFileVarNames (Vector plot_var_names ) { Vector names; diff --git a/Source/IO/ReadFromMetgrid.cpp b/Source/IO/ReadFromMetgrid.cpp index 094469275..43a82fb7d 100644 --- a/Source/IO/ReadFromMetgrid.cpp +++ b/Source/IO/ReadFromMetgrid.cpp @@ -1,42 +1,98 @@ -#include "NCWpsFile.H" -#include "AMReX_FArrayBox.H" +#include +#include +#include using namespace amrex; #ifdef ERF_USE_NETCDF + void -read_from_metgrid(int lev, - const Box& domain, - const std::string& fname, - FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab, - FArrayBox& NC_temp_fab, FArrayBox& NC_rhum_fab, - FArrayBox& NC_pres_fab, FArrayBox& NC_hgt_fab, - FArrayBox& NC_msfu_fab, FArrayBox& NC_msfv_fab, - FArrayBox& NC_msfm_fab) +read_from_metgrid (int lev, const Box& domain, const std::string& fname, + std::string& NC_dateTime, Real& NC_epochTime, + int& flag_psfc, int& flag_msfu, int& flag_msfv, int& flag_msfm, + int& flag_hgt, int& flag_sst, int& flag_lmask, + int& NC_nx, int& NC_ny, + Real& NC_dx, Real& NC_dy, + FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab, + FArrayBox& NC_temp_fab, FArrayBox& NC_rhum_fab, + FArrayBox& NC_pres_fab, FArrayBox& NC_ght_fab, + FArrayBox& NC_hgt_fab, FArrayBox& NC_psfc_fab, + FArrayBox& NC_msfu_fab, FArrayBox& NC_msfv_fab, + FArrayBox& NC_msfm_fab, FArrayBox& NC_sst_fab, + IArrayBox& NC_lmask_iab) { amrex::Print() << "Loading initial data from NetCDF file at level " << lev << std::endl; - int ncomp = 1; + if (amrex::ParallelDescriptor::IOProcessor()) { + auto ncf = ncutils::NCFile::open(fname, NC_CLOBBER | NC_NETCDF4); + { // Global Attributes (int) + std::vector attr; + ncf.get_attr("FLAG_PSFC", attr); flag_psfc = attr[0]; + ncf.get_attr("FLAG_MAPFAC_U", attr); flag_msfu = attr[0]; + ncf.get_attr("FLAG_MAPFAC_V", attr); flag_msfv = attr[0]; + ncf.get_attr("FLAG_MAPFAC_M", attr); flag_msfm = attr[0]; + ncf.get_attr("FLAG_HGT_M", attr); flag_hgt = attr[0]; + ncf.get_attr("FLAG_SST", attr); flag_sst = attr[0]; + ncf.get_attr("FLAG_LANDMASK", attr); flag_lmask = attr[0]; + ncf.get_attr("WEST-EAST_GRID_DIMENSION", attr); NC_nx = attr[0]; + ncf.get_attr("SOUTH-NORTH_GRID_DIMENSION", attr); NC_ny = attr[0]; + } + { // Global Attributes (string) + NC_dateTime = ncf.get_attr("SIMULATION_START_DATE")+"UTC"; + const std::string dateTimeFormat = "%Y-%m-%d_%H:%M:%S%Z"; + NC_epochTime = getEpochTime(NC_dateTime, dateTimeFormat); + } + { // Global Attributes (Real) + std::vector attr; + ncf.get_attr("DX", attr); NC_dx = attr[0]; + ncf.get_attr("DY", attr); NC_dy = attr[0]; + } + ncf.close(); + } + int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank + ParallelDescriptor::Bcast(&flag_psfc, 1, ioproc); + ParallelDescriptor::Bcast(&flag_msfu, 1, ioproc); + ParallelDescriptor::Bcast(&flag_msfv, 1, ioproc); + ParallelDescriptor::Bcast(&flag_msfm, 1, ioproc); + ParallelDescriptor::Bcast(&flag_hgt, 1, ioproc); + ParallelDescriptor::Bcast(&flag_sst, 1, ioproc); + ParallelDescriptor::Bcast(&flag_lmask, 1, ioproc); + ParallelDescriptor::Bcast(&NC_nx, 1, ioproc); + ParallelDescriptor::Bcast(&NC_ny, 1, ioproc); + ParallelDescriptor::Bcast(&NC_epochTime, 1, ioproc); + ParallelDescriptor::Bcast(&NC_dx, 1, ioproc); + ParallelDescriptor::Bcast(&NC_dy, 1, ioproc); + + Vector NC_fabs; + Vector NC_iabs; + Vector NC_fnames; + Vector NC_inames; + Vector NC_fdim_types; + Vector NC_idim_types; - Vector NC_fabs; - Vector NC_names; - Vector NC_dim_types; + NC_fabs.push_back(&NC_xvel_fab); NC_fnames.push_back("UU"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + NC_fabs.push_back(&NC_yvel_fab); NC_fnames.push_back("VV"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + NC_fabs.push_back(&NC_temp_fab); NC_fnames.push_back("TT"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + NC_fabs.push_back(&NC_rhum_fab); NC_fnames.push_back("RH"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + NC_fabs.push_back(&NC_pres_fab); NC_fnames.push_back("PRES"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + NC_fabs.push_back(&NC_ght_fab); NC_fnames.push_back("GHT"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); - NC_fabs.push_back(&NC_xvel_fab); NC_names.push_back("UU"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); - NC_fabs.push_back(&NC_yvel_fab); NC_names.push_back("VV"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); - NC_fabs.push_back(&NC_temp_fab); NC_names.push_back("TT"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); - NC_fabs.push_back(&NC_rhum_fab); NC_names.push_back("RH"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); - NC_fabs.push_back(&NC_pres_fab); NC_names.push_back("PRES"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); + if (flag_psfc) { NC_fabs.push_back(&NC_psfc_fab); NC_fnames.push_back("PSFC"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } + if (flag_msfu) { NC_fabs.push_back(&NC_msfu_fab); NC_fnames.push_back("MAPFAC_U"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } + if (flag_msfv) { NC_fabs.push_back(&NC_msfv_fab); NC_fnames.push_back("MAPFAC_V"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } + if (flag_msfm) { NC_fabs.push_back(&NC_msfm_fab); NC_fnames.push_back("MAPFAC_M"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } + if (flag_hgt) { NC_fabs.push_back(&NC_hgt_fab); NC_fnames.push_back("HGT_M"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } + if (flag_sst) { NC_fabs.push_back(&NC_sst_fab); NC_fnames.push_back("SST"); NC_fdim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } - NC_fabs.push_back(&NC_hgt_fab); NC_names.push_back("HGT_M"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); - NC_fabs.push_back(&NC_msfu_fab); NC_names.push_back("MAPFAC_U"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); - NC_fabs.push_back(&NC_msfv_fab); NC_names.push_back("MAPFAC_V"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); - NC_fabs.push_back(&NC_msfm_fab); NC_names.push_back("MAPFAC_M"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); + if (flag_lmask) { NC_iabs.push_back(&NC_lmask_iab); NC_inames.push_back("LANDMASK"); NC_idim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); } // Read the netcdf file and fill these FABs amrex::Print() << "Building initial FABS from file " << fname << std::endl; - BuildFABsFromNetCDFFile(domain, fname, NC_names, NC_dim_types, NC_fabs); + BuildFABsFromNetCDFFile(domain, fname, NC_fnames, NC_fdim_types, NC_fabs); + // Read the netcdf file and fill these IABs + amrex::Print() << "Building initial IABS from file " << fname << std::endl; + BuildFABsFromNetCDFFile(domain, fname, NC_inames, NC_idim_types, NC_iabs); // TODO: FIND OUT IF WE NEED TO DIVIDE VELS BY MAPFAC // diff --git a/Source/IO/ReadFromWRFBdy.cpp b/Source/IO/ReadFromWRFBdy.cpp index f111ea490..434d190d1 100644 --- a/Source/IO/ReadFromWRFBdy.cpp +++ b/Source/IO/ReadFromWRFBdy.cpp @@ -10,7 +10,6 @@ #include "DataStruct.H" #include "NCInterface.H" -#include "NCWpsFile.H" #include "AMReX_FArrayBox.H" #include "AMReX_Print.H" @@ -27,36 +26,13 @@ namespace WRFBdyTypes { }; } -// Converts UTC time string to a time_t value. -std::time_t getEpochTime(const std::string& dateTime, const std::string& dateTimeFormat) -{ - // Create a stream which we will use to parse the string, - // which we provide to constructor of stream to fill the buffer. - std::istringstream ss{ dateTime }; - - // Create a tm object to store the parsed date and time. - std::tm tmTime; - memset(&tmTime, 0, sizeof(tmTime)); - - // Now we read from buffer using get_time manipulator - // and formatting the input appropriately. - strptime(dateTime.c_str(), dateTimeFormat.c_str(), &tmTime); - - // Convert the tm structure to time_t value and return. - auto epoch = std::mktime(&tmTime); - // Print() << "Time Stamp: "<< std::put_time(&tmTime, "%c") - // << " , Epoch: " << epoch << std::endl; - - return epoch; -} - Real -read_from_wrfbdy(const std::string& nc_bdy_file, const Box& domain, - Vector>& bdy_data_xlo, - Vector>& bdy_data_xhi, - Vector>& bdy_data_ylo, - Vector>& bdy_data_yhi, - int& width, Real& start_bdy_time) +read_from_wrfbdy (const std::string& nc_bdy_file, const Box& domain, + Vector>& bdy_data_xlo, + Vector>& bdy_data_xhi, + Vector>& bdy_data_ylo, + Vector>& bdy_data_yhi, + int& width, Real& start_bdy_time) { amrex::Print() << "Loading boundary data from NetCDF file " << std::endl; @@ -96,10 +72,11 @@ read_from_wrfbdy(const std::string& nc_bdy_file, const Box& domain, auto epochTime = getEpochTime(date, dateTimeFormat); epochTimes.push_back(epochTime); - if (nt == 1) + if (nt == 1) { timeInterval = epochTimes[1] - epochTimes[0]; - else if (nt >= 1) + } else if (nt >= 1) { AMREX_ALWAYS_ASSERT(epochTimes[nt] - epochTimes[nt-1] == timeInterval); + } } start_bdy_time = epochTimes[0]; } @@ -547,15 +524,15 @@ read_from_wrfbdy(const std::string& nc_bdy_file, const Box& domain, } void -convert_wrfbdy_data(int which, const Box& domain, Vector>& bdy_data, - const FArrayBox& NC_MUB_fab, - const FArrayBox& NC_MSFU_fab, const FArrayBox& NC_MSFV_fab, - const FArrayBox& NC_MSFM_fab, - const FArrayBox& NC_PH_fab, const FArrayBox& NC_PHB_fab, - const FArrayBox& NC_C1H_fab, const FArrayBox& NC_C2H_fab, - const FArrayBox& NC_RDNW_fab, - const FArrayBox& NC_xvel_fab, const FArrayBox& NC_yvel_fab, - const FArrayBox& NC_rho_fab, const FArrayBox& NC_rhotheta_fab) +convert_wrfbdy_data (int which, const Box& domain, Vector>& bdy_data, + const FArrayBox& NC_MUB_fab, + const FArrayBox& NC_MSFU_fab, const FArrayBox& NC_MSFV_fab, + const FArrayBox& NC_MSFM_fab, + const FArrayBox& NC_PH_fab, const FArrayBox& NC_PHB_fab, + const FArrayBox& NC_C1H_fab, const FArrayBox& NC_C2H_fab, + const FArrayBox& NC_RDNW_fab, + const FArrayBox& NC_xvel_fab, const FArrayBox& NC_yvel_fab, + const FArrayBox& NC_rho_fab, const FArrayBox& NC_rhotheta_fab) { // These were filled from wrfinput Array4 c1h_arr = NC_C1H_fab.const_array(); diff --git a/Source/IO/ReadFromWRFInput.cpp b/Source/IO/ReadFromWRFInput.cpp index f2b69ea85..f930af6bb 100644 --- a/Source/IO/ReadFromWRFInput.cpp +++ b/Source/IO/ReadFromWRFInput.cpp @@ -5,25 +5,25 @@ using namespace amrex; #ifdef ERF_USE_NETCDF void -read_from_wrfinput(int lev, - const Box& domain, - const std::string& fname, - FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab, - FArrayBox& NC_zvel_fab, FArrayBox& NC_rho_fab, - FArrayBox& NC_rhop_fab, FArrayBox& NC_rhotheta_fab, - FArrayBox& NC_MUB_fab , - FArrayBox& NC_MSFU_fab, FArrayBox& NC_MSFV_fab, - FArrayBox& NC_MSFM_fab, FArrayBox& NC_SST_fab, - FArrayBox& NC_C1H_fab , FArrayBox& NC_C2H_fab, - FArrayBox& NC_RDNW_fab, +read_from_wrfinput (int lev, + const Box& domain, + const std::string& fname, + FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab, + FArrayBox& NC_zvel_fab, FArrayBox& NC_rho_fab, + FArrayBox& NC_rhop_fab, FArrayBox& NC_rhotheta_fab, + FArrayBox& NC_MUB_fab , + FArrayBox& NC_MSFU_fab, FArrayBox& NC_MSFV_fab, + FArrayBox& NC_MSFM_fab, FArrayBox& NC_SST_fab, + FArrayBox& NC_C1H_fab , FArrayBox& NC_C2H_fab, + FArrayBox& NC_RDNW_fab, #if defined(ERF_USE_MOISTURE) - FArrayBox& NC_QVAPOR_fab, - FArrayBox& NC_QCLOUD_fab, - FArrayBox& NC_QRAIN_fab, + FArrayBox& NC_QVAPOR_fab, + FArrayBox& NC_QCLOUD_fab, + FArrayBox& NC_QRAIN_fab, #elif defined(ERF_USE_WARM_NO_PRECIP) #endif - FArrayBox& NC_PH_fab , FArrayBox& NC_PHB_fab, - FArrayBox& NC_ALB_fab , FArrayBox& NC_PB_fab) + FArrayBox& NC_PH_fab , FArrayBox& NC_PHB_fab, + FArrayBox& NC_ALB_fab , FArrayBox& NC_PB_fab) { amrex::Print() << "Loading initial data from NetCDF file at level " << lev << std::endl; @@ -60,7 +60,7 @@ read_from_wrfinput(int lev, // Read the netcdf file and fill these FABs amrex::Print() << "Building initial FABS from file " << fname << std::endl; - BuildFABsFromNetCDFFile(domain, fname, NC_names, NC_dim_types, NC_fabs); + BuildFABsFromNetCDFFile(domain, fname, NC_names, NC_dim_types, NC_fabs); // // Convert the velocities using the map factors diff --git a/Source/IO/writeJobInfo.cpp b/Source/IO/writeJobInfo.cpp index 0d461ed9d..6c02d1d75 100644 --- a/Source/IO/writeJobInfo.cpp +++ b/Source/IO/writeJobInfo.cpp @@ -4,7 +4,7 @@ extern std::string inputs_name; void -ERF::writeJobInfo(const std::string& dir) const +ERF::writeJobInfo (const std::string& dir) const { // job_info file with details about the run std::ofstream jobInfoFile; @@ -132,7 +132,7 @@ ERF::writeJobInfo(const std::string& dir) const } void -ERF::writeBuildInfo(std::ostream& os) +ERF::writeBuildInfo (std::ostream& os) { std::string PrettyLine = std::string(78, '=') + "\n"; std::string OtherLine = std::string(78, '-') + "\n"; diff --git a/Source/IndexDefines.H b/Source/IndexDefines.H index 306e7a8a6..9c90436a1 100644 --- a/Source/IndexDefines.H +++ b/Source/IndexDefines.H @@ -75,6 +75,17 @@ namespace WRFBdyVars { }; } +namespace MetGridBdyVars { + enum { + U = 0, + V = 1, + R = 2, + T = 3, + QV, + NumTypes + }; +} + namespace Vars { enum { cons = 0, diff --git a/Source/Initialization/ERF_init_from_metgrid.cpp b/Source/Initialization/ERF_init_from_metgrid.cpp index 26453d2db..6037c0de7 100644 --- a/Source/Initialization/ERF_init_from_metgrid.cpp +++ b/Source/Initialization/ERF_init_from_metgrid.cpp @@ -2,52 +2,10 @@ * \file ERF_init_from_metgrid.cpp */ -#include -#include -#include -#include -#include +#include using namespace amrex; -void -read_from_metgrid (int lev, const Box& domain, const std::string& fname, - FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab, - FArrayBox& NC_temp_fab, FArrayBox& NC_rhum_fab, - FArrayBox& NC_pres_fab, FArrayBox& NC_hgt_fab, - FArrayBox& NC_msfu_fab, FArrayBox& NC_msfv_fab, - FArrayBox& NC_msfm_fab); -void -interpolate_column (int i, int j, int src_comp, int dest_comp, - const Array4& orig_z, const Array4& orig_data, - const Array4& new_z, const Array4& new_data); - -void -init_terrain_from_metgrid (int lev, FArrayBox& z_phys_nd_fab, - const Vector& NC_hgt_fab); - -void -init_state_from_metgrid (int lev, FArrayBox& state_fab, - FArrayBox& x_vel_fab, FArrayBox& y_vel_fab, - FArrayBox& z_vel_fab, FArrayBox& z_phys_nd_fab, - const Vector& NC_hgt_fab, - const Vector& NC_xvel_fab, - const Vector& NC_yvel_fab, - const Vector& NC_zvel_fab, - const Vector& NC_rho_fab, - const Vector& NC_rhotheta_fab); -void -init_msfs_from_metgrid (int lev, FArrayBox& msfu_fab, - FArrayBox& msfv_fab, FArrayBox& msfm_fab, - const Vector& NC_MSFU_fab, - const Vector& NC_MSFV_fab, - const Vector& NC_MSFM_fab); -void -init_base_state_from_metgrid (int lev, const Box& valid_bx, Real l_rdOcp, - FArrayBox& p_hse, FArrayBox& pi_hse, FArrayBox& r_hse, - const Vector& NC_ALB_fab, - const Vector& NC_PB_fab); - #ifdef ERF_USE_NETCDF /** * Initializes ERF data using metgrid data supplied by an external NetCDF file. @@ -57,20 +15,18 @@ init_base_state_from_metgrid (int lev, const Box& valid_bx, Real l_rdOcp, void ERF::init_from_metgrid (int lev) { - // *** FArrayBox's at this level for holding the INITIAL data - Vector NC_xvel_fab ; NC_xvel_fab.resize(num_boxes_at_level[lev]); - Vector NC_yvel_fab ; NC_yvel_fab.resize(num_boxes_at_level[lev]); - Vector NC_temp_fab ; NC_temp_fab.resize(num_boxes_at_level[lev]); - Vector NC_rhum_fab ; NC_rhum_fab.resize(num_boxes_at_level[lev]); - Vector NC_pres_fab ; NC_pres_fab.resize(num_boxes_at_level[lev]); - - Vector NC_hgt_fab ; NC_hgt_fab.resize(num_boxes_at_level[lev]); - - Vector NC_MSFU_fab ; NC_MSFU_fab.resize(num_boxes_at_level[lev]); - Vector NC_MSFV_fab ; NC_MSFV_fab.resize(num_boxes_at_level[lev]); - Vector NC_MSFM_fab ; NC_MSFM_fab.resize(num_boxes_at_level[lev]); +#ifndef AMREX_USE_GPU +#if defined(ERF_USE_MOISTURE) + amrex::Print() << "Init with met_em with ERF_USE_MOISTURE" << std::endl; +#elif defined(ERF_USE_WARM_NO_PRECIP) + amrex::Print() << "Init with met_em with ERF_USE_WARM_NO_PRECIP" << std::endl; +#else + amrex::Print() << "Init with met_em without moisture" << std::endl; +#endif +#endif int nboxes = num_boxes_at_level[lev]; + int ntimes = num_files_at_level[lev]; if (nc_init_file.empty()) amrex::Error("NetCDF initialization file name must be provided via input"); @@ -78,13 +34,109 @@ ERF::init_from_metgrid (int lev) if (nc_init_file[lev].empty()) amrex::Error("NetCDF initialization file name must be provided via input"); - for (int idx = 0; idx < nboxes; idx++) - { - read_from_metgrid(lev, boxes_at_level[lev][idx], nc_init_file[lev][idx], - NC_xvel_fab[idx], NC_yvel_fab[idx], - NC_temp_fab[idx], NC_rhum_fab[idx], - NC_pres_fab[idx], NC_hgt_fab[idx], - NC_MSFU_fab[idx], NC_MSFV_fab[idx], NC_MSFM_fab[idx] ); + // At least two met_em files are necessary to calculate tendency terms. + AMREX_ALWAYS_ASSERT(ntimes >= 2); + + // Size the SST and LANDMASK + sst_lev[lev].resize(ntimes); + lmask_lev[lev].resize(ntimes); + + // *** FArrayBox's at this level for holding the metgrid data + Vector NC_xvel_fab; NC_xvel_fab.resize(ntimes); + Vector NC_yvel_fab; NC_yvel_fab.resize(ntimes); + Vector NC_temp_fab; NC_temp_fab.resize(ntimes); + Vector NC_rhum_fab; NC_rhum_fab.resize(ntimes); + Vector NC_pres_fab; NC_pres_fab.resize(ntimes); + Vector NC_ght_fab; NC_ght_fab.resize( ntimes); + Vector NC_hgt_fab; NC_hgt_fab.resize( ntimes); + Vector NC_psfc_fab; NC_psfc_fab.resize(ntimes); + Vector NC_MSFU_fab; NC_MSFU_fab.resize(ntimes); + Vector NC_MSFV_fab; NC_MSFV_fab.resize(ntimes); + Vector NC_MSFM_fab; NC_MSFM_fab.resize(ntimes); + Vector NC_sst_fab; NC_sst_fab.resize (ntimes); + + // *** IArrayBox's at this level for holding mask data + Vector NC_lmask_iab; NC_lmask_iab.resize(ntimes); + + // *** Variables at this level for holding metgrid file global attributes + Vector flag_psfc; flag_psfc.resize( ntimes); + Vector flag_msfu; flag_msfu.resize( ntimes); + Vector flag_msfv; flag_msfv.resize( ntimes); + Vector flag_msfm; flag_msfm.resize( ntimes); + Vector flag_hgt; flag_hgt.resize( ntimes); + Vector flag_sst; flag_sst.resize( ntimes); + Vector flag_lmask; flag_lmask.resize( ntimes); + Vector NC_nx; NC_nx.resize( ntimes); + Vector NC_ny; NC_ny.resize( ntimes); + Vector NC_dateTime; NC_dateTime.resize( ntimes); + Vector NC_epochTime; NC_epochTime.resize(ntimes); + Vector NC_dx; NC_dx.resize( ntimes); + Vector NC_dy; NC_dy.resize( ntimes); + + for (int it = 0; it < ntimes; it++) { +#ifndef AMREX_USE_GPU + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << ": nc_init_file[" << lev << "][" << it << "]\t" << nc_init_file[lev][it] << std::endl; +#endif + read_from_metgrid(lev, boxes_at_level[lev][0], nc_init_file[lev][it], + NC_dateTime[it], NC_epochTime[it], + flag_psfc[it], flag_msfu[it], flag_msfv[it], flag_msfm[it], + flag_hgt[it], flag_sst[it], flag_lmask[it], + NC_nx[it], NC_ny[it], NC_dx[it], NC_dy[it], + NC_xvel_fab[it], NC_yvel_fab[it], + NC_temp_fab[it], NC_rhum_fab[it], NC_pres_fab[it], + NC_ght_fab[it], NC_hgt_fab[it], NC_psfc_fab[it], + NC_MSFU_fab[it], NC_MSFV_fab[it], NC_MSFM_fab[it], + NC_sst_fab[it], NC_lmask_iab[it]); +#ifndef AMREX_USE_GPU + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": flag_psfc \t" << flag_psfc[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": flag_msfu \t" << flag_msfu[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": flag_msfv \t" << flag_msfv[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": flag_msfm \t" << flag_msfm[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": flag_hgt \t" << flag_hgt[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": NC_nx \t" << NC_nx[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": NC_ny \t" << NC_ny[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": NC_dx \t" << NC_dx[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": NC_dy \t" << NC_dy[it] << std::endl; + amrex::AllPrint() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << " it-" << it << ": NC_epochTime\t" << NC_epochTime[it] << std::endl; + // NC_dateTime is only on the IOProcessor. + amrex::Print() << " DJW init_from_metgrid proc-" << ParallelDescriptor::MyProc() << ": NC_dateTime \t" << NC_dateTime[it] << std::endl; +#endif + } // it + + // Verify that files in nc_init_file[lev] are ordered from earliest to latest. + for (int it = 1; it < ntimes; it++) AMREX_ALWAYS_ASSERT(NC_epochTime[it] > NC_epochTime[it-1]); + + // Start at the earliest time in nc_init_file[lev]. + start_bdy_time = NC_epochTime[0]; + t_new[lev] = start_bdy_time; + t_old[lev] = start_bdy_time - 1.e200; + + // Determine the spacing between met_em files. + bdy_time_interval = NC_epochTime[1]-NC_epochTime[0]; + + // Verify that met_em files have even spacing in time. + for (int it = 1; it < ntimes; it++) { + Real NC_dt = NC_epochTime[it]-NC_epochTime[it-1]; +#ifndef AMREX_USE_GPU + amrex::Print() << " " << nc_init_file[lev][it-1] << " / " << nc_init_file[lev][it] << " are " << NC_dt << " seconds apart" << std::endl; +#endif + if (NC_dt != bdy_time_interval) amrex::Error("Time interval between consecutive met_em files must be consistent."); + } + + // Set up a FAB for mixing ratio and another for potential temperature. + // Necessary because the input data has relative humidity and temperature, not mixing ratio and potential temperature. + // TODO: add alternate pathways for other origin models where different combinations of variables may be present. + Vector mxrat_fab; mxrat_fab.resize(ntimes); + Vector theta_fab; theta_fab.resize(ntimes); + for (int it = 0; it < ntimes; it++) { + Box NC_box_unstag = NC_rhum_fab[it].box(); +#ifdef AMREX_USE_GPU + mxrat_fab[it].resize(NC_box_unstag, 1, The_Pinned_Arena()); + theta_fab[it].resize(NC_box_unstag, 1, The_Pinned_Arena()); +#else + mxrat_fab[it].resize(NC_box_unstag, 1); + theta_fab[it].resize(NC_box_unstag, 1); +#endif } auto& lev_new = vars_new[lev]; @@ -93,108 +145,418 @@ ERF::init_from_metgrid (int lev) AMREX_ALWAYS_ASSERT(solverChoice.use_terrain); - z_phys->setVal(0.); + // Verify that the terrain height (HGT_M) was in each met_em file. + for (int it = 0; it < ntimes; it++) AMREX_ALWAYS_ASSERT(flag_hgt[it] == 1); - for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { + z_phys->setVal(0.0); + + for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { // This defines only the z(i,j,0) values given the FAB filled from the NetCDF input FArrayBox& z_phys_nd_fab = (*z_phys)[mfi]; - init_terrain_from_metgrid(lev, z_phys_nd_fab, NC_hgt_fab); + init_terrain_from_metgrid(z_phys_nd_fab, NC_hgt_fab); } // mf // This defines all the z(i,j,k) values given z(i,j,0) from above. init_terrain_grid(geom[lev], *z_phys, zlevels_stag); + // Copy SST and LANDMASK data into MF and iMF data structures + auto& ba = lev_new[Vars::cons].boxArray(); + auto& dm = lev_new[Vars::cons].DistributionMap(); + auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0; + BoxList bl2d = ba.boxList(); + for (auto& b : bl2d) { + b.setRange(2,0); + } + BoxArray ba2d(std::move(bl2d)); + int i_lo = geom[lev].Domain().smallEnd(0); int i_hi = geom[lev].Domain().bigEnd(0); + int j_lo = geom[lev].Domain().smallEnd(1); int j_hi = geom[lev].Domain().bigEnd(1); + if (flag_sst[0]) { + for (int it = 0; it < ntimes; ++it) { + sst_lev[lev][it] = std::make_unique(ba2d,dm,1,ngv); + for ( MFIter mfi(*(sst_lev[lev][it]), TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + Box gtbx = mfi.growntilebox(); + FArrayBox& dst = (*(sst_lev[lev][it]))[mfi]; + FArrayBox& src = NC_sst_fab[it]; + const Array4< Real>& dst_arr = dst.array(); + const Array4& src_arr = src.const_array(); + amrex::ParallelFor(gtbx, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept + { + int li = amrex::min(amrex::max(i, i_lo), i_hi); + int lj = amrex::min(amrex::max(j, j_lo), j_hi); + dst_arr(i,j,0) = src_arr(li,lj,0); + }); + } + sst_lev[lev][it]->FillBoundary(geom[lev].periodicity()); + } + } else { + for (int it = 0; it < ntimes; ++it) sst_lev[lev][it] = nullptr; + } + if (flag_lmask[0]) { + for (int it = 0; it < ntimes; ++it) { + lmask_lev[lev][it] = std::make_unique(ba2d,dm,1,ngv); + for ( MFIter mfi(*(lmask_lev[lev][it]), TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + Box gtbx = mfi.growntilebox(); + IArrayBox& dst = (*(lmask_lev[lev][it]))[mfi]; + IArrayBox& src = NC_lmask_iab[it]; + const Array4< int>& dst_arr = dst.array(); + const Array4& src_arr = src.const_array(); + amrex::ParallelFor(gtbx, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept + { + int li = amrex::min(amrex::max(i, i_lo), i_hi); + int lj = amrex::min(amrex::max(j, j_lo), j_hi); + dst_arr(i,j,0) = src_arr(li,lj,0); + }); + } + lmask_lev[lev][it]->FillBoundary(geom[lev].periodicity()); + } + } else { + for (int it = 0; it < ntimes; ++it) lmask_lev[lev][it] = nullptr; + } + + for (int it = 0; it < ntimes; it++) { + // Verify that the grid size and resolution from met_em file matches that in geom (from ERF inputs file). + AMREX_ALWAYS_ASSERT(geom[lev].CellSizeArray()[0] == NC_dx[it]); + AMREX_ALWAYS_ASSERT(geom[lev].CellSizeArray()[1] == NC_dy[it]); + // NC_nx-2 because NC_nx is the number of staggered grid points indexed from 1. + AMREX_ALWAYS_ASSERT(geom[lev].Domain().bigEnd(0) == NC_nx[it]-2); + // NC_ny-2 because NC_ny is the number of staggered grid points indexed from 1. + AMREX_ALWAYS_ASSERT(geom[lev].Domain().bigEnd(1) == NC_ny[it]-2); + } // it + // This makes the Jacobian. - make_J (geom[lev],*z_phys, *detJ_cc[lev]); + make_J(geom[lev],*z_phys, *detJ_cc[lev]); // This defines z at w-cell faces. make_zcc(geom[lev],*z_phys,*z_phys_cc[lev]); + // Set up FABs to hold data that will be used to set lateral boundary conditions. +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + int MetGridBdyEnd = MetGridBdyVars::NumTypes; +#else + int MetGridBdyEnd = MetGridBdyVars::NumTypes-1; +#endif + //amrex::Vector > fabs_for_bcs; + amrex::Vector> fabs_for_bcs; + fabs_for_bcs.resize(ntimes); + for (int it(0); it < ntimes; it++) { + fabs_for_bcs[it].resize(MetGridBdyEnd); + + Box gdomain; + Box ldomain; + for (int nvar(0); nvar(0.0); + } + } + + + const Real l_rdOcp = solverChoice.rdOcp; + std::unique_ptr mask_c = OwnerMask(lev_new[Vars::cons], geom[lev].periodicity());//, lev_new[Vars::cons].nGrowVect()); + std::unique_ptr mask_u = OwnerMask(lev_new[Vars::xvel], geom[lev].periodicity());//, lev_new[Vars::xvel].nGrowVect()); + std::unique_ptr mask_v = OwnerMask(lev_new[Vars::yvel], geom[lev].periodicity());//, lev_new[Vars::yvel].nGrowVect()); #ifdef _OPENMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - // Define fabs for holding the initial data + for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + Box tbxc = mfi.tilebox(); + Box tbxu = mfi.tilebox(IntVect(1,0,0)); + Box tbxv = mfi.tilebox(IntVect(0,1,0)); + Box tbxw = mfi.tilebox(IntVect(0,0,1)); + + // Define FABs for hlding some of the initial data FArrayBox &cons_fab = lev_new[Vars::cons][mfi]; FArrayBox &xvel_fab = lev_new[Vars::xvel][mfi]; FArrayBox &yvel_fab = lev_new[Vars::yvel][mfi]; FArrayBox &zvel_fab = lev_new[Vars::zvel][mfi]; - - FArrayBox& z_phys_nd_fab = (*z_phys)[mfi]; - init_state_from_metgrid(lev, cons_fab, xvel_fab, yvel_fab, zvel_fab, + FArrayBox &z_phys_nd_fab = (*z_phys)[mfi]; + + const Array4& mask_c_arr = mask_c->const_array(mfi); + const Array4& mask_u_arr = mask_u->const_array(mfi); + const Array4& mask_v_arr = mask_v->const_array(mfi); + + // Fill state data using origin data (initialization and BC arrays) + // x_vel interpolated from origin levels + // y_vel interpolated from origin levels + // z_vel set to 0.0 + // theta calculate on origin levels then interpolate + // mxrat convert RH -> Q on origin levels then interpolate + init_state_from_metgrid(l_rdOcp, + tbxc, tbxu, tbxv, + cons_fab, xvel_fab, yvel_fab, zvel_fab, z_phys_nd_fab, - NC_hgt_fab, NC_xvel_fab, NC_yvel_fab, NC_temp_fab, - NC_rhum_fab, NC_pres_fab); + NC_hgt_fab, NC_ght_fab, NC_xvel_fab, + NC_yvel_fab, NC_temp_fab, NC_rhum_fab, + NC_pres_fab, theta_fab, mxrat_fab, + fabs_for_bcs, mask_c_arr, mask_u_arr, mask_v_arr); } // mf + #ifdef _OPENMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - // Map scale factors common for "ideal" as well as "real" simulation - for ( MFIter mfi(*mapfac_u[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { + // Use map scale factors directly from the met_em files + for ( MFIter mfi(*mapfac_u[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { // Define fabs for holding the initial data FArrayBox &msfu_fab = (*mapfac_u[lev])[mfi]; FArrayBox &msfv_fab = (*mapfac_v[lev])[mfi]; FArrayBox &msfm_fab = (*mapfac_m[lev])[mfi]; - init_msfs_from_metgrid(lev, msfu_fab, msfv_fab, msfm_fab, + init_msfs_from_metgrid(msfu_fab, msfv_fab, msfm_fab, + flag_msfu[0], flag_msfv[0], flag_msfm[0], NC_MSFU_fab, NC_MSFV_fab, NC_MSFM_fab); } // mf + MultiFab r_hse (base_state[lev], make_alias, 0, 1); // r_0 is first component MultiFab p_hse (base_state[lev], make_alias, 1, 1); // p_0 is second component MultiFab pi_hse(base_state[lev], make_alias, 2, 1); // pi_0 is third component + for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + FArrayBox& p_hse_fab = p_hse[mfi]; + FArrayBox& pi_hse_fab = pi_hse[mfi]; + FArrayBox& r_hse_fab = r_hse[mfi]; + FArrayBox& cons_fab = lev_new[Vars::cons][mfi]; + FArrayBox& z_phys_nd_fab = (*z_phys)[mfi]; - const Real l_rdOcp = solverChoice.rdOcp; + const Array4& mask_c_arr = mask_c->const_array(mfi); + + // Fill base state data using origin data (initialization and BC arrays) + // p_hse calculate dry pressure + // r_hse calculate dry density + // pi_hse calculate Exner term given pressure + const Box valid_bx = mfi.validbox(); + init_base_state_from_metgrid(l_rdOcp, + valid_bx, + flag_psfc, + cons_fab, r_hse_fab, p_hse_fab, pi_hse_fab, + z_phys_nd_fab, NC_ght_fab, NC_psfc_fab, + fabs_for_bcs, mask_c_arr); + } // mf - if (init_type == "real") { - for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - FArrayBox& p_hse_fab = p_hse[mfi]; - FArrayBox& pi_hse_fab = pi_hse[mfi]; - FArrayBox& r_hse_fab = r_hse[mfi]; - const Box& bx = mfi.validbox(); - //init_base_state_from_metgrid(lev, bx, l_rdOcp, p_hse_fab, pi_hse_fab, r_hse_fab, - // NC_ALB_fab, NC_PB_fab); + // NOTE: fabs_for_bcs is defined over the whole domain on each rank. + // However, the operations needed to define the data on the ERF + // grid are done over MultiFab boxes that are local to the rank. + // So when we save the data in fabs_for_bc, only regions owned + // by the rank are populated. Use an allreduce sum to make the + // complete data set; initialized to 0 above. + for (int it(0); it < ntimes; it++) { + for (int nvar(0); nvar& R_bcs_arr = fabs_for_bcs[it][MetGridBdyVars::R].const_array(); + + for (int ivar(MetGridBdyVars::U); ivar < MetGridBdyEnd; ivar++) { + + auto xlo_arr = bdy_data_xlo[it][ivar].array(); + auto xhi_arr = bdy_data_xhi[it][ivar].array(); + auto ylo_arr = bdy_data_ylo[it][ivar].array(); + auto yhi_arr = bdy_data_yhi[it][ivar].array(); + const Array4& fabs_for_bcs_arr = fabs_for_bcs[it][ivar].const_array(); + + if (ivar == MetGridBdyVars::U) { + multiply_rho = false; + xlo_plane = xlo_plane_x_stag; xhi_plane = xhi_plane_x_stag; + ylo_plane = ylo_plane_x_stag; yhi_plane = yhi_plane_x_stag; + } else if (ivar == MetGridBdyVars::V) { + multiply_rho = false; + xlo_plane = xlo_plane_y_stag; xhi_plane = xhi_plane_y_stag; + ylo_plane = ylo_plane_y_stag; yhi_plane = yhi_plane_y_stag; + } else if (ivar == MetGridBdyVars::R) { + multiply_rho = false; + xlo_plane = xlo_plane_no_stag; xhi_plane = xhi_plane_no_stag; + ylo_plane = ylo_plane_no_stag; yhi_plane = yhi_plane_no_stag; + } else if (ivar == MetGridBdyVars::T) { + multiply_rho = false; + xlo_plane = xlo_plane_no_stag; xhi_plane = xhi_plane_no_stag; + ylo_plane = ylo_plane_no_stag; yhi_plane = yhi_plane_no_stag; + } else if (ivar == MetGridBdyVars::QV) { + multiply_rho = true; + xlo_plane = xlo_plane_no_stag; xhi_plane = xhi_plane_no_stag; + ylo_plane = ylo_plane_no_stag; yhi_plane = yhi_plane_no_stag; + } // MetGridBdyVars::QV + + // west boundary + amrex::ParallelFor(xlo_plane, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real Factor = (multiply_rho) ? R_bcs_arr(i,j,k) : 1.0; + xlo_arr(i,j,k,0) = fabs_for_bcs_arr(i,j,k)*Factor; + }); + // xvel at east boundary + amrex::ParallelFor(xhi_plane, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real Factor = (multiply_rho) ? R_bcs_arr(i,j,k) : 1.0; + xhi_arr(i,j,k,0) = fabs_for_bcs_arr(i,j,k)*Factor; + }); + // xvel at south boundary + amrex::ParallelFor(ylo_plane, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real Factor = (multiply_rho) ? R_bcs_arr(i,j,k) : 1.0; + ylo_arr(i,j,k,0) = fabs_for_bcs_arr(i,j,k)*Factor; + }); + // xvel at north boundary + amrex::ParallelFor(yhi_plane, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real Factor = (multiply_rho) ? R_bcs_arr(i,j,k) : 1.0; + yhi_arr(i,j,k,0) = fabs_for_bcs_arr(i,j,k)*Factor; + }); + + } // ivar + } // it +} + /** - * Helper function to initialize terrain nodal z coordinates for a Fab - * given metgrid data. + * Helper function to initialize terrain nodal z coordinates given metgrid data. * - * @param lev Integer specifying the current level * @param z_phys_nd_fab FArrayBox (Fab) holding the nodal z coordinates for terrain data we want to fill * @param NC_hgt_fab Vector of FArrayBox objects holding height data read from NetCDF files for metgrid data */ void -init_terrain_from_metgrid (int lev, FArrayBox& z_phys_nd_fab, +init_terrain_from_metgrid (FArrayBox& z_phys_nd_fab, const Vector& NC_hgt_fab) { - int nboxes = NC_hgt_fab.size(); - - // NOTE NOTE NOTE -- this routine currently only fills the k=0 value - // TODO: we need to fill the rest of the values from the pressure (?) variable... + int ntimes = 1; // Use terrain from the first met_em file. - for (int idx = 0; idx < nboxes; idx++) - { + for (int it = 0; it < ntimes; it++) { #ifndef AMREX_USE_GPU - amrex::Print() << " SIZE OF HGT FAB " << NC_hgt_fab[idx].box() << std::endl; + amrex::Print() << " SIZE OF HGT FAB " << NC_hgt_fab[it].box() << std::endl; amrex::Print() << " SIZE OF ZP FAB " << z_phys_nd_fab.box() << std::endl; #endif // This copies from NC_zphys on z-faces to z_phys_nd on nodes const Array4& z_arr = z_phys_nd_fab.array(); - const Array4& nc_hgt_arr = NC_hgt_fab[idx].const_array(); + const Array4& nc_hgt_arr = NC_hgt_fab[it].const_array(); - const Box z_hgt_box = NC_hgt_fab[idx].box(); + const Box z_hgt_box = NC_hgt_fab[it].box(); int ilo = z_hgt_box.smallEnd()[0]; int ihi = z_hgt_box.bigEnd()[0]; @@ -202,272 +564,467 @@ init_terrain_from_metgrid (int lev, FArrayBox& z_phys_nd_fab, int jhi = z_hgt_box.bigEnd()[1]; Box z_phys_box = z_phys_nd_fab.box(); - Box from_box = surroundingNodes(NC_hgt_fab[idx].box()); from_box.growHi(2,-1); + Box from_box = surroundingNodes(NC_hgt_fab[it].box()); + from_box.growHi(2,-1); + Box bx = z_phys_box & from_box; + Box bxu = bx; bxu.growHi(0,1); + Box bxv = bx; bxv.growHi(1,1); #ifndef AMREX_USE_GPU amrex::Print() << "FROM BOX " << from_box << std::endl; amrex::Print() << "BX " << bx << std::endl; #endif - // - // We must be careful not to read out of bounds of the WPS data - // - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { int ii = std::max(std::min(i,ihi-1),ilo+1); int jj = std::max(std::min(j,jhi-1),jlo+1); z_arr(i,j,k) = 0.25 * ( nc_hgt_arr (ii,jj ,k) + nc_hgt_arr(ii-1,jj ,k) + nc_hgt_arr (ii,jj-1,k) + nc_hgt_arr(ii-1,jj-1,k) ); }); - } // idx + } // it } /** - * Helper function to initialize state and velocity data - * read from metgrid data. + * Helper function to initialize state and velocity data read from metgrid data. * - * @param lev Integer specifying the current level + * @param l_rdOcp Real constant specifying Rhydberg constant ($R_d$) divided by specific heat at constant pressure ($c_p$) * @param state_fab FArrayBox holding the state data to initialize * @param x_vel_fab FArrayBox holding the x-velocity data to initialize * @param y_vel_fab FArrayBox holding the y-velocity data to initialize * @param z_vel_fab FArrayBox holding the z-velocity data to initialize * @param z_phys_nd_fab FArrayBox holding nodal z coordinate data for terrain - * @param NC_hgt_fab Vector of FArrayBox obects holding metgrid data for height + * @param NC_hgt_fab Vector of FArrayBox obects holding metgrid data for terrain height + * @param NC_ght_fab Vector of FArrayBox objects holding metgrid data for height of cell centers * @param NC_xvel_fab Vector of FArrayBox obects holding metgrid data for x-velocity * @param NC_yvel_fab Vector of FArrayBox obects holding metgrid data for y-velocity * @param NC_zvel_fab Vector of FArrayBox obects holding metgrid data for z-velocity - * @param NC_rho_fab Vector of FArrayBox obects holding metgrid data for density - * @param NC_rhotheta_fab Vector of FArrayBox obects holding metgrid data for (density * potential temperature) + * @param NC_temp_fab Vector of FArrayBox obects holding metgrid data for temperature + * @param NC_rhum_fab Vector of FArrayBox obects holding metgrid data for relative humidity + * @param NC_pres_fab Vector of FArrayBox obects holding metgrid data for pressure + * @param theta_fab Vector of FArrayBox obects holding potential temperature calculated from temperature and pressure + * @param mxrat_fab Vector of FArrayBox obects holding vapor mixing ratio calculated from relative humidity + * @param fabs_for_bcs Vector of Vector of FArrayBox objects holding MetGridBdyVars at each met_em time. */ void -init_state_from_metgrid (int lev, FArrayBox& state_fab, - FArrayBox& x_vel_fab, FArrayBox& y_vel_fab, - FArrayBox& z_vel_fab, FArrayBox& z_phys_nd_fab, +init_state_from_metgrid (const Real l_rdOcp, + Box& tbxc, + Box& tbxu, + Box& tbxv, + FArrayBox& state_fab, + FArrayBox& x_vel_fab, + FArrayBox& y_vel_fab, + FArrayBox& z_vel_fab, + FArrayBox& z_phys_nd_fab, const Vector& NC_hgt_fab, + const Vector& NC_ght_fab, const Vector& NC_xvel_fab, const Vector& NC_yvel_fab, - const Vector& NC_zvel_fab, - const Vector& NC_rho_fab, - const Vector& NC_rhotheta_fab) + const Vector& NC_temp_fab, + const Vector& NC_rhum_fab, + const Vector& NC_pres_fab, + Vector& theta_fab, + Vector& mxrat_fab, + amrex::Vector>& fabs_for_bcs, + const amrex::Array4& mask_c_arr, + const amrex::Array4& mask_u_arr, + const amrex::Array4& mask_v_arr) { - int nboxes = NC_hgt_fab.size(); - -#ifndef AMREX_USE_GPU - amrex::Print() << " U FROM NC " << NC_xvel_fab[0].box() << std::endl; - amrex::Print() << " U INTO FAB " << x_vel_fab.box() << std::endl; - exit(0); -#endif - for (int idx = 0; idx < nboxes; idx++) + int ntimes = NC_hgt_fab.size(); + for (int it = 0; it < ntimes; it++) { // ******************************************************** // U // ******************************************************** { - Box bx2d = NC_xvel_fab[idx].box() & x_vel_fab.box(); + Box bx2d = NC_xvel_fab[it].box() & tbxu; bx2d.setRange(2,0); - auto const orig_data = NC_xvel_fab[idx].const_array(); - auto const orig_z = NC_hgt_fab[idx].const_array(); - + auto const orig_data = NC_xvel_fab[it].const_array(); + auto const orig_z = NC_ght_fab[it].const_array(); auto new_data = x_vel_fab.array(); + auto bc_data = fabs_for_bcs[it][MetGridBdyVars::U].array(); auto const new_z = z_phys_nd_fab.const_array(); - ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) + int kmax = amrex::ubound(tbxu).z; + + ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept { - interpolate_column(i,j,0,0,orig_z,orig_data,new_z,new_data); + for (int k = 0; k<=kmax; k++) { + Real Interp_Val = interpolate_column_metgrid(i,j,k,'X',0,orig_z,orig_data,new_z); + if (mask_u_arr(i,j,k)) bc_data(i,j,k,0) = Interp_Val; + if (it==0) new_data(i,j,k,0) = Interp_Val; + } }); } + // ******************************************************** // V // ******************************************************** { - Box bx2d = NC_yvel_fab[idx].box() & y_vel_fab.box(); + Box bx2d = NC_yvel_fab[it].box() & tbxv; bx2d.setRange(2,0); - auto const orig_data = NC_yvel_fab[idx].const_array(); - auto const orig_z = NC_hgt_fab[idx].const_array(); - + auto const orig_data = NC_yvel_fab[it].const_array(); + auto const orig_z = NC_ght_fab[it].const_array(); auto new_data = y_vel_fab.array(); + auto bc_data = fabs_for_bcs[it][MetGridBdyVars::V].array(); auto const new_z = z_phys_nd_fab.const_array(); - ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) + int kmax = amrex::ubound(tbxv).z; + + ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept { - interpolate_column(i,j,0,0,orig_z,orig_data,new_z,new_data); + for (int k = 0; k<=kmax; k++) { + Real Interp_Val = interpolate_column_metgrid(i,j,k,'Y',0,orig_z,orig_data,new_z); + if (mask_v_arr(i,j,k)) bc_data(i,j,k,0) = Interp_Val; + if (it==0) new_data(i,j,k,0) = Interp_Val; + } }); } + // ******************************************************** // W // ******************************************************** - z_vel_fab.template setVal(0.); + if (it == 0) { // update at initialization + z_vel_fab.template setVal(0.0); + } + + // ******************************************************** + // Initialize all state_fab variables to zero + // ******************************************************** + if (it == 0) { // update at initialization + state_fab.template setVal(0.0); + } + // ******************************************************** - // rho + // theta // ******************************************************** + { // calculate potential temperature. + Box bx = NC_rhum_fab[it].box() & tbxc; + auto const temp = NC_temp_fab[it].const_array(); + auto const pres = NC_pres_fab[it].const_array(); + auto theta = theta_fab[it].array(); + + ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + theta(i,j,k) = getThgivenPandT(temp(i,j,k),pres(i,j,k),l_rdOcp); + //theta(i,j,k) = 300.0; // TODO: Remove when not needed. Force an isothermal atmosphere for debugging. + }); + } + + // vertical interpolation of potential temperature. { - Box bx2d = NC_rho_fab[idx].box() & state_fab.box(); + Box bx2d = NC_temp_fab[it].box() & tbxc; bx2d.setRange(2,0); - - auto const orig_data = NC_rho_fab[idx].const_array(); - auto const orig_z = NC_hgt_fab[idx].const_array(); - auto new_data = state_fab.array(); + auto const orig_data = theta_fab[it].const_array(); + auto const orig_z = NC_ght_fab[it].const_array(); + auto new_data = state_fab.array(); + auto bc_data = fabs_for_bcs[it][MetGridBdyVars::T].array(); auto const new_z = z_phys_nd_fab.const_array(); - ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) + int kmax = amrex::ubound(tbxc).z; + + ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept { - interpolate_column(i,j,0,Rho_comp,orig_z,orig_data,new_z,new_data); + for (int k = 0; k<=kmax; k++) { + Real Interp_Val = interpolate_column_metgrid(i,j,k,'M',0,orig_z,orig_data,new_z); + if (mask_c_arr(i,j,k)) bc_data(i,j,k,0) = Interp_Val; + if (it==0) new_data(i,j,k,RhoTheta_comp) = Interp_Val; + } }); } +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) // ******************************************************** - // rho_theta + // specific humidity / relative humidity / mixing ratio // ******************************************************** + // TODO: we will need to check what input data we have for moisture + // and then, if necessary, compute mixing ratio. For now, we will + // focus on the case where we have relative humidity. Alternate cases + // could be specific humidity or a mixing ratio. + // + { // calculate vapor mixing ratio from relative humidity. + Box bx = NC_temp_fab[it].box() & tbxc; + auto const rhum = NC_rhum_fab[it].const_array(); + auto const temp = NC_temp_fab[it].const_array(); + auto const pres = NC_pres_fab[it].const_array(); + auto mxrat = mxrat_fab[it].array(); + + ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + rh_to_mxrat(i,j,k,rhum,temp,pres,mxrat); + }); + } + + // vertical interpolation of vapor mixing ratio. { - Box bx2d = NC_rhotheta_fab[idx].box() & state_fab.box(); + Box bx2d = NC_temp_fab[it].box() & tbxc; bx2d.setRange(2,0); - - auto const orig_data = NC_rhotheta_fab[idx].const_array(); - auto const orig_z = NC_hgt_fab[idx].const_array(); + auto const orig_data = mxrat_fab[it].const_array(); + auto const orig_z = NC_ght_fab[it].const_array(); auto new_data = state_fab.array(); + auto bc_data = fabs_for_bcs[it][MetGridBdyVars::QV].array(); auto const new_z = z_phys_nd_fab.const_array(); - ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) + int kmax = amrex::ubound(tbxc).z; + +#if defined(ERF_USE_MOISTURE) + int state_indx = RhoQt_comp; +#elif defined(ERF_USE_WARM_NO_PRECIP) + int state_indx = RhoQv_comp; +#endif + ParallelFor(bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept { - interpolate_column(i,j,0,RhoTheta_comp,orig_z,orig_data,new_z,new_data); + for (int k = 0; k<=kmax; k++) { + Real Interp_Val = interpolate_column_metgrid(i,j,k,'M',0,orig_z,orig_data,new_z); + if (mask_c_arr(i,j,k)) bc_data(i,j,k,0) = Interp_Val; + if (it==0) new_data(i,j,k,state_indx) = Interp_Val; + } }); } - } // idx -} - -/** - * Helper function to initialize map factors from metgrid data - * - * @param lev Integer specifying current level - * @param msfu_fab FArrayBox specifying x-velocity map factors - * @param msfv_fab FArrayBox specifying y-velocity map factors - * @param msfm_fab FArrayBox specifying z-velocity map factors - * @param NC_MSFU_fab Vector of FArrayBox objects holding metgrid data for x-velocity map factors - * @param NC_MSFV_fab Vector of FArrayBox objects holding metgrid data for y-velocity map factors - * @param NC_MSFM_fab Vector of FArrayBox objects holding metgrid data for z-velocity map factors - */ -void -init_msfs_from_metgrid (int lev, FArrayBox& msfu_fab, - FArrayBox& msfv_fab, FArrayBox& msfm_fab, - const Vector& NC_MSFU_fab, - const Vector& NC_MSFV_fab, - const Vector& NC_MSFM_fab) -{ - int nboxes = NC_MSFU_fab.size(); - - for (int idx = 0; idx < nboxes; idx++) - { - // - // FArrayBox to FArrayBox copy does "copy on intersection" - // This only works here because we have broadcast the FArrayBox of data from the netcdf file to all ranks - // - // This copies mapfac_u - msfu_fab.template copy(NC_MSFU_fab[idx]); +#endif - // This copies mapfac_v - msfv_fab.template copy(NC_MSFV_fab[idx]); + // TODO: TEMPORARY CODE TO RUN QUIESCENT, REMOVE WHEN NOT NEEDED. +// if (it == 0) { +// x_vel_fab.template setVal(0.0); // TODO: temporary code to initialize with quiescent atmosphere. +// y_vel_fab.template setVal(0.0); // TODO: temporary code to initialize with quiescent atmosphere. +// } +// fabs_for_bcs[it][MetGridBdyVars::U].template setVal(0.0); // TODO: temporary code to force with quiescent atmosphere. +// fabs_for_bcs[it][MetGridBdyVars::V].template setVal(0.0); // TODO: temporary code to force with quiescent atmosphere. - // This copies mapfac_m - msfm_fab.template copy(NC_MSFM_fab[idx]); - } // idx + } // it } + /** * Helper function for initializing hydrostatic base state data from metgrid data * - * @param lev Integer specifying the current level - * @param valid_bx Box specifying the index space we are initializing * @param l_rdOcp Real constant specifying Rhydberg constant ($R_d$) divided by specific heat at constant pressure ($c_p$) - * @param p_hse FArrayBox holding the hydrostatic base state pressure we are initializing - * @param pi_hse FArrayBox holding the hydrostatic base Exner pressure we are initializing - * @param r_hse FArrayBox holding the hydrostatic base state density we are initializing - * @param NC_ALB_fab Vector of FArrayBox objects holding metgrid data specifying 1/density - * @param NC_PB_fab Vector of FArrayBox objects holding metgrid data specifying pressure + * @param valid_bx Box specifying the index space we are to initialize + * @param flag_psfc Vector of Integer 1 if surface pressure is in metgrid data, 0 otherwise + * @param state_fab FArrayBox holding the state data to initialize + * @param r_hse_fab FArrayBox holding the hydrostatic base state density we are initializing + * @param p_hse_fab FArrayBox holding the hydrostatic base state pressure we are initializing + * @param pi_hse_fab FArrayBox holding the hydrostatic base Exner pressure we are initializing + * @param z_phys_nd_fab FArrayBox holding nodal z coordinate data for terrain + * @param NC_ght_fab Vector of FArrayBox objects holding metgrid data for height of cell centers + * @param NC_psfc_fab Vector of FArrayBox objects holding metgrid data for surface pressure + * @param fabs_for_bcs Vector of Vector of FArrayBox objects holding MetGridBdyVars at each met_em time. */ void -init_base_state_from_metgrid (int lev, const Box& valid_bx, const Real l_rdOcp, - FArrayBox& p_hse, FArrayBox& pi_hse, FArrayBox& r_hse, - const Vector& NC_ALB_fab, - const Vector& NC_PB_fab) +init_base_state_from_metgrid (const Real l_rdOcp, + const Box& valid_bx, + const Vector& flag_psfc, + FArrayBox& state_fab, + FArrayBox& r_hse_fab, + FArrayBox& p_hse_fab, + FArrayBox& pi_hse_fab, + FArrayBox& z_phys_nd_fab, + const Vector& NC_ght_fab, + const Vector& NC_psfc_fab, + Vector>& fabs_for_bcs, + const amrex::Array4& mask_c_arr) { - int nboxes = NC_ALB_fab.size(); +#if defined(ERF_USE_MOISTURE) + int RhoQ_comp = RhoQt_comp; +#elif defined(ERF_USE_WARM_NO_PRECIP) + int RhoQ_comp = RhoQv_comp; +#endif + int kmax = amrex::ubound(valid_bx).z; + + // Device vectors for columnwise operations + Gpu::DeviceVector z_vec_d(kmax+2,0); Real* z_vec = z_vec_d.data(); + Gpu::DeviceVector Thetad_vec_d(kmax+1,0); Real* Thetad_vec = Thetad_vec_d.data(); + Gpu::DeviceVector Thetam_vec_d(kmax+1,0); Real* Thetam_vec = Thetam_vec_d.data(); + Gpu::DeviceVector Rhod_vec_d(kmax+1,0); Real* Rhod_vec = Rhod_vec_d.data(); + Gpu::DeviceVector Rhom_vec_d(kmax+1,0); Real* Rhom_vec = Rhom_vec_d.data(); + Gpu::DeviceVector Pd_vec_d(kmax+1,0); Real* Pd_vec = Pd_vec_d.data(); + Gpu::DeviceVector Pm_vec_d(kmax+1,0); Real* Pm_vec = Pm_vec_d.data(); +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + Gpu::DeviceVector Q_vec_d(kmax+1,0); Real* Q_vec = Q_vec_d.data(); +#endif - for (int idx = 0; idx < nboxes; idx++) - { - // - // FArrayBox to FArrayBox copy does "copy on intersection" - // This only works here because we have broadcast the FArrayBox of data from the netcdf file to all ranks - // - const Array4& p_hse_arr = p_hse.array(); - const Array4& pi_hse_arr = pi_hse.array(); - const Array4& r_hse_arr = r_hse.array(); - const Array4& alpha_arr = NC_ALB_fab[idx].const_array(); - const Array4& nc_pb_arr = NC_PB_fab[idx].const_array(); - - amrex::ParallelFor(valid_bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - p_hse_arr(i,j,k) = nc_pb_arr(i,j,k); + // Device vectors for psfc flags + Gpu::DeviceVectorflag_psfc_d(flag_psfc.size()); + Gpu::copy(Gpu::hostToDevice, flag_psfc.begin(), flag_psfc.end(), flag_psfc_d.begin()); + int* flag_psfc_vec = flag_psfc_d.data(); + + { // set pressure and density at initialization. + const Array4& r_hse_arr = r_hse_fab.array(); + const Array4& p_hse_arr = p_hse_fab.array(); + const Array4& pi_hse_arr = pi_hse_fab.array(); + + // ******************************************************** + // calculate dry density and dry pressure + // ******************************************************** + // calculate density and dry pressure on the new grid. + Box valid_bx2d = valid_bx; + valid_bx2d.setRange(2,0); + auto const orig_psfc = NC_psfc_fab[0].const_array(); + auto new_data = state_fab.array(); + auto const new_z = z_phys_nd_fab.const_array(); + + amrex::ParallelFor(valid_bx2d, [=] AMREX_GPU_DEVICE (int i, int j, int) noexcept + { + for (int k=0; k<=kmax; k++) { + z_vec[k] = new_z(i,j,k); + Thetad_vec[k] = new_data(i,j,k,RhoTheta_comp); +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + Q_vec[k] = new_data(i,j,k,RhoQ_comp); +#endif + } + z_vec[kmax+1] = new_z(i,j,kmax+1); + + calc_rho_p(kmax,flag_psfc_vec[0],orig_psfc(i,j,0),Thetad_vec,Thetam_vec, +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + Q_vec, +#endif + z_vec,Rhod_vec,Rhom_vec,Pd_vec,Pm_vec); + + for (int k=0; k<=kmax; k++) { + p_hse_arr(i,j,k) = Pd_vec[k]; + r_hse_arr(i,j,k) = Rhod_vec[k]; + } + }); + + amrex::ParallelFor(valid_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + new_data(i,j,k,Rho_comp) = r_hse_arr(i,j,k); + new_data(i,j,k,RhoScalar_comp) = 0.0; + // RhoTheta and RhoQt or RhoQv currently hold Theta and Qt or Qv. Multiply by Rho. + Real RhoTheta = r_hse_arr(i,j,k)*new_data(i,j,k,RhoTheta_comp); + new_data(i,j,k,RhoTheta_comp) = RhoTheta; +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + Real RhoQ = r_hse_arr(i,j,k)*new_data(i,j,k,RhoQ_comp); + new_data(i,j,k,RhoQ_comp) = RhoQ; +#endif pi_hse_arr(i,j,k) = getExnergivenP(p_hse_arr(i,j,k), l_rdOcp); - r_hse_arr(i,j,k) = 1.0 / alpha_arr(i,j,k); + //pi_hse_arr(i,j,k) = getExnergivenRTh(RhoTheta, l_rdOcp); + }); + } + int ntimes = NC_psfc_fab.size(); + for (int it=0; it& orig_z, const Array4& orig_data, - const Array4& new_z, const Array4& new_data) +init_msfs_from_metgrid (FArrayBox& msfu_fab, + FArrayBox& msfv_fab, + FArrayBox& msfm_fab, + const int& flag_msfu, + const int& flag_msfv, + const int& flag_msfm, + const Vector& NC_MSFU_fab, + const Vector& NC_MSFV_fab, + const Vector& NC_MSFM_fab) { - // CAVEAT: we only consider interpolation here - if we go past end of array this won't work for now +// int ntimes = NC_MSFU_fab.size(); + int ntimes = 1; + for (int it = 0; it < ntimes; it++) { + // + // FArrayBox to FArrayBox copy does "copy on intersection" + // This only works here because we have broadcast the FArrayBox of data from the netcdf file to all ranks + // - int kmax = amrex::ubound(Box(new_data)).z; - int kmax_old = amrex::ubound(Box(orig_data)).z; + // This copies or sets mapfac_m + if (flag_msfm == 1) { + msfm_fab.template copy(NC_MSFM_fab[it]); + } else { #ifndef AMREX_USE_GPU - amrex::Print() << "KMAX IN INTERP " << kmax << std::endl; - amrex::Print() << "KMAX_OLD IN INTERP " << kmax_old << std::endl; + amrex::Print() << " MAPFAC_M not present in met_em files. Setting to 1.0" << std::endl; #endif - int klast = 0; - for (int k = 0; k < kmax; k++) { - - Real z = new_z(i,j,k); + msfm_fab.template setVal(1.0); + } - bool kfound = false; + // This copies or sets mapfac_u + if (flag_msfu == 1) { + msfu_fab.template copy(NC_MSFU_fab[it]); + } else { +#ifndef AMREX_USE_GPU + amrex::Print() << " MAPFAC_U not present in met_em files. Setting to 1.0" << std::endl; +#endif + msfu_fab.template setVal(1.0); + } + // This copies or sets mapfac_v + if (flag_msfv == 1) { + msfv_fab.template copy(NC_MSFV_fab[it]); + } else { #ifndef AMREX_USE_GPU - amrex::Print() << "AT THIS K WE ARE USING KLAST " << k << " " << klast << std::endl; + amrex::Print() << " MAPFAC_V not present in met_em files. Setting to 1.0" << std::endl; #endif - for (int kk = klast; kk < kmax_old; kk++) { - if (z >= orig_z(i,j,k)) { - Real y0 = orig_data(i,j,kk ,src_comp); - Real y1 = orig_data(i,j,kk+1,src_comp); - Real z0 = orig_z(i,j,kk ); - Real z1 = orig_z(i,j,kk+1); - new_data(i,j,k,dest_comp) = y0 + (y1 - y0)*(z - z0) / (z1 - z0); - klast = kk; - return; - } + msfv_fab.template setVal(1.0); } - if (!kfound) amrex::Error("Wasnt able to interpolate"); - } -} + } // it +} #endif // ERF_USE_NETCDF diff --git a/Source/Initialization/ERF_init_from_wrfinput.cpp b/Source/Initialization/ERF_init_from_wrfinput.cpp index 0bfec76ea..d29fe922d 100644 --- a/Source/Initialization/ERF_init_from_wrfinput.cpp +++ b/Source/Initialization/ERF_init_from_wrfinput.cpp @@ -254,6 +254,10 @@ ERF::init_from_wrfinput (int lev) NC_C1H_fab[0], NC_C2H_fab[0], NC_RDNW_fab[0], NC_xvel_fab[0],NC_yvel_fab[0],NC_rho_fab[0],NC_rhoth_fab[0]); } + + // Start at the earliest time (read_from_wrfbdy) + t_new[lev] = start_bdy_time; + t_old[lev] = start_bdy_time - 1.e200; } /** diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 9115a0dac..d3bb99d4b 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -8,6 +8,7 @@ CEXE_sources += ERF_init_bcs.cpp CEXE_sources += ERF_init1d.cpp ifeq ($(USE_NETCDF),TRUE) +CEXE_headers += Metgrid_utils.H CEXE_sources += ERF_init_from_wrfinput.cpp CEXE_sources += ERF_init_from_metgrid.cpp endif diff --git a/Source/Initialization/Metgrid_utils.H b/Source/Initialization/Metgrid_utils.H new file mode 100644 index 000000000..af06ecd5d --- /dev/null +++ b/Source/Initialization/Metgrid_utils.H @@ -0,0 +1,347 @@ +#ifndef _METGRIDUTIL_H_ +#define _METGRIDUTIL_H_ + +#include +#include +#include +#include +#include + +void +read_from_metgrid (int lev, + const amrex::Box& domain, + const std::string& fname, + std::string& NC_dateTime, + amrex::Real& NC_epochTime, + int& flag_psfc, + int& flag_msfu, + int& flag_msfv, + int& flag_msfm, + int& flag_hgt, + int& flag_sst, + int& flag_lmask, + int& NC_nx, + int& NC_ny, + amrex::Real& NC_dx, + amrex::Real& NC_dy, + amrex::FArrayBox& NC_xvel_fab, + amrex::FArrayBox& NC_yvel_fab, + amrex::FArrayBox& NC_temp_fab, + amrex::FArrayBox& NC_rhum_fab, + amrex::FArrayBox& NC_pres_fab, + amrex::FArrayBox& NC_ght_fab, + amrex::FArrayBox& NC_hgt_fab, + amrex::FArrayBox& NC_psfc_fab, + amrex::FArrayBox& NC_msfu_fab, + amrex::FArrayBox& NC_msfv_fab, + amrex::FArrayBox& NC_msfm_fab, + amrex::FArrayBox& NC_sst_fab, + amrex::IArrayBox& NC_lmask_iab); + + + +void +init_terrain_from_metgrid (amrex::FArrayBox& z_phys_nd_fab, + const amrex::Vector& NC_hgt_fab); + +void +init_state_from_metgrid (const amrex::Real l_rdOcp, + amrex::Box& tbxc, + amrex::Box& tbxu, + amrex::Box& tbxv, + amrex::FArrayBox& state_fab, + amrex::FArrayBox& x_vel_fab, + amrex::FArrayBox& y_vel_fab, + amrex::FArrayBox& z_vel_fab, + amrex::FArrayBox& z_phys_nd_fab, + const amrex::Vector& NC_hgt_fab, + const amrex::Vector& NC_ght_fab, + const amrex::Vector& NC_xvel_fab, + const amrex::Vector& NC_yvel_fab, + const amrex::Vector& NC_zvel_fab, + const amrex::Vector& NC_temp_fab, + const amrex::Vector& NC_rhum_fab, + amrex::Vector& theta_fab, + amrex::Vector& mxrat_fab, + amrex::Vector>& fabs_for_bcs, + const amrex::Array4& mask_c_arr, + const amrex::Array4& mask_u_arr, + const amrex::Array4& mask_v_arr); + +void +init_msfs_from_metgrid (amrex::FArrayBox& msfu_fab, + amrex::FArrayBox& msfv_fab, + amrex::FArrayBox& msfm_fab, + const int& flag_msfu, + const int& flag_msfv, + const int& flag_msfm, + const amrex::Vector& NC_MSFU_fab, + const amrex::Vector& NC_MSFV_fab, + const amrex::Vector& NC_MSFM_fab); + +void +init_base_state_from_metgrid (const amrex::Real l_rdOcp, + const amrex::Box& valid_bx, + const amrex::Vector& flag_psfc, + amrex::FArrayBox& state, + amrex::FArrayBox& r_hse_fab, + amrex::FArrayBox& p_hse_fab, + amrex::FArrayBox& pi_hse_fab, + amrex::FArrayBox& z_phys_nd_fab, + const amrex::Vector& NC_ght_fab, + const amrex::Vector& NC_psfc_fab, + amrex::Vector>& fabs_for_bcs, + const amrex::Array4& mask_c_arr); + +AMREX_FORCE_INLINE +AMREX_GPU_DEVICE +void +calc_rho_p (const int& kmax, + const int& flag_psfc, + const amrex::Real& psfc, + amrex::Real* Thetad_vec, + amrex::Real* Thetam_vec, +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + amrex::Real* Q_vec, +#endif + amrex::Real* z_vec, + amrex::Real* Rhod_vec, + amrex::Real* Rhom_vec, + amrex::Real* Pd_vec, + amrex::Real* Pm_vec) +{ + const int maxiter = 10; + + // Calculate or use moist pressure at the surface. + if (flag_psfc == 1) { + Pm_vec[0] = psfc; + } else { + amrex::Real t_0 = 290.0; // WRF's model_config_rec%base_temp + amrex::Real a = 50.0; // WRF's model_config_rec%base_lapse + Pm_vec[0] = p_0*exp(-t_0/a+std::pow((std::pow(t_0/a, 2)-2.0*CONST_GRAV*z_vec[0]/(a*R_d)), 0.5)); + } + + // calculate virtual potential temperature at the surface. + { +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + amrex::Real qvf = 1.0+(R_v/R_d+1.0)*Q_vec[0]; +#else + amrex::Real qvf = 1.0; +#endif + Thetam_vec[0] = Thetad_vec[0]*qvf; + } + + // calculate moist density at the surface. + Rhom_vec[0] = 1.0/(R_d/p_0*Thetam_vec[0]*std::pow(Pm_vec[0]/p_0, -iGamma)); + + // integrate from the surface to the top boundary. + for (int k=1; k<=kmax; k++) { + amrex::Real dz = z_vec[k]-z_vec[k-1]; +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) + amrex::Real qvf = 1.0+(R_v/R_d+1.0)*Q_vec[k]; +#else + amrex::Real qvf = 1.0; +#endif + Thetam_vec[k] = Thetad_vec[k]*qvf; + Rhom_vec[k] = Rhom_vec[k-1]; // an initial guess. + for (int it=0; it=0; k--) { + amrex::Real dz = z_vec[k+1]-z_vec[k]; + Rhod_vec[k] = Rhod_vec[k+1]; // an initial guess. + for (int it=0; it < maxiter; it++) { + Pd_vec[k] = Pd_vec[k+1]+0.5*dz*(Rhod_vec[k]+Rhod_vec[k+1])*CONST_GRAV; + if (Pd_vec[k] < 0.0) Pd_vec[k] = 0.0; + Rhod_vec[k] = 1.0/(R_d/p_0*Thetam_vec[k]*std::pow(Pd_vec[k]/p_0, -iGamma)); + } // it + } // k +} + +AMREX_FORCE_INLINE +AMREX_GPU_DEVICE +amrex::Real +interpolate_column_metgrid (const int& i, + const int& j, + const int& k, + char stag, + int src_comp, + const amrex::Array4& orig_z, + const amrex::Array4& orig_data, + const amrex::Array4& new_z) +{ + // This subroutine is a bit ham-handed and can be cleaned up later. + int imax_orig = amrex::ubound(amrex::Box(orig_data)).x; + int jmax_orig = amrex::ubound(amrex::Box(orig_data)).y; + int kmax_orig = amrex::ubound(amrex::Box(orig_data)).z; + + amrex::Real z; + if (stag == 'X') { + z = 0.25*(new_z(i,j,k)+new_z(i,j+1,k)+new_z(i,j,k+1)+new_z(i,j+1,k+1)); + } + else if (stag == 'Y') { + z = 0.25*(new_z(i,j,k)+new_z(i+1,j,k)+new_z(i,j,k+1)+new_z(i+1,j,k+1)); + } + else if (stag == 'M') { + z = 0.125*(new_z(i,j,k )+new_z(i,j+1,k )+new_z(i+1,j,k )+new_z(i+1,j+1,k )+ + new_z(i,j,k+1)+new_z(i,j+1,k+1)+new_z(i+1,j,k+1)+new_z(i+1,j+1,k+1)); + } + + amrex::Real z0, z1; + int klow = -1; + int khi0 = -1; + amrex::Real dzlow = 10000.0; + amrex::Real dzhi0 = -10000.0; + for (int kk = 0; kk < kmax_orig; kk++) { + amrex::Real orig_z_stag = 0.0; + if (stag == 'M') { + orig_z_stag = orig_z(i,j,kk); + } + if (stag == 'X') { + if (i == 0) { + orig_z_stag = orig_z(i,j,kk); + } + else if (i == imax_orig) { + orig_z_stag = orig_z(imax_orig-1,j,kk); + } + else { + orig_z_stag = 0.5*(orig_z(i,j,kk)+orig_z(i-1,j,kk)); + } + } + else if (stag == 'Y') { + if (j == 0) { + orig_z_stag = orig_z(i,j,kk); + } + else if (j == jmax_orig) { + orig_z_stag = orig_z(i,jmax_orig-1,kk); + } + else { + orig_z_stag = 0.5*(orig_z(i,j,kk)+orig_z(i,j-1,kk)); + } + } + amrex::Real dz = z - orig_z_stag; + if ((dz < 0.0) && (dz > dzhi0)) { + dzhi0 = dz; + khi0 = kk; + z1 = orig_z_stag; + } + if ((dz >= 0.0) && (dz < dzlow)) { + dzlow = dz; + klow = kk; + z0 = orig_z_stag; + } + } // kk + + if (klow == -1) { + // extrapolate + int khi1 = -1; + amrex::Real dzhi1 = -10000.0; + for (int kk = 0; kk < kmax_orig; kk++) { + amrex::Real orig_z_stag = 0.0; + if (stag == 'M') { + orig_z_stag = orig_z(i,j,kk); + } + else if (stag == 'X') { + if (i == 0) { + orig_z_stag = orig_z(i,j,kk); + } + else if (i == imax_orig) { + orig_z_stag = orig_z(imax_orig-1,j,kk); + } + else { + orig_z_stag = 0.5*(orig_z(i,j,kk)+orig_z(i-1,j,kk)); + } + } + else if (stag == 'Y') { + if (j == 0) { + orig_z_stag = orig_z(i,j,kk); + } + else if (j == jmax_orig) { + orig_z_stag = orig_z(i,jmax_orig-1,kk); + } + else { + orig_z_stag = 0.5*(orig_z(i,j,kk)+orig_z(i,j-1,kk)); + } + } + amrex::Real dz = z - orig_z_stag; + if ((dz < 0.0) && (dz > dzhi1) && (kk != khi0)) { + dzhi1 = dz; + khi1 = kk; + z1 = orig_z_stag; + } + } + amrex::Real y0 = orig_data(i,j,khi0,src_comp); + amrex::Real y1 = orig_data(i,j,khi1,src_comp); + return ( y0-(y1-y0)/(z1-z0)*(z0-z) ); + } else { + // interpolate + amrex::Real y0 = orig_data(i,j,klow,src_comp); + amrex::Real y1 = orig_data(i,j,khi0,src_comp); + return ( y0+(y1-y0)/(z1-z0)*(z-z0) ); + + } +} + +AMREX_FORCE_INLINE +AMREX_GPU_DEVICE +void +rh_to_mxrat (int i, + int j, + int k, + const amrex::Array4& rhum, + const amrex::Array4& temp, + const amrex::Array4& pres, + const amrex::Array4& mxrat) +{ + amrex::Real qv_max_p_safe = 10000.0; // WRF default value + amrex::Real qv_max_flag = 1.0e-5; // WRF default value + amrex::Real qv_max_value = 3.0e-6; // WRF default value + amrex::Real qv_min_p_safe = 110000.0; // WRF default value + amrex::Real qv_min_flag = 1.0e-6; // WRF default value + amrex::Real qv_min_value = 1.0e-6; // WRF default value + amrex::Real eps = 0.622; + amrex::Real svp1 = 0.6112; + amrex::Real svp2 = 17.67; + amrex::Real svp3 = 29.65; + amrex::Real svpt0 = 273.15; + // WRF's method when model_config_rec%rh2qv_wrt_liquid=.true. (default behavior) + if (temp(i,j,k) != 0.0) { + amrex::Real es=0.01*rhum(i,j,k)*svp1*10.0*exp(svp2*(temp(i,j,k)-svpt0)/(temp(i,j,k)-svp3)); + if (es >= pres(i,j,k)/100.0) { + // vapor pressure exceeds total pressure + mxrat(i,j,k) = std::pow(10.0, -6); + } + else { + mxrat(i,j,k) = amrex::max(eps*es/(pres(i,j,k)/100.0-es), 1.0e-6); + } + } + else { + // I don't know why there's a fringe case handled in WRF where T is absolute zero... + // Let's just deal with it here in case we also end up needing it. + mxrat(i,j,k) = 1.0e-6; + } + // See the below comment from WRF dyn_em/module_initialize_real.F rh_to_mxrat1. + // For pressures above a defined level, reasonable Qv values should be + // a certain value or smaller. If they are larger than this, the input data + // probably had "missing" RH, and we filled in some values. This is an + // attempt to catch those. Also, set the minimum value for the entire + // domain that is above the selected pressure level. + if (pres(i,j,k) < qv_max_p_safe) { + if (mxrat(i,j,k) > qv_max_flag) { + mxrat(i,j,k) = qv_max_value; + } + } + if (pres(i,j,k) < qv_min_p_safe) { + if (mxrat(i,j,k) < qv_min_flag) { + mxrat(i,j,k) = qv_min_value; + } + } +} +#endif diff --git a/Source/Radiation/Aero_rad_props.H b/Source/Radiation/Aero_rad_props.H index b92df6049..de440576c 100644 --- a/Source/Radiation/Aero_rad_props.H +++ b/Source/Radiation/Aero_rad_props.H @@ -4,7 +4,7 @@ #ifndef ERF_AER_RAD_PROPS_H_ #define ERF_AER_RAD_PROPS_H_ -#include "ERF.H" +#include #include "Mam4_aero.H" #include "YAKL_netcdf.h" #include "rrtmgp_const.h" diff --git a/Source/Radiation/Aero_rad_props.cpp b/Source/Radiation/Aero_rad_props.cpp index 02381dd7b..b01c94b9c 100644 --- a/Source/Radiation/Aero_rad_props.cpp +++ b/Source/Radiation/Aero_rad_props.cpp @@ -3,8 +3,8 @@ // #include -#include "Aero_rad_props.H" #include "ERF_Constants.H" +#include "Aero_rad_props.H" #include "Water_vapor_saturation.H" void AerRadProps::initialize() { diff --git a/Source/Radiation/Albedo.H b/Source/Radiation/Albedo.H new file mode 100644 index 000000000..7fc8cc150 --- /dev/null +++ b/Source/Radiation/Albedo.H @@ -0,0 +1,13 @@ +/*------------------------------------------------------------ + Computes surface albedos + For more details , see Briegleb, Bruce P., 1992: Delta-Eddington + Approximation for Solar Radiation in the NCAR Community Climate Model, + Journal of Geophysical Research, Vol 97, D7, pp7603-7612). + + NOTE: this is the simplest formula for albedo calculation, we have to + update to the latest implementation from microphysics output. +*/ +#ifndef ERF_ALBEDO_H_ +#define ERF_ALBEDO_H_ +void set_albedo(const real1d& coszrs, real2d& albedo_dir, real2d& albedo_dif); +#endif diff --git a/Source/Radiation/Albedo.cpp b/Source/Radiation/Albedo.cpp new file mode 100644 index 000000000..d4414d4e9 --- /dev/null +++ b/Source/Radiation/Albedo.cpp @@ -0,0 +1,15 @@ + +#include +#include "Albedo.H" + +void set_albedo(const real1d& coszrs, real2d& albedo_dir, real2d& albedo_dif) +{ + // Albedos for land type I (Briegleb) + auto nswbands = albedo_dir.extent(0); + auto ncol = albedo_dif.extent(1); + yakl::c::parallel_for(yakl::c::Bounds<2>(nswbands,ncol), YAKL_LAMBDA (int ibnd, int icol) { + albedo_dir(ibnd, icol) = 1.4 * 0.24 / ( 1. + 0.8 * coszrs(icol)); + albedo_dif(ibnd, icol) = 1.2 * 0.24; + }); +} + diff --git a/Source/Radiation/Cloud_rad_props.H b/Source/Radiation/Cloud_rad_props.H index 748812bcf..dbcd80606 100644 --- a/Source/Radiation/Cloud_rad_props.H +++ b/Source/Radiation/Cloud_rad_props.H @@ -4,9 +4,11 @@ #ifndef ERF_CLOUD_RAD_PROPS_H_ #define ERF_CLOUD_RAD_PROPS_H_ -#include "ERF.H" -#include "Linear_interpolate.H" +#include +#include "ERF_Constants.H" #include "rrtmgp_const.h" +#include "Water_vapor_saturation.H" +#include "Linear_interpolate.H" class CloudRadProps { public: diff --git a/Source/Radiation/Cloud_rad_props.cpp b/Source/Radiation/Cloud_rad_props.cpp index 3c23fac83..c865d4702 100644 --- a/Source/Radiation/Cloud_rad_props.cpp +++ b/Source/Radiation/Cloud_rad_props.cpp @@ -3,8 +3,6 @@ // #include "YAKL_netcdf.h" #include "Cloud_rad_props.H" -#include "Water_vapor_saturation.H" -#include "Linear_interpolate.H" void CloudRadProps::initialize() { realHost1d g_mu_h; // mu samples on grid diff --git a/Source/Radiation/Init_rrtmgp.cpp b/Source/Radiation/Init_rrtmgp.cpp index 12da46121..f00045cf4 100644 --- a/Source/Radiation/Init_rrtmgp.cpp +++ b/Source/Radiation/Init_rrtmgp.cpp @@ -10,7 +10,9 @@ // Rrtmgp #include "Rrtmgp.H" -void Rrtmgp::initialize() +void Rrtmgp::initialize(int num_gas, std::vector active_gas_names, + const char* rrtmgp_coefficients_file_sw, + const char* rrtmgp_coefficients_file_lw) { // First, make sure yakl has been initialized if (!yakl::isInitialized()) { @@ -26,6 +28,13 @@ void Rrtmgp::initialize() // impossible from this initialization routine because I do not think the // rad_cnst objects are setup yet. // the other tasks! + ngas = num_gas; + for (auto i = 0; i < ngas; ++i) + gas_names[i] = active_gas_names[i].c_str(); + + coefficients_file_sw = rrtmgp_coefficients_file_sw; + coefficients_file_lw = rrtmgp_coefficients_file_lw; + auto active_gases = string1d("active_gases", ngas); for (int igas=0; igas 271.) { - aldir(icol) = ( .026 / (std::pow(coszrs(icol), 1.7) + .065)) + - (.15*(coszrs(icol) - 0.10) * (coszrs(icol) - 0.50) * (coszrs(icol) - 1.00) ); - asdir(icol) = aldir(icol); - aldif(icol) = adif; - asdif(icol) = adif; - } else { - // albedo of sea-ice/snow surface - aldir(icol) = 0.45; - asdir(icol) = 0.75; - aldif(icol) = 0.45; - asdif(icol) = 0.75; - } - }); - } else {// land - yakl::c::parallel_for(ncol, YAKL_LAMBDA (int icol) { - if (coszrs(icol) <= 0.) { - aldir(icol) = 0.; - asdir(icol) = 0.; - aldif(icol) = 0.; - asdif(icol) = 0.; - } else { - // Albedos for land type I (Briegleb) - asdir(icol) = 1.4 * 0.06 / ( 1. + 0.8 * coszrs(icol)); - asdif(icol) = 1.2 * 0.06; - aldir(icol) = 1.4 * 0.24 / ( 1. + 0.8 * coszrs(icol)); - aldif(icol) = 1.2 * 0.24; - } - }); - } - } #endif diff --git a/Source/Radiation/Radiation.H b/Source/Radiation/Radiation.H index cfa7a7f6c..8284a97fa 100644 --- a/Source/Radiation/Radiation.H +++ b/Source/Radiation/Radiation.H @@ -4,7 +4,7 @@ * https://github.com/earth-system-radiation/rte-rrtmgp * Please reference to the following paper, * https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2019MS001621 - * NOTE: we use the C++ version of RTE-RRTMGP, which is reimplemented the original Fortran + * NOTE: we use the C++ version of RTE-RRTMGP, which is the implemention of the original Fortran * code using C++ YAKL for CUDA, HiP and SYCL application by E3SM ECP team, the C++ version * of the rte-rrtmgp code is located at: * https://github.com/E3SM-Project/rte-rrtmgp @@ -28,6 +28,8 @@ #include "Rrtmgp.H" #include "Optics.H" #include "Aero_rad_props.H" +#include "Parameterizations.H" +#include "Albedo.H" // Radiation code interface class class Radiation { @@ -36,57 +38,54 @@ class Radiation { ~Radiation() = default; // init - void initialize (const amrex::MultiFab& cons_in, - const amrex::MultiFab& qc_in, - const amrex::MultiFab& qv_in, - const amrex::MultiFab& qi_in, - const amrex::BoxArray& grids, - const amrex::Geometry& geom, - const amrex::Real& dt_advance, - const bool& do_sw_rad, - const bool& do_lw_rad, - const bool& do_aero_rad, - const bool& do_snow_opt, - const bool& is_cmip6_volcano); + void initialize(const amrex::MultiFab& cons_in, amrex::MultiFab& qmoist, + const amrex::BoxArray& grids, + const amrex::Geometry& geom, + const amrex::Real& dt_advance, + const bool& do_sw_rad, + const bool& do_lw_rad, + const bool& do_aero_rad, + const bool& do_snow_opt, + const bool& is_cmip6_volcano); // run radiation model - void run (); + void run(); // call back - void on_complete (); - - void radiation_driver_lw (int ncol, int nlev, - real3d& gas_vmr, - real2d& pmid, real2d& pint, real2d& tmid, real2d& tint, - real3d& cld_tau_gpt, real3d& aer_tau_bnd, - FluxesByband& fluxes_clrsky, FluxesByband& fluxes_allsky, - real2d& qrl, real2d& qrlc); - - void radiation_driver_sw (int ncol, - real3d& gas_vmr, real2d& pmid, real2d& pint, real2d& tmid, - real2d& albedo_dir, real2d& albedo_dif, real1d& coszrs, - real3d& cld_tau_gpt, real3d& cld_ssa_gpt, real3d& cld_asm_gpt, - real3d& aer_tau_bnd, real3d& aer_ssa_bnd, real3d& aer_asm_bnd, - FluxesByband& fluxes_clrsky, FluxesByband& fluxes_allsky, - real2d& qrs, real2d& qrsc); - - void set_daynight_indices (real1d& coszrs, - int1d& day_indices, - int1d& night_indices); - - void get_gas_vmr (std::vector& gas_names, - real3d& gas_vmr); - - void calculate_heating_rate (real2d& flux_up, - real2d& flux_dn, - real2d& pint, - real2d& heating_rate); -private: + void on_complete(); + + void radiation_driver_lw(int ncol, int nlev, + real3d& gas_vmr, + real2d& pmid, real2d& pint, real2d& tmid, real2d& tint, + real3d& cld_tau_gpt, real3d& aer_tau_bnd, + FluxesByband& fluxes_clrsky, FluxesByband& fluxes_allsky, + real2d& qrl, real2d& qrlc); + + void radiation_driver_sw(int ncol, + real3d& gas_vmr, real2d& pmid, real2d& pint, real2d& tmid, + real2d& albedo_dir, real2d& albedo_dif, real1d& coszrs, + real3d& cld_tau_gpt, real3d& cld_ssa_gpt, real3d& cld_asm_gpt, + real3d& aer_tau_bnd, real3d& aer_ssa_bnd, real3d& aer_asm_bnd, + FluxesByband& fluxes_clrsky, FluxesByband& fluxes_allsky, + real2d& qrs, real2d& qrsc); + + void set_daynight_indices(real1d& coszrs, + int1d& day_indices, + int1d& night_indices); + + void get_gas_vmr(const std::vector& gas_names, + real3d& gas_vmr); + + void calculate_heating_rate(real2d& flux_up, + real2d& flux_dn, + real2d& pint, + real2d& heating_rate); + private: // geometry amrex::Geometry m_geom; // valid boxes on which to evolve the solution - amrex::BoxArray m_gtoe; + amrex::BoxArray m_box; // number of vertical levels int nlev, zlo, zhi; @@ -119,7 +118,7 @@ private: // // Net flux calculated in this routine; used to check energy conservation in // the physics package driver? - real1d net_flux; //("net_flux",ncol); + real1d net_flux; // This should be module data or something specific to aerosol where it is used? bool is_cmip6_volc; // true if cmip6 style volcanic file is read otherwise false @@ -127,11 +126,11 @@ private: real dt; // time step(s) - needed for aerosol optics call // Surface and top fluxes - real1d fsns; //(pcols) ! Surface solar absorbed flux - real1d fsnt; //(pcols) ! Net column abs solar flux at model top - real1d flns; //(pcols) ! Srf longwave cooling (up-down) flux - real1d flnt; //(pcols) ! Net outgoing lw flux at model top - real1d fsds; //(pcols) ! Surface solar down flux + real1d fsns; //(ncols) ! Surface solar absorbed flux + real1d fsnt; //(ncols) ! Net column abs solar flux at model top + real1d flns; //(ncols) ! Srf longwave cooling (up-down) flux + real1d flnt; //(ncols) ! Net outgoing lw flux at model top + real1d fsds; //(ncols) ! Surface solar down flux // radiation data const std::vector active_gases = { @@ -186,8 +185,8 @@ private: int1d rrtmg_to_rrtmgp; // Pointers to heating rates on physics buffer - real2d qrs; //(:,:) ! shortwave radiative heating rate - real2d qrl; //(:,:) ! longwave radiative heating rate + real2d qrs; // shortwave radiative heating rate + real2d qrl; // longwave radiative heating rate // Pointers to fields on the physics buffer real2d zi; @@ -195,11 +194,8 @@ private: // Clear-sky heating rates are not on the physics buffer, and we have no // reason to put them there, so declare these are regular arrays here - real2d qrsc; //(pcols,pver) - real2d qrlc; //(pcols,pver) - - // Temporary variable for heating rate output - real2d hr; + real2d qrsc; + real2d qrlc; real2d tmid, pmid, pdel; real2d pint, tint; diff --git a/Source/Radiation/Radiation.cpp b/Source/Radiation/Radiation.cpp index d60bdad54..f7170f8d1 100644 --- a/Source/Radiation/Radiation.cpp +++ b/Source/Radiation/Radiation.cpp @@ -72,20 +72,18 @@ namespace internal { } // init -void Radiation::initialize(const amrex::MultiFab& cons_in, - const amrex::MultiFab& qc_in, - const amrex::MultiFab& qv_in, - const amrex::MultiFab& qi_in, - const amrex::BoxArray& grids, - const amrex::Geometry& geom, - const amrex::Real& dt_advance, +void Radiation::initialize(const MultiFab& cons_in, MultiFab& qmoist, + const BoxArray& grids, + const Geometry& geom, + const Real& dt_advance, const bool& do_sw_rad, const bool& do_lw_rad, const bool& do_aero_rad, const bool& do_snow_opt, const bool& is_cmip6_volcano) { + m_geom = geom; - m_gtoe = grids; + m_box = grids; auto dz = m_geom.CellSize(2); auto lowz = m_geom.ProbLo(2); @@ -108,14 +106,11 @@ void Radiation::initialize(const amrex::MultiFab& cons_in, // initialize cloud, aerosol, and radiation optics.initialize(); - radiation.initialize(); - - // Setup the RRTMGP interface - //rrtmgp_initialize(ngas, active_gases, rrtmgp_coefficients_file_sw, rrtmgp_coefficients_file_lw); - radiation.initialize(); + radiation.initialize(ngas, active_gases, + rrtmgp_coefficients_file_sw.c_str(), + rrtmgp_coefficients_file_lw.c_str()); // initialize the radiation data - auto nswbands = radiation.get_nband_sw(); auto nswgpts = radiation.get_ngpt_sw(); auto nlwbands = radiation.get_nband_lw(); @@ -130,9 +125,6 @@ void Radiation::initialize(const amrex::MultiFab& cons_in, } }); - // Temporary variable for heating rate output - hr = real2d("hr", ncol, nlev); - tmid = real2d("tmid", ncol, nlev); pmid = real2d("pmid", ncol, nlev); @@ -142,6 +134,14 @@ void Radiation::initialize(const amrex::MultiFab& cons_in, albedo_dir = real2d("albedo_dir", nswbands, ncol); albedo_dif = real2d("albedo_dif", nswbands, ncol); + qrs = real2d("qrs", ncol, nlev); // shortwave radiative heating rate + qrl = real2d("qrl", ncol, nlev); // longwave radiative heating rate + + // Clear-sky heating rates are not on the physics buffer, and we have no + // reason to put them there, so declare these are regular arrays here + qrsc = real2d("qrsc", ncol, nlev); + qrlc = real2d("qrlc", ncol, nlev); + amrex::Print() << " LW coefficents file: " // a/, & << " SW coefficents file: " // a/, & << " Frequency (timesteps) of Shortwave Radiation calc: " //,i5/, & @@ -158,8 +158,12 @@ void Radiation::initialize(const amrex::MultiFab& cons_in, // run radiation model void Radiation::run() { - real2d qrs("qrs", ncol, nlev); // shortwave radiative heating rate - real2d qrl("qrl", ncol, nlev); // longwave radiative heating rate + // local variables + // Temporary variable for heating rate output + real2d hr("hr", ncol, nlev); + + // Cosine solar zenith angle for all columns in chunk + real1d coszrs("coszrs", ncol); // Pointers to fields on the physics buffer real2d cld("cld", ncol, nlev), cldfsnow("cldfsnow", ncol, nlev), @@ -168,24 +172,6 @@ void Radiation::run() { des("des", ncol, nlev), lambdac("lambdac", ncol, nlev), mu("mu", ncol, nlev), rei("rei", ncol, nlev), rel("rel", ncol, nlev); - // Clear-sky heating rates are not on the physics buffer, and we have no - // reason to put them there, so declare these are regular arrays here - real2d qrsc("qrsc", ncol, nlev); - real2d qrlc("qrlc", ncol, nlev); - - // Temporary variable for heating rate output - real2d hr("hr", ncol, nlev); - - real2d tmid("tmid", ncol, nlev), - pmid("pmid", ncol, nlev); - real2d pint("pint", ncol, nlev+1 ), - tint("tint", ncol, nlev+1); - real2d albedo_dir("albedo_dir", nswbands, ncol), - albedo_dif("albedo_dif", nswbands, ncol); - - // Cosine solar zenith angle for all columns in chunk - real1d coszrs("coszrs", ncol); - // Cloud, snow, and aerosol optical properties real3d cld_tau_gpt_sw("cld_tau_gpt_sw", ncol, nlev, nswgpts), cld_ssa_gpt_sw("cld_ssa_gpt_sw", ncol, nlev, nswgpts), @@ -199,6 +185,7 @@ void Radiation::run() { real3d cld_tau_bnd_lw("cld_tau_bnd_lw", ncol, nlev, nlwbands), aer_tau_bnd_lw("aer_tau_bnd_lw", ncol, nlev, nlwbands); real3d cld_tau_gpt_lw("cld_tau_gpt_lw", ncol, nlev, nlwgpts); + // NOTE: these are diagnostic only real3d liq_tau_bnd_sw("liq_tau_bnd_sw", ncol, nlev, nswbands), ice_tau_bnd_sw("ice_tau_bnd_sw", ncol, nlev, nswbands), @@ -234,12 +221,13 @@ void Radiation::run() { internal::initial_fluxes(ncol, nlev, nlwbands, fluxes_allsky); internal::initial_fluxes(ncol, nlev, nlwbands, fluxes_clrsky); + // Get cosine solar zenith angle for current time step. ( still NOT YET implemented here) +// set_cosine_solar_zenith_angle(state, dt_avg, coszrs(1:ncol)) + yakl::memset(coszrs, 0.2); // we set constant value here to avoid numerical overflow. + // Get albedo. This uses CAM routines internally and just provides a // wrapper to improve readability of the code here. -// set_albedo(cam_in, albedo_dir(1:nswbands,1:ncol), albedo_dif(1:nswbands,1:ncol)) - - // Get cosine solar zenith angle for current time step. -// set_cosine_solar_zenith_angle(state, dt_avg, coszrs(1:ncol)) + set_albedo(coszrs, albedo_dir, albedo_dif); // Do shortwave cloud optics calculations yakl::memset(cld_tau_gpt_sw, 0.); @@ -252,11 +240,6 @@ void Radiation::run() { cld_tau_bnd_sw, cld_ssa_bnd_sw, cld_asm_bnd_sw, liq_tau_bnd_sw, ice_tau_bnd_sw, snw_tau_bnd_sw); - // Output the band-by-band cloud optics BEFORE we reorder bands, because - // we hard-coded the indices for diagnostic bands in radconstants.F90 to - // correspond to the optical property look-up tables. -// output_cloud_optics_sw(state, cld_tau_bnd_sw, cld_ssa_bnd_sw, cld_asm_bnd_sw); - // Now reorder bands to be consistent with RRTMGP // We need to fix band ordering because the old input files assume RRTMG // band ordering, but this has changed in RRTMGP. @@ -305,13 +288,10 @@ void Radiation::run() { if (night_indices(icol) > 0) nnight++; } - // Loop over diagnostic calls -// rad_cnst_get_call_list(active_calls); - - for (auto icall = 1 /*N_DIAG*/; icall > 0; --icall) { + for (auto icall = ngas /*N_DIAG*/; icall > 0; --icall) { // if (active_calls(icall)) { // Get gas concentrations -// get_gas_vmr(icall, active_gases, gas_vmr); + get_gas_vmr(active_gases, gas_vmr); // Get aerosol optics if (do_aerosol_rad) { @@ -354,14 +334,6 @@ void Radiation::run() { yakl::memset(aer_asm_bnd_sw, 0.); } - // Check (and possibly clip) values before passing to RRTMGP driver - // handle_error(clip_values(cld_tau_gpt_sw, 0._r8, huge(cld_tau_gpt_sw), trim(subname) // ' cld_tau_gpt_sw', tolerance=1e-10_r8)) - // handle_error(clip_values(cld_ssa_gpt_sw, 0._r8, 1._r8, trim(subname) // ' cld_ssa_gpt_sw', tolerance=1e-10_r8)) - // handle_error(clip_values(cld_asm_gpt_sw, -1._r8, 1._r8, trim(subname) // ' cld_asm_gpt_sw', tolerance=1e-10_r8)) - // handle_error(clip_values(aer_tau_bnd_sw, 0._r8, huge(aer_tau_bnd_sw), trim(subname) // ' aer_tau_bnd_sw', tolerance=1e-10_r8)) - // handle_error(clip_values(aer_ssa_bnd_sw, 0._r8, 1._r8, trim(subname) // ' aer_ssa_bnd_sw', tolerance=1e-10_r8)) - // handle_error(clip_values(aer_asm_bnd_sw, -1._r8, 1._r8, trim(subname) // ' aer_asm_bnd_sw', tolerance=1e-10_r8)) - // Call the shortwave radiation driver radiation_driver_sw( ncol, gas_vmr, @@ -372,8 +344,6 @@ void Radiation::run() { ); } - // Set net fluxes used by other components (land?) -// set_net_fluxes_sw(fluxes_allsky, fsds, fsns, fsnt); } else { // Conserve energy @@ -413,10 +383,10 @@ void Radiation::run() { // Loop over diagnostic calls //rad_cnst_get_call_list(active_calls); - for (auto icall = 1; /*N_DIAG;*/ icall > 0; --icall) { + for (auto icall = ngas; /*N_DIAG;*/ icall > 0; --icall) { // if (active_calls(icall)) { // Get gas concentrations - //get_gas_vmr(icall, active_gases, gas_vmr); + get_gas_vmr(active_gases, gas_vmr); // Get aerosol optics yakl::memset(aer_tau_bnd_lw, 0.); @@ -425,22 +395,10 @@ void Radiation::run() { aer_rad.aer_rad_props_lw(is_cmip6_volc, icall, dt, zi, aer_tau_bnd_lw, clear_rh); } - // Check (and possibly clip) values before passing to RRTMGP driver - //handle_error(clip_values(cld_tau_gpt_lw, 0._r8, huge(cld_tau_gpt_lw), trim(subname) // ': cld_tau_gpt_lw', tolerance=1e-10_r8)) - //handle_error(clip_values(aer_tau_bnd_lw, 0._r8, huge(aer_tau_bnd_lw), trim(subname) // ': aer_tau_bnd_lw', tolerance=1e-10_r8)) - // Call the longwave radiation driver to calculate fluxes and heating rates radiation_driver_lw(ncol, nlev, gas_vmr, pmid, pint, tmid, tint, cld_tau_gpt_lw, aer_tau_bnd_lw, fluxes_allsky, fluxes_clrsky, qrl, qrlc); - // Send fluxes to history buffer - // call output_fluxes_lw(icall, state, fluxes_allsky, fluxes_clrsky, qrl, qrlc) } - - // Set net fluxes used in other components - //set_net_fluxes_lw(fluxes_allsky, flns, flnt); - - // Export surface fluxes that are used by the land model - //call export_surface_fluxes(fluxes_allsky, cam_out, 'longwave') } else { // Conserve energy (what does this mean exactly?) @@ -454,11 +412,6 @@ void Radiation::run() { } // dolw - // Compute net radiative heating tendency -// radheat_tend(ptend, qrl, qrs, -// fsns, fsnt, flns, flnt, -// cam_in%asdir, net_flux); - // Compute heating rate for dtheta/dt for (auto ilay = 1; ilay < nlev; ++nlev) { for (auto icol = 1; icol < ncol; ++icol) { @@ -691,8 +644,7 @@ void Radiation::set_daynight_indices(real1d& coszrs, int1d& day_indices, int1d& // Loop over columns and identify daytime columns as those where the cosine // solar zenith angle exceeds zero. Note that we wrap the setting of // day_indices in an if-then to make sure we are not accesing day_indices out - // of bounds, and stopping with an informative error message if we do for some - // reason. + // of bounds, and stopping with an informative error message if we do for some reason. int iday = 0; int inight = 0; for (auto icol = 0; icol < ncol; ++icol) { @@ -706,7 +658,7 @@ void Radiation::set_daynight_indices(real1d& coszrs, int1d& day_indices, int1d& } } -void Radiation::get_gas_vmr(std::vector& gas_names, real3d& gas_vmr) { +void Radiation::get_gas_vmr(const std::vector& gas_names, real3d& gas_vmr) { // Mass mixing ratio real2d mmr("mmr", ncol, nlev); // Gases and molecular weights. Note that we do NOT have CFCs yet (I think diff --git a/Source/Radiation/Rrtmgp.H b/Source/Radiation/Rrtmgp.H index 8d0790f5b..54b9412c3 100644 --- a/Source/Radiation/Rrtmgp.H +++ b/Source/Radiation/Rrtmgp.H @@ -20,7 +20,6 @@ #include #include -#include #include // rrtmgp includes @@ -38,12 +37,13 @@ class Rrtmgp { public: // constructor Rrtmgp() = default; - explicit Rrtmgp(int ngases, char* gas_names[]); // deconstructor ~Rrtmgp() = default; // initialize and load gas property data for rrtmgp radiation - void initialize(); + void initialize(int num_gas, std::vector active_gas_names, + const char* rrtmgp_coefficients_file_sw, + const char* rrtmgp_coefficients_file_lw); // finalize/clean up void finalize(); @@ -117,7 +117,7 @@ class Rrtmgp { // number of gas for radiation model int ngas; - char** gas_names; + const char** gas_names; string1d active_gases; diff --git a/Source/Radiation/data/cam5_ice_optics.nc b/Source/Radiation/data/cam5_ice_optics.nc new file mode 100644 index 000000000..9d60ddc12 Binary files /dev/null and b/Source/Radiation/data/cam5_ice_optics.nc differ diff --git a/Source/Radiation/data/cam5_liq_optics.nc b/Source/Radiation/data/cam5_liq_optics.nc new file mode 100644 index 000000000..6dd6b077e Binary files /dev/null and b/Source/Radiation/data/cam5_liq_optics.nc differ diff --git a/Source/TimeIntegration/ERF_Advance.cpp b/Source/TimeIntegration/ERF_Advance.cpp index 207440853..621bc96d7 100644 --- a/Source/TimeIntegration/ERF_Advance.cpp +++ b/Source/TimeIntegration/ERF_Advance.cpp @@ -119,14 +119,14 @@ ERF::Advance (int lev, Real time, Real dt_lev, int /*iteration*/, int /*ncycle*/ // Update the dycore advance_dycore(lev, - cons_mf, S_new, - U_old, V_old, W_old, - U_new, V_new, W_new, - rU_old[lev], rV_old[lev], rW_old[lev], - rU_new[lev], rV_new[lev], rW_new[lev], - rU_crse, rV_crse, rW_crse, - source, buoyancy, - Geom(lev), dt_lev, time, &ifr); + cons_mf, S_new, + U_old, V_old, W_old, + U_new, V_new, W_new, + rU_old[lev], rV_old[lev], rW_old[lev], + rU_new[lev], rV_new[lev], rW_new[lev], + rU_crse, rV_crse, rW_crse, + source, buoyancy, + Geom(lev), dt_lev, time, &ifr); #if defined(ERF_USE_MOISTURE) // Update the microphysics diff --git a/Source/TimeIntegration/ERF_TimeStep.cpp b/Source/TimeIntegration/ERF_TimeStep.cpp index 5d5361c20..5244dcefb 100644 --- a/Source/TimeIntegration/ERF_TimeStep.cpp +++ b/Source/TimeIntegration/ERF_TimeStep.cpp @@ -36,7 +36,7 @@ ERF::timeStep (int lev, Real time, int iteration) // NOTE: Def & Reg index lev backwards (so we add 1 here) // Redefine & register the ERFFillpatcher objects - if (coupling_type=="OneWay" && cf_width>0) { + if (solverChoice.coupling_type == CouplingType::OneWay && cf_width>0) { Define_ERFFillPatchers(lev+1); Register_ERFFillPatchers(lev+1); if (lev < max_level-1) { @@ -87,7 +87,7 @@ ERF::timeStep (int lev, Real time, int iteration) timeStep(lev+1, time+(i-1)*dt[lev+1], i); } - if (coupling_type == "TwoWay") { + if (solverChoice.coupling_type == CouplingType::TwoWay) { AverageDownTo(lev); // average lev+1 down to lev } } diff --git a/Source/TimeIntegration/ERF_advance_dycore.cpp b/Source/TimeIntegration/ERF_advance_dycore.cpp index e07bc1881..25a080309 100644 --- a/Source/TimeIntegration/ERF_advance_dycore.cpp +++ b/Source/TimeIntegration/ERF_advance_dycore.cpp @@ -314,7 +314,7 @@ void ERF::advance_dycore(int level, mri_integrator.advance(state_old, state_new, old_time, dt_advance); // Register coarse data for coarse-fine fill - if (level0) { + if (level0) { FPr_c[level].RegisterCoarseData({&cons_old, &cons_new}, {old_time, old_time + dt_advance}); FPr_u[level].RegisterCoarseData({&xvel_old, &xvel_new}, {old_time, old_time + dt_advance}); FPr_v[level].RegisterCoarseData({&yvel_old, &yvel_new}, {old_time, old_time + dt_advance}); diff --git a/Source/TimeIntegration/ERF_advance_radiation.cpp b/Source/TimeIntegration/ERF_advance_radiation.cpp new file mode 100644 index 000000000..c04904c0f --- /dev/null +++ b/Source/TimeIntegration/ERF_advance_radiation.cpp @@ -0,0 +1,29 @@ + +#include + +using namespace amrex; + +#if defined(ERF_USE_RRTMGP) +void ERF::advance_radiation (int lev, + MultiFab& cons, + const Real& dt_advance) +{ + bool do_sw_rad {true}; + bool do_lw_rad {true}; + bool do_aero_rad {true}; + bool do_snow_opt {true}; + bool is_cmip6_volcano {true}; + + rad.initialize(cons, qmoist[lev], + grids[lev], + Geom(lev), + dt_advance, + do_sw_rad, + do_lw_rad, + do_aero_rad, + do_snow_opt, + is_cmip6_volcano); +// rad.run(); + rad.on_complete(); +} +#endif diff --git a/Source/TimeIntegration/ERF_fast_rhs_T.cpp b/Source/TimeIntegration/ERF_fast_rhs_T.cpp index 51a3134f2..8cdd9be40 100644 --- a/Source/TimeIntegration/ERF_fast_rhs_T.cpp +++ b/Source/TimeIntegration/ERF_fast_rhs_T.cpp @@ -192,7 +192,7 @@ void erf_fast_rhs_T (int step, int /*level*/, const Array4 & stage_xmom = S_stage_data[IntVar::xmom].const_array(mfi); const Array4 & stage_ymom = S_stage_data[IntVar::ymom].const_array(mfi); -#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_MOISTURE) +#if defined(ERF_USE_MOISTURE) || defined(ERF_USE_WARM_NO_PRECIP) const Array4 & prim = S_stage_prim.const_array(mfi); #endif diff --git a/Source/TimeIntegration/ERF_slow_rhs_post.cpp b/Source/TimeIntegration/ERF_slow_rhs_post.cpp index 32579dbfb..51c10dd7b 100644 --- a/Source/TimeIntegration/ERF_slow_rhs_post.cpp +++ b/Source/TimeIntegration/ERF_slow_rhs_post.cpp @@ -44,7 +44,7 @@ using namespace amrex; * @param[in] mapfac_v map factor at y-faces */ -void erf_slow_rhs_post (int level, +void erf_slow_rhs_post (int level, int finest_level, int nrk, Real dt, Vector& S_rhs, @@ -93,8 +93,9 @@ void erf_slow_rhs_post (int level, const MultiFab* t_mean_mf = nullptr; if (most) t_mean_mf = most->get_mac_avg(0,2); - const bool l_use_terrain = solverChoice.use_terrain; - const bool l_moving_terrain = (solverChoice.terrain_type == 1); + const bool l_use_terrain = solverChoice.use_terrain; + const bool l_two_way_coupling = (solverChoice.coupling_type == CouplingType::TwoWay); + const bool l_moving_terrain = (solverChoice.terrain_type == TerrainType::Moving); if (l_moving_terrain) AMREX_ALWAYS_ASSERT(l_use_terrain); const bool l_use_ndiff = solverChoice.use_NumDiff; @@ -250,27 +251,27 @@ void erf_slow_rhs_post (int level, if (l_use_deardorff) { start_comp = RhoKE_comp; num_comp = 1; - AdvectionSrcForScalars(tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, - cur_prim, cell_rhs, detJ_arr, dxInv, mf_m, + AdvectionSrcForScalars(level, finest_level, mfi, tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, + cur_prim, cell_rhs, detJ_arr, dt, dxInv, mf_m, horiz_adv_type, vert_adv_type, - l_use_terrain); + l_use_terrain, l_two_way_coupling); } if (l_use_QKE) { start_comp = RhoQKE_comp; num_comp = 1; - AdvectionSrcForScalars(tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, - cur_prim, cell_rhs, detJ_arr, dxInv, mf_m, + AdvectionSrcForScalars(level, finest_level, mfi, tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, + cur_prim, cell_rhs, detJ_arr, dt, dxInv, mf_m, horiz_adv_type, vert_adv_type, - l_use_terrain); + l_use_terrain, l_two_way_coupling); } // This is simply an advected scalar for convenience start_comp = RhoScalar_comp; num_comp = 1; - AdvectionSrcForScalars(tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, - cur_prim, cell_rhs, detJ_arr, dxInv, mf_m, - horiz_adv_type, vert_adv_type, - l_use_terrain); + AdvectionSrcForScalars(level, finest_level, mfi, tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, + cur_prim, cell_rhs, detJ_arr, dt, dxInv, mf_m, + horiz_adv_type, vert_adv_type, + l_use_terrain, l_two_way_coupling); #ifdef ERF_USE_MOISTURE start_comp = RhoQt_comp; @@ -283,10 +284,10 @@ void erf_slow_rhs_post (int level, moist_horiz_adv_type = EfficientAdvType(nrk,ac.moistscal_horiz_adv_type); moist_vert_adv_type = EfficientAdvType(nrk,ac.moistscal_vert_adv_type); } - AdvectionSrcForScalars(tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, - cur_prim, cell_rhs, detJ_arr, dxInv, mf_m, + AdvectionSrcForScalars(level, finest_level, mfi, tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, + cur_prim, cell_rhs, detJ_arr, dt, dxInv, mf_m, moist_horiz_adv_type, moist_vert_adv_type, - l_use_terrain); + l_use_terrain, l_two_way_coupling); #elif defined(ERF_USE_WARM_NO_PRECIP) start_comp = RhoQv_comp; @@ -300,10 +301,10 @@ void erf_slow_rhs_post (int level, moist_vert_adv_type = EfficientAdvType(nrk,solverChoice.moistscal_vert_adv_type); } - AdvectionSrcForScalars(tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, - cur_prim, cell_rhs, detJ_arr, dxInv, mf_m, + AdvectionSrcForScalars(level, finest_level, mfi, tbx, start_comp, num_comp, avg_xmom, avg_ymom, avg_zmom, + cur_prim, cell_rhs, detJ_arr, dt, dxInv, mf_m, moist_horiz_adv_type, moist_vert_adv_type, - l_use_terrain); + l_use_terrain, l_two_way_coupling); #endif if (l_use_diff) { diff --git a/Source/TimeIntegration/ERF_slow_rhs_pre.cpp b/Source/TimeIntegration/ERF_slow_rhs_pre.cpp index 58a768be7..50b509895 100644 --- a/Source/TimeIntegration/ERF_slow_rhs_pre.cpp +++ b/Source/TimeIntegration/ERF_slow_rhs_pre.cpp @@ -119,7 +119,7 @@ void erf_slow_rhs_pre (int level, const AdvType l_horiz_adv_type = solverChoice.advChoice.dycore_horiz_adv_type; const AdvType l_vert_adv_type = solverChoice.advChoice.dycore_vert_adv_type; const bool l_use_terrain = solverChoice.use_terrain; - const bool l_moving_terrain = (solverChoice.terrain_type == 1); + const bool l_moving_terrain = (solverChoice.terrain_type == TerrainType::Moving); if (l_moving_terrain) AMREX_ALWAYS_ASSERT (l_use_terrain); const bool l_use_ndiff = solverChoice.use_NumDiff; @@ -801,8 +801,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i-1,j,k,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i-1,j,k,PrimQc_comp) ); #endif - rho_u_rhs(i, j, k) += -gpx / (1.0 + q) - - abl_pressure_grad[0] + rho_u_rhs(i, j, k) += (-gpx - abl_pressure_grad[0]) / (1.0 + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i-1,j,k,Rho_comp)) * abl_geo_forcing[0]; // Add Coriolis forcing (that assumes east is +x, north is +y) @@ -810,8 +809,7 @@ void erf_slow_rhs_pre (int level, { Real rho_v_loc = 0.25 * (rho_v(i,j+1,k) + rho_v(i,j,k) + rho_v(i-1,j+1,k) + rho_v(i-1,j,k)); Real rho_w_loc = 0.25 * (rho_w(i,j,k+1) + rho_w(i,j,k) + rho_w(i,j-1,k+1) + rho_w(i,j-1,k)); - rho_u_rhs(i, j, k) += coriolis_factor * - (rho_v_loc * sinphi - rho_w_loc * cosphi); + rho_u_rhs(i, j, k) += coriolis_factor * (rho_v_loc * sinphi - rho_w_loc * cosphi); } // Add Rayleigh damping @@ -846,8 +844,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i-1,j,k,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i-1,j,k,PrimQc_comp) ); #endif - rho_u_rhs(i, j, k) += -gpx / (1.0 + q) - - abl_pressure_grad[0] + rho_u_rhs(i, j, k) += (-gpx - abl_pressure_grad[0]) / (1.0 + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i-1,j,k,Rho_comp)) * abl_geo_forcing[0]; // Add Coriolis forcing (that assumes east is +x, north is +y) @@ -855,8 +852,7 @@ void erf_slow_rhs_pre (int level, { Real rho_v_loc = 0.25 * (rho_v(i,j+1,k) + rho_v(i,j,k) + rho_v(i-1,j+1,k) + rho_v(i-1,j,k)); Real rho_w_loc = 0.25 * (rho_w(i,j,k+1) + rho_w(i,j,k) + rho_w(i,j-1,k+1) + rho_w(i,j-1,k)); - rho_u_rhs(i, j, k) += coriolis_factor * - (rho_v_loc * sinphi - rho_w_loc * cosphi); + rho_u_rhs(i, j, k) += coriolis_factor * (rho_v_loc * sinphi - rho_w_loc * cosphi); } // Add Rayleigh damping @@ -911,8 +907,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i,j-1,k,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i,j-1,k,PrimQc_comp) ); #endif - rho_v_rhs(i, j, k) += -gpy / (1.0_rt + q) - - abl_pressure_grad[1] + rho_v_rhs(i, j, k) += (-gpy - abl_pressure_grad[1]) / (1.0_rt + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i,j-1,k,Rho_comp)) * abl_geo_forcing[1]; // Add Coriolis forcing (that assumes east is +x, north is +y) if (use_coriolis) @@ -953,8 +948,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i,j-1,k,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i,j-1,k,PrimQc_comp) ); #endif - rho_v_rhs(i, j, k) += -gpy / (1.0_rt + q) - - abl_pressure_grad[1] + rho_v_rhs(i, j, k) += (-gpy - abl_pressure_grad[1]) / (1.0_rt + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i,j-1,k,Rho_comp)) * abl_geo_forcing[1]; // Add Coriolis forcing (that assumes east is +x, north is +y) @@ -1007,8 +1001,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i,j,k-1,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i,j,k-1,PrimQc_comp) ); #endif - rho_w_rhs(i, j, k) += (buoyancy_fab(i,j,k) - gpz) / (1.0_rt + q) - - abl_pressure_grad[2] + rho_w_rhs(i, j, k) += (buoyancy_fab(i,j,k) - gpz - abl_pressure_grad[2]) / (1.0_rt + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i,j,k-1,Rho_comp)) * abl_geo_forcing[2]; // Add Coriolis forcing (that assumes east is +x, north is +y) @@ -1048,8 +1041,7 @@ void erf_slow_rhs_pre (int level, q = 0.5 * ( cell_prim(i,j,k,PrimQv_comp) + cell_prim(i,j,k-1,PrimQv_comp) +cell_prim(i,j,k,PrimQc_comp) + cell_prim(i,j,k-1,PrimQc_comp) ); #endif - rho_w_rhs(i, j, k) += (buoyancy_fab(i,j,k) - gpz) / (1.0_rt + q) - - abl_pressure_grad[2] + rho_w_rhs(i, j, k) += (buoyancy_fab(i,j,k) - gpz - abl_pressure_grad[2]) / (1.0_rt + q) + 0.5*(cell_data(i,j,k,Rho_comp)+cell_data(i,j,k-1,Rho_comp)) * abl_geo_forcing[2]; // Add Coriolis forcing (that assumes east is +x, north is +y) diff --git a/Source/TimeIntegration/TI_fast_rhs_fun.H b/Source/TimeIntegration/TI_fast_rhs_fun.H index 6ee23f68c..f27ea13ad 100644 --- a/Source/TimeIntegration/TI_fast_rhs_fun.H +++ b/Source/TimeIntegration/TI_fast_rhs_fun.H @@ -24,7 +24,7 @@ auto fast_rhs_fun = [&](int fast_step, int n_sub, // Moving terrain std::unique_ptr z_t_pert; - if ( solverChoice.use_terrain && (solverChoice.terrain_type == 1) ) + if ( solverChoice.use_terrain && (solverChoice.terrain_type == TerrainType::Moving) ) { // Make "old" fast geom -- store in z_phys_nd for convenience if (verbose) Print() << "Making geometry at start of substep time: " << old_substep_time << std::endl; @@ -94,7 +94,7 @@ auto fast_rhs_fun = [&](int fast_step, int n_sub, dtau, beta_s, inv_fac, mapfac_m[level], mapfac_u[level], mapfac_v[level]); } - } else if (solverChoice.use_terrain && solverChoice.terrain_type == 0) { + } else if (solverChoice.use_terrain && solverChoice.terrain_type == TerrainType::Static) { if (fast_step == 0) { // If this is the first substep we make the coefficients since they are based only on stage data diff --git a/Source/TimeIntegration/TI_headers.H b/Source/TimeIntegration/TI_headers.H index f1eb5fbaf..4c226be3b 100644 --- a/Source/TimeIntegration/TI_headers.H +++ b/Source/TimeIntegration/TI_headers.H @@ -62,7 +62,7 @@ void erf_slow_rhs_pre (int level, int nrk, * Function for computing the slow RHS for the evolution equations for the scalars other than density or potential temperature * */ -void erf_slow_rhs_post (int level, int nrk, +void erf_slow_rhs_post (int level, int finest_level, int nrk, amrex::Real dt, amrex::Vector& S_rhs, amrex::Vector& S_old, diff --git a/Source/TimeIntegration/TI_no_substep_fun.H b/Source/TimeIntegration/TI_no_substep_fun.H index 3e0f57383..14bdc3d37 100644 --- a/Source/TimeIntegration/TI_no_substep_fun.H +++ b/Source/TimeIntegration/TI_no_substep_fun.H @@ -49,7 +49,7 @@ Array4* fslow = fslow_d.dataPtr(); // Moving terrain - if ( solverChoice.use_terrain && solverChoice.terrain_type == 1 ) + if ( solverChoice.use_terrain && solverChoice.terrain_type == TerrainType::Moving ) { const Array4& dJ_old = detJ_cc[level]->const_array(mfi); const Array4& dJ_new = detJ_cc_new[level]->const_array(mfi); diff --git a/Source/TimeIntegration/TI_slow_rhs_fun.H b/Source/TimeIntegration/TI_slow_rhs_fun.H index f30077ddc..9be9eb6fc 100644 --- a/Source/TimeIntegration/TI_slow_rhs_fun.H +++ b/Source/TimeIntegration/TI_slow_rhs_fun.H @@ -18,7 +18,7 @@ Real slow_dt = new_stage_time - old_step_time; // Moving terrain - if ( solverChoice.use_terrain && (solverChoice.terrain_type == 1) ) + if ( solverChoice.use_terrain && (solverChoice.terrain_type == TerrainType::Moving) ) { // Note that the "old" and "new" metric terms correspond to // t^n and the RK stage (either t^*, t^** or t^{n+1} that this source @@ -194,16 +194,24 @@ #ifdef ERF_USE_NETCDF // Populate RHS for relaxation zones - if (init_type=="real" && level==0) { - wrfbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, - wrfbdy_width-1, wrfbdy_set_width, fine_geom, + if (((init_type == "real") || (init_type == "metgrid")) && level == 0) { + int width, set_width; + if (init_type == "real") { + width = wrfbdy_width; + set_width = wrfbdy_set_width; + } else if (init_type == "metgrid") { + width = metgrid_bdy_width; + set_width = metgrid_bdy_set_width; + } + wrfbdy_compute_interior_ghost_rhs(init_type, bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, + width-1, set_width, fine_geom, S_rhs, S_data, bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi); } #endif // Compute RHS for fine interior ghost - if (level>0 && coupling_type=="OneWay" && cf_width>0) { + if (level>0 && solverChoice.coupling_type == CouplingType::OneWay && cf_width>0) { fine_compute_interior_ghost_rhs(new_stage_time, slow_dt, cf_width, cf_set_width, fine_geom, &FPr_c[level-1], &FPr_u[level-1], &FPr_v[level-1], &FPr_w[level-1], @@ -264,11 +272,12 @@ // Flag for moisture relaxation zone bool moist_zero = false; if (init_type=="real" && level==0 && wrfbdy_set_width>0) moist_zero = true; + if (init_type=="metgrid" && level==0 && metgrid_bdy_set_width > 0) moist_zero = true; #endif // Moving terrain - if ( solverChoice.use_terrain && (solverChoice.terrain_type == 1) ) { - erf_slow_rhs_post(level, nrk, slow_dt, + if ( solverChoice.use_terrain && (solverChoice.terrain_type == TerrainType::Moving) ) { + erf_slow_rhs_post(level, finest_level, nrk, slow_dt, S_rhs, S_old, S_new, S_data, S_prim, S_scratch, xvel_new, yvel_new, zvel_new, source, SmnSmn, eddyDiffs, @@ -283,7 +292,7 @@ #endif ); } else { - erf_slow_rhs_post(level, nrk, slow_dt, + erf_slow_rhs_post(level, finest_level, nrk, slow_dt, S_rhs, S_old, S_new, S_data, S_prim, S_scratch, xvel_new, yvel_new, zvel_new, source, SmnSmn, eddyDiffs, @@ -341,9 +350,17 @@ #ifdef ERF_USE_NETCDF // Populate RHS for relaxation zones - if (init_type=="real" && level==0) { - wrfbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, - wrfbdy_width-1, wrfbdy_set_width, fine_geom, + if (((init_type == "real") || (init_type == "metgrid")) && level == 0) { + int width, set_width; + if (init_type == "real") { + width = wrfbdy_width; + set_width = wrfbdy_set_width; + } else if (init_type == "metgrid") { + width = metgrid_bdy_width; + set_width = metgrid_bdy_set_width; + } + wrfbdy_compute_interior_ghost_rhs(init_type, bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, + width-1, set_width, fine_geom, S_rhs, S_data, bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi); } diff --git a/Source/Utils/InteriorGhostCells.cpp b/Source/Utils/InteriorGhostCells.cpp index 9ad271b00..4d9b5950e 100644 --- a/Source/Utils/InteriorGhostCells.cpp +++ b/Source/Utils/InteriorGhostCells.cpp @@ -89,6 +89,7 @@ compute_interior_ghost_bxs_xy (const Box& bx, /** * Compute the RHS in the relaxation zone * + * @param[in] init_type initialization method for this simulation * @param[in] bdy_time_interval time interval between boundary condition time stamps * @param[in] time current time * @param[in] delta_t timestep @@ -104,7 +105,8 @@ compute_interior_ghost_bxs_xy (const Box& bx, * @param[in] start_bdy_time time of the first boundary data read in */ void -wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, +wrfbdy_compute_interior_ghost_rhs (const std::string& init_type, + const Real& bdy_time_interval, const Real& start_bdy_time, const Real& time, const Real& delta_t, @@ -126,8 +128,8 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, // Time interpolation Real dT = bdy_time_interval; - Real time_since_start = (time - start_bdy_time) / 1.e10; - int n_time = static_cast( time_since_start / dT); + Real time_since_start = time - start_bdy_time; + int n_time = static_cast( time_since_start / dT); amrex::Real alpha = (time_since_start - n_time * dT) / dT; AMREX_ALWAYS_ASSERT( alpha >= 0. && alpha <= 1.0); amrex::Real oma = 1.0 - alpha; @@ -145,13 +147,24 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, // Variable icomp map Vector comp_map = {0, 0, Rho_comp, RhoTheta_comp}; - // End of loop for WRFBdyVars - int WRFBdyEnd = WRFBdyVars::NumTypes-3; - + int BdyEnd, ivarU, ivarV, ivarR, ivarT; + if (init_type == "real") { + BdyEnd = WRFBdyVars::NumTypes-3; + ivarU = WRFBdyVars::U; + ivarV = WRFBdyVars::V; + ivarR = WRFBdyVars::R; + ivarT = WRFBdyVars::T; + } else if (init_type == "metgrid") { + BdyEnd = MetGridBdyVars::NumTypes-1; + ivarU = MetGridBdyVars::U; + ivarV = MetGridBdyVars::V; + ivarR = MetGridBdyVars::R; + ivarT = MetGridBdyVars::T; + } // Size the FABs //========================================================== - for (int ivar(WRFBdyVars::U); ivar < WRFBdyEnd; ivar++) + for (int ivar(ivarU); ivar < BdyEnd; ivar++) { int var_idx = var_map[ivar]; Box domain = geom.Domain(); @@ -167,16 +180,17 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, bx_ylo, bx_yhi, ng_vect, true); - if (ivar == WRFBdyVars::U) { + // Size the FABs + if (ivar == ivarU) { U_xlo.resize(bx_xlo,1); U_xhi.resize(bx_xhi,1); U_ylo.resize(bx_ylo,1); U_yhi.resize(bx_yhi,1); - } else if (ivar == WRFBdyVars::V) { + } else if (ivar == ivarV) { V_xlo.resize(bx_xlo,1); V_xhi.resize(bx_xhi,1); V_ylo.resize(bx_ylo,1); V_yhi.resize(bx_yhi,1); - } else if (ivar == WRFBdyVars::R) { + } else if (ivar == ivarR) { R_xlo.resize(bx_xlo,1); R_xhi.resize(bx_xhi,1); R_ylo.resize(bx_ylo,1); R_yhi.resize(bx_yhi,1); - } else if (ivar == WRFBdyVars::T){ + } else if (ivar == ivarT){ T_xlo.resize(bx_xlo,1); T_xhi.resize(bx_xhi,1); T_ylo.resize(bx_ylo,1); T_yhi.resize(bx_yhi,1); } else { @@ -197,10 +211,9 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, Elixir T_xlo_eli = T_xlo.elixir(); Elixir T_xhi_eli = T_xhi.elixir(); Elixir T_ylo_eli = T_ylo.elixir(); Elixir T_yhi_eli = T_yhi.elixir(); - // Populate FABs from boundary interpolation //========================================================== - for (int ivar(WRFBdyVars::U); ivar < WRFBdyEnd; ivar++) + for (int ivar(ivarU); ivar < BdyEnd; ivar++) { int var_idx = var_map[ivar]; Box domain = geom.Domain(); @@ -220,16 +233,16 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, Array4 arr_xlo; Array4 arr_xhi; Array4 arr_ylo; Array4 arr_yhi; - if (ivar == WRFBdyVars::U) { + if (ivar == ivarU) { arr_xlo = U_xlo.array(); arr_xhi = U_xhi.array(); arr_ylo = U_ylo.array(); arr_yhi = U_yhi.array(); - } else if (ivar == WRFBdyVars::V) { + } else if (ivar == ivarV) { arr_xlo = V_xlo.array(); arr_xhi = V_xhi.array(); arr_ylo = V_ylo.array(); arr_yhi = V_yhi.array(); - } else if (ivar == WRFBdyVars::R) { + } else if (ivar == ivarR) { arr_xlo = R_xlo.array(); arr_xhi = R_xhi.array(); arr_ylo = R_ylo.array(); arr_yhi = R_yhi.array(); - } else if (ivar == WRFBdyVars::T){ + } else if (ivar == ivarT){ arr_xlo = T_xlo.array(); arr_xhi = T_xhi.array(); arr_ylo = T_ylo.array(); arr_yhi = T_yhi.array(); } else { @@ -288,10 +301,9 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, }); } // ivar - // Velocity to momentum //========================================================== - for (int ivar(WRFBdyVars::U); ivar <= WRFBdyVars::V; ivar++) + for (int ivar(ivarU); ivar <= ivarV; ivar++) { int ivar_idx = ivar_map[ivar]; Box domain = geom.Domain(); @@ -312,7 +324,7 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, Array4 arr_xlo; Array4 arr_xhi; Array4 arr_ylo; Array4 arr_yhi; - if (ivar == WRFBdyVars::U) { + if (ivar == ivarU) { arr_xlo = U_xlo.array(); arr_xhi = U_xhi.array(); arr_ylo = U_ylo.array(); arr_yhi = U_yhi.array(); @@ -425,10 +437,11 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, } } // mfi +// return; // DJW debugging return statement to shut off relaxation zone. // Compute RHS in relaxation region //========================================================== - for (int ivar(WRFBdyVars::U); ivar < WRFBdyEnd; ivar++) + for (int ivar(ivarU); ivar < BdyEnd; ivar++) { int ivar_idx = ivar_map[ivar]; int icomp = comp_map[ivar]; @@ -455,22 +468,22 @@ wrfbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, Array4 rhs_arr; Array4 data_arr; Array4 arr_xlo; Array4 arr_xhi; Array4 arr_ylo; Array4 arr_yhi; - if (ivar == WRFBdyVars::U) { + if (ivar == ivarU) { arr_xlo = U_xlo.array(); arr_xhi = U_xhi.array(); arr_ylo = U_ylo.array(); arr_yhi = U_yhi.array(); rhs_arr = S_rhs[IntVar::xmom].array(mfi); data_arr = S_data[IntVar::xmom].array(mfi); - } else if (ivar == WRFBdyVars::V) { + } else if (ivar == ivarV) { arr_xlo = V_xlo.array(); arr_xhi = V_xhi.array(); arr_ylo = V_ylo.array(); arr_yhi = V_yhi.array(); rhs_arr = S_rhs[IntVar::ymom].array(mfi); data_arr = S_data[IntVar::ymom].array(mfi); - } else if (ivar == WRFBdyVars::R) { + } else if (ivar == ivarR) { arr_xlo = R_xlo.array(); arr_xhi = R_xhi.array(); arr_ylo = R_ylo.array(); arr_yhi = R_yhi.array(); rhs_arr = S_rhs[IntVar::cons].array(mfi); data_arr = S_data[IntVar::cons].array(mfi); - } else if (ivar == WRFBdyVars::T){ + } else if (ivar == ivarT){ arr_xlo = T_xlo.array(); arr_xhi = T_xhi.array(); arr_ylo = T_ylo.array(); arr_yhi = T_yhi.array(); rhs_arr = S_rhs[IntVar::cons].array(mfi); diff --git a/Source/Utils/Interpolation_UPW.H b/Source/Utils/Interpolation_UPW.H index 89586d5e2..e5f061dea 100644 --- a/Source/Utils/Interpolation_UPW.H +++ b/Source/Utils/Interpolation_UPW.H @@ -18,19 +18,13 @@ struct CENTERED2 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp1,s); - // Interpolate lo val_lo = Evaluate(s,sm1); } @@ -42,19 +36,13 @@ struct CENTERED2 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp1,s); - // Interpolate lo val_lo = Evaluate(s,sm1); } @@ -62,7 +50,7 @@ struct CENTERED2 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -77,24 +65,6 @@ struct CENTERED2 val_lo = Evaluate(s,sm1); } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real /*upw_hi*/) const - { - // Data to interpolate on - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - - // Interpolate hi - val_hi = Evaluate(sp1,s); - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -128,25 +98,18 @@ struct UPWIND3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); amrex::Real sm2 = m_phi(i-2, j , k , qty_index); // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; if (upw_lo != 0.) upw_lo = (upw_lo > 0) ? 1. : -1.; - // Interpolate hi - val_hi = Evaluate(sp2,sp1,s,sm1,upw_hi); - // Interpolate lo val_lo = Evaluate(sp1,s,sm1,sm2,upw_lo); } @@ -158,25 +121,18 @@ struct UPWIND3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); amrex::Real sm2 = m_phi(i , j-2, k , qty_index); // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; if (upw_lo != 0.) upw_lo = (upw_lo > 0) ? 1. : -1.; - // Interpolate hi - val_hi = Evaluate(sp2,sp1,s,sm1,upw_hi); - // Interpolate lo val_lo = Evaluate(sp1,s,sm1,sm2,upw_lo); } @@ -184,7 +140,7 @@ struct UPWIND3 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -204,29 +160,6 @@ struct UPWIND3 val_lo = Evaluate(sp1,s,sm1,sm2,upw_lo); } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - - // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; - - // Interpolate lo - val_hi = Evaluate(sp2,sp1,s,sm1,upw_hi); - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -268,21 +201,15 @@ struct CENTERED4 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); amrex::Real sm2 = m_phi(i-2, j , k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp2,sp1,s,sm1); - // Interpolate lo val_lo = Evaluate(sp1,s,sm1,sm2); } @@ -294,21 +221,15 @@ struct CENTERED4 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); amrex::Real sm2 = m_phi(i , j-2, k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp2,sp1,s,sm1); - // Interpolate lo val_lo = Evaluate(sp1,s,sm1,sm2); } @@ -316,7 +237,7 @@ struct CENTERED4 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -333,26 +254,6 @@ struct CENTERED4 val_lo = Evaluate(sp1,s,sm1,sm2); } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real /*upw_hi*/) const - { - // Data to interpolate on - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - - // Interpolate hi - val_hi = Evaluate(sp2,sp1,s,sm1); - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -390,13 +291,10 @@ struct UPWIND5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i+3, j , k , qty_index); amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -405,12 +303,8 @@ struct UPWIND5 amrex::Real sm3 = m_phi(i-3, j , k , qty_index); // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; if (upw_lo != 0.) upw_lo = (upw_lo > 0) ? 1. : -1.; - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2,upw_hi); - // Interpolate lo val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3,upw_lo); } @@ -422,13 +316,10 @@ struct UPWIND5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i , j+3, k , qty_index); amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -437,12 +328,8 @@ struct UPWIND5 amrex::Real sm3 = m_phi(i , j-3, k , qty_index); // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; if (upw_lo != 0.) upw_lo = (upw_lo > 0) ? 1. : -1.; - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2,upw_hi); - // Interpolate lo val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3,upw_lo); } @@ -450,7 +337,7 @@ struct UPWIND5 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -472,31 +359,6 @@ struct UPWIND5 val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3,upw_lo); } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp3 = m_phi(i , j , k+3, qty_index); - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - amrex::Real sm2 = m_phi(i , j , k-2, qty_index); - - // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; - - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2,upw_hi); - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -542,13 +404,10 @@ struct CENTERED6 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i+3, j , k , qty_index); amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -556,9 +415,6 @@ struct CENTERED6 amrex::Real sm2 = m_phi(i-2, j , k , qty_index); amrex::Real sm3 = m_phi(i-3, j , k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2); - // Interpolate lo val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3); } @@ -570,13 +426,10 @@ struct CENTERED6 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real /*upw_hi*/, amrex::Real /*upw_lo*/) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i , j+3, k , qty_index); amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -584,9 +437,6 @@ struct CENTERED6 amrex::Real sm2 = m_phi(i , j-2, k , qty_index); amrex::Real sm3 = m_phi(i , j-3, k , qty_index); - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2); - // Interpolate lo val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3); } @@ -594,7 +444,7 @@ struct CENTERED6 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -613,28 +463,6 @@ struct CENTERED6 val_lo = Evaluate(sp2,sp1,s,sm1,sm2,sm3); } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real /*upw_hi*/) const - { - // Data to interpolate on - amrex::Real sp3 = m_phi(i , j , k+3, qty_index); - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - amrex::Real sm2 = m_phi(i , j , k-2, qty_index); - - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2); - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -672,7 +500,7 @@ struct UPWINDALL AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -700,37 +528,6 @@ struct UPWINDALL } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi, - const AdvType adv_type) const - { - if (adv_type == AdvType::Centered_2nd) { - val_hi = 0.5 * ( m_phi(i,j,k,qty_index) + m_phi(i,j,k+1,qty_index) ); - return; - } else { - // Data to interpolate on - amrex::Real sp3 = (adv_type == AdvType::Upwind_5th || adv_type == AdvType::Centered_6th ) ? m_phi(i , j , k+3, qty_index): 0.; - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - amrex::Real sm2 = (adv_type == AdvType::Upwind_5th || adv_type == AdvType::Centered_6th ) ? m_phi(i , j , k-2, qty_index) : 0.; - - // Upwinding flags - if (upw_hi != 0.) upw_hi = (upw_hi > 0) ? 1. : -1.; - - // Interpolate hi - val_hi = Evaluate(sp3,sp2,sp1,s,sm1,sm2,upw_hi,adv_type); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real diff --git a/Source/Utils/Interpolation_WENO.H b/Source/Utils/Interpolation_WENO.H index bf9a24f40..01a4ca68d 100644 --- a/Source/Utils/Interpolation_WENO.H +++ b/Source/Utils/Interpolation_WENO.H @@ -18,26 +18,15 @@ struct WENO3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); amrex::Real sm2 = m_phi(i-2, j , k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -54,26 +43,15 @@ struct WENO3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); amrex::Real sm2 = m_phi(i , j-2, k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -86,7 +64,7 @@ struct WENO3 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -108,31 +86,6 @@ struct WENO3 } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -182,13 +135,10 @@ struct WENO5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i+3, j , k , qty_index); amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -196,14 +146,6 @@ struct WENO5 amrex::Real sm2 = m_phi(i-2, j , k , qty_index); amrex::Real sm3 = m_phi(i-3, j , k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm3,sm2,sm1,s ,sp1); } else if (upw_lo < -tol) { @@ -220,13 +162,10 @@ struct WENO5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i , j+3, k , qty_index); amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -234,14 +173,6 @@ struct WENO5 amrex::Real sm2 = m_phi(i , j-2, k , qty_index); amrex::Real sm3 = m_phi(i , j-3, k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm3,sm2,sm1,s ,sp1); } else if (upw_lo < -tol) { @@ -254,7 +185,7 @@ struct WENO5 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -278,33 +209,6 @@ struct WENO5 } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp3 = m_phi(i , j , k+3, qty_index); - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - amrex::Real sm2 = m_phi(i , j , k-2, qty_index); - - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real diff --git a/Source/Utils/Interpolation_WENO_Z.H b/Source/Utils/Interpolation_WENO_Z.H index 0d77bec86..dc65811b0 100644 --- a/Source/Utils/Interpolation_WENO_Z.H +++ b/Source/Utils/Interpolation_WENO_Z.H @@ -18,26 +18,15 @@ struct WENO_Z3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); amrex::Real sm2 = m_phi(i-2, j , k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -54,26 +43,15 @@ struct WENO_Z3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); amrex::Real sm2 = m_phi(i , j-2, k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -86,7 +64,7 @@ struct WENO_Z3 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -108,31 +86,6 @@ struct WENO_Z3 } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -183,26 +136,15 @@ struct WENO_MZQ3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i-1, j , k , qty_index); amrex::Real sm2 = m_phi(i-2, j , k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -219,26 +161,15 @@ struct WENO_MZQ3 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); amrex::Real sm1 = m_phi(i , j-1, k , qty_index); amrex::Real sm2 = m_phi(i , j-2, k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm2,sm1,s ); } else if (upw_lo < -tol) { @@ -251,7 +182,7 @@ struct WENO_MZQ3 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -273,31 +204,6 @@ struct WENO_MZQ3 } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - - if (upw_hi > tol){ - val_hi = Evaluate(sm1,s ,sp1); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp2,sp1,s ); - } else { - val_hi = 0.5 * (s + sp1); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real @@ -353,13 +259,10 @@ struct WENO_Z5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i+3, j , k , qty_index); amrex::Real sp2 = m_phi(i+2, j , k , qty_index); amrex::Real sp1 = m_phi(i+1, j , k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -367,14 +270,6 @@ struct WENO_Z5 amrex::Real sm2 = m_phi(i-2, j , k , qty_index); amrex::Real sm3 = m_phi(i-3, j , k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm3,sm2,sm1,s ,sp1); } else if (upw_lo < -tol) { @@ -391,13 +286,10 @@ struct WENO_Z5 const int& j, const int& k, const int& qty_index, - amrex::Real& val_hi, amrex::Real& val_lo, - amrex::Real upw_hi, amrex::Real upw_lo) const { // Data to interpolate on - amrex::Real sp3 = m_phi(i , j+3, k , qty_index); amrex::Real sp2 = m_phi(i , j+2, k , qty_index); amrex::Real sp1 = m_phi(i , j+1, k , qty_index); amrex::Real s = m_phi(i , j , k , qty_index); @@ -405,14 +297,6 @@ struct WENO_Z5 amrex::Real sm2 = m_phi(i , j-2, k , qty_index); amrex::Real sm3 = m_phi(i , j-3, k , qty_index); - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - if (upw_lo > tol) { val_lo = Evaluate(sm3,sm2,sm1,s ,sp1); } else if (upw_lo < -tol) { @@ -425,7 +309,7 @@ struct WENO_Z5 AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - InterpolateInZ_lo (const int& i, + InterpolateInZ (const int& i, const int& j, const int& k, const int& qty_index, @@ -449,33 +333,6 @@ struct WENO_Z5 } } - AMREX_GPU_DEVICE - AMREX_FORCE_INLINE - void - InterpolateInZ_hi (const int& i, - const int& j, - const int& k, - const int& qty_index, - amrex::Real& val_hi, - amrex::Real upw_hi) const - { - // Data to interpolate on - amrex::Real sp3 = m_phi(i , j , k+3, qty_index); - amrex::Real sp2 = m_phi(i , j , k+2, qty_index); - amrex::Real sp1 = m_phi(i , j , k+1, qty_index); - amrex::Real s = m_phi(i , j , k , qty_index); - amrex::Real sm1 = m_phi(i , j , k-1, qty_index); - amrex::Real sm2 = m_phi(i , j , k-2, qty_index); - - if (upw_hi > tol){ - val_hi = Evaluate(sm2,sm1,s,sp1,sp2); - } else if (upw_hi < -tol) { - val_hi = Evaluate(sp3,sp2,sp1,s ,sm1); - } else { - val_hi = 0.5 * (s + sp1); - } - } - AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real diff --git a/Source/Utils/TerrainMetrics.cpp b/Source/Utils/TerrainMetrics.cpp index 4182a92b5..175ccd2c3 100644 --- a/Source/Utils/TerrainMetrics.cpp +++ b/Source/Utils/TerrainMetrics.cpp @@ -69,7 +69,6 @@ init_zlevels (amrex::Vector& zlevels_stag, void init_terrain_grid (const Geometry& geom, MultiFab& z_phys_nd, amrex::Vector const& z_levels_h) { - auto dx = geom.CellSizeArray(); auto ProbHiArr = geom.ProbHiArray(); // z_nd is nodal in all directions @@ -84,9 +83,6 @@ init_terrain_grid (const Geometry& geom, MultiFab& z_phys_nd, amrex::Vector z_levels_d; z_levels_d.resize(nz); #ifdef AMREX_USE_GPU diff --git a/Source/Utils/Utils.H b/Source/Utils/Utils.H index 6abc45a7d..e154a4946 100644 --- a/Source/Utils/Utils.H +++ b/Source/Utils/Utils.H @@ -68,7 +68,8 @@ void compute_interior_ghost_bxs_xy (const amrex::Box& bx, /* * Compute relaxation region RHS with wrfbdy */ -void wrfbdy_compute_interior_ghost_rhs (const amrex::Real& bdy_time_interval, +void wrfbdy_compute_interior_ghost_rhs (const std::string& init_type, + const amrex::Real& bdy_time_interval, const amrex::Real& start_bdy_time, const amrex::Real& time, const amrex::Real& delta_t, diff --git a/Submodules/AMReX b/Submodules/AMReX index 388738dc7..ae7b64bcf 160000 --- a/Submodules/AMReX +++ b/Submodules/AMReX @@ -1 +1 @@ -Subproject commit 388738dc788cc4a72c33a5d5d9940ef3a37a76b7 +Subproject commit ae7b64bcf6a4ed36dd03e17357cae83b0a394912 diff --git a/Tests/test_files/DensityCurrent_detJ2_MT/DensityCurrent_detJ2_MT.i b/Tests/test_files/DensityCurrent_detJ2_MT/DensityCurrent_detJ2_MT.i index 1382e7ba3..0498ffd33 100644 --- a/Tests/test_files/DensityCurrent_detJ2_MT/DensityCurrent_detJ2_MT.i +++ b/Tests/test_files/DensityCurrent_detJ2_MT/DensityCurrent_detJ2_MT.i @@ -67,6 +67,6 @@ erf.init_shrink = 1.0 # scale back initial timestep erf.terrain_z_levels = 0. 100. 200. 300. 400. 500. 600. 700. 800. 900. 1000. 1100. 1200. 1300. 1400. 1500. 1600. 1700. 1800. 1900. 2000. 2100. 2200. 2300. 2400. 2500. 2600. 2700. 2800. 2900. 3000. 3100. 3200. 3300. 3400. 3500. 3600. 3700. 3800. 3900. 4000. 4100. 4200. 4300. 4400. 4500. 4600. 4700. 4800. 4900. 5000. 5100. 5200. 5300. 5400. 5500. 5600. 5700. 5800. 5900. 6000. 6100. 6200. 6300. 6400. # TERRRAIN GRID TYPE -erf.use_terrain = 1 -erf.terrain_type = 1 +erf.use_terrain = true +erf.terrain_type = Moving erf.terrain_smoothing = 2 // Sullivan 2004 approach diff --git a/Tests/test_files/MovingTerrain_nosub/MovingTerrain_nosub.i b/Tests/test_files/MovingTerrain_nosub/MovingTerrain_nosub.i index 0642c91fb..e100c34f4 100644 --- a/Tests/test_files/MovingTerrain_nosub/MovingTerrain_nosub.i +++ b/Tests/test_files/MovingTerrain_nosub/MovingTerrain_nosub.i @@ -15,8 +15,8 @@ zlo.type = "SlipWall" zhi.type = "SlipWall" # TERRRAIN GRID TYPE -erf.use_terrain = 1 # enable terrain stencils -erf.terrain_type = 1 # moving terrain +erf.use_terrain = true # enable terrain stencils +erf.terrain_type = Moving # moving terrain erf.terrain_smoothing = 2 # Sullivan 2004 approach erf.no_substepping = 1 diff --git a/Tests/test_files/MovingTerrain_sub/MovingTerrain_sub.i b/Tests/test_files/MovingTerrain_sub/MovingTerrain_sub.i index bb9e799c2..9186f4556 100644 --- a/Tests/test_files/MovingTerrain_sub/MovingTerrain_sub.i +++ b/Tests/test_files/MovingTerrain_sub/MovingTerrain_sub.i @@ -15,8 +15,8 @@ zlo.type = "SlipWall" zhi.type = "SlipWall" # TERRRAIN GRID TYPE -erf.use_terrain = 1 # enable terrain stencils -erf.terrain_type = 1 # moving terrain +erf.use_terrain = true # enable terrain stencils +erf.terrain_type = Moving # moving terrain erf.terrain_smoothing = 2 # Sullivan 2004 approach erf.use_lagged_delta_rt = false