From beefed43b853c25ad3751a9aa63746e57a03496a Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Tue, 30 Aug 2022 11:18:24 -0700 Subject: [PATCH 01/48] fix variable mismatch --- fv3core/fv3core/stencils/dyn_core.py | 4 ++-- pace-util/pace/util/checkpointer/validation.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fv3core/fv3core/stencils/dyn_core.py b/fv3core/fv3core/stencils/dyn_core.py index dd87caa6e..41f25b284 100644 --- a/fv3core/fv3core/stencils/dyn_core.py +++ b/fv3core/fv3core/stencils/dyn_core.py @@ -649,7 +649,7 @@ def _checkpoint_dsw_in(self, state): ucd=state.uc, vcd=state.vc, wd=state.w, - delpcd=self.delpc, + delpcd=state.vt, delpd=state.delp, ud=state.u, vd=state.v, @@ -671,7 +671,7 @@ def _checkpoint_dsw_out(self, state): ucd=state.uc, vcd=state.vc, wd=state.w, - delpcd=self.delpc, + delpcd=state.vt, delpd=state.delp, ud=state.u, vd=state.v, diff --git a/pace-util/pace/util/checkpointer/validation.py b/pace-util/pace/util/checkpointer/validation.py index 34059dda5..9260f81b1 100644 --- a/pace-util/pace/util/checkpointer/validation.py +++ b/pace-util/pace/util/checkpointer/validation.py @@ -117,10 +117,14 @@ def __call__(self, savepoint_name: str, **kwargs: ArrayLike) -> None: raise ValueError(f"argument {varname} not in netCDF file {nc_file}") expected = ds[varname][n_calls, self._rank].values + expected[expected == 1.0e40] = 0.0 output = _clip_pace_array_to_target(cast_to_ndarray(array), expected.shape) - + if savepoint_name == "FVDynamics-Out": + expected = expected[3:-3, 3:-3, :] + output = output[3:-3, 3:-3, :] # cannot use relative threshold when comparing to zero value - expected_not_zero = expected != 0 + near_zero = 1.0e-15 + expected_not_zero = abs(expected) > near_zero np.testing.assert_allclose( output[expected_not_zero], expected[expected_not_zero], From 7ad0486fd7b7120e64e138ece39fc6d40bb06a35 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Thu, 1 Sep 2022 14:43:57 -0700 Subject: [PATCH 02/48] reinitialize dycore to reset temp storages --- .../pace/util/checkpointer/validation.py | 4 +- tests/mpi/test_checkpoints.py | 15 +- tests/mpi/thresholds/fv_dynamics.yaml | 156 +++++++++--------- 3 files changed, 93 insertions(+), 82 deletions(-) diff --git a/pace-util/pace/util/checkpointer/validation.py b/pace-util/pace/util/checkpointer/validation.py index 9260f81b1..958127bed 100644 --- a/pace-util/pace/util/checkpointer/validation.py +++ b/pace-util/pace/util/checkpointer/validation.py @@ -122,9 +122,7 @@ def __call__(self, savepoint_name: str, **kwargs: ArrayLike) -> None: if savepoint_name == "FVDynamics-Out": expected = expected[3:-3, 3:-3, :] output = output[3:-3, 3:-3, :] - # cannot use relative threshold when comparing to zero value - near_zero = 1.0e-15 - expected_not_zero = abs(expected) > near_zero + expected_not_zero = expected != 0 np.testing.assert_allclose( output[expected_not_zero], expected[expected_not_zero], diff --git a/tests/mpi/test_checkpoints.py b/tests/mpi/test_checkpoints.py index af306b1d6..c7a2390a9 100644 --- a/tests/mpi/test_checkpoints.py +++ b/tests/mpi/test_checkpoints.py @@ -48,9 +48,14 @@ def __init__( self._translate = translate self._consv_te = consv_te self._n_split = n_split + self._communicator = communicator + self._stencil_factory = stencil_factory + self._grid = grid + self._dycore_config = dycore_config input_data = dataset_to_dict(self._ds.copy()) state, grid_data = self._translate.prepare_data(input_data) + self._grid_data = grid_data self._dycore = fv3core.DynamicalCore( comm=communicator, grid_data=grid_data, @@ -64,7 +69,15 @@ def __init__( def new_state(self) -> Tuple[DycoreState, GridData]: input_data = dataset_to_dict(self._ds.copy()) state, grid_data = self._translate.prepare_data(input_data) - + self._dycore = fv3core.DynamicalCore( + comm=self._communicator, + grid_data=grid_data, + stencil_factory=self._stencil_factory, + damping_coefficients=self._grid.damping_coefficients, + config=self._dycore_config, + phis=state.phis, + state=state, + ) self._dycore.update_state( self._consv_te, input_data["do_adiabatic_init"], diff --git a/tests/mpi/thresholds/fv_dynamics.yaml b/tests/mpi/thresholds/fv_dynamics.yaml index 87073e0d1..6c51bf153 100644 --- a/tests/mpi/thresholds/fv_dynamics.yaml +++ b/tests/mpi/thresholds/fv_dynamics.yaml @@ -4,85 +4,85 @@ savepoints: absolute: 2.2737367544323206e-12 relative: 2.2200743449158477e-15 divgdd: - absolute: 0.0004160007342990252 - relative: 11.111111112365606 + absolute: 0.0 + relative: 0.0 ptd: absolute: 2.8421709430404007e-13 - relative: 2.005319252332659e-14 + relative: 1.851063925230147e-14 uad: absolute: 640.0 relative: 2.2204251720152353e-15 ucd: absolute: 1.4210854715202004e-13 - relative: 2.2204312519237058e-15 + relative: 2.2204312519237066e-15 ud: absolute: 1.4210854715202004e-13 relative: 2.2204431850207167e-15 utd: - absolute: 57514742812.51459 - relative: 11.111111111138278 + absolute: 0.0 + relative: 0.0 vad: absolute: 640.0 relative: 2.2204439912402017e-15 vcd: absolute: 1.4210854715202004e-13 - relative: 2.2204224462363224e-15 + relative: 2.220444857175165e-15 vd: absolute: 1.4210854715202004e-13 relative: 2.2204380358035745e-15 vtd: - absolute: 55238702279.38376 - relative: 11.111111111317662 + absolute: 0.0 + relative: 0.0 wd: absolute: 2.7755575615628914e-16 - relative: 2.220430475548686e-15 + relative: 2.2204304755486847e-15 C_SW-Out: - delpd: absolute: 2.2737367544323206e-12 relative: 2.2200743449158477e-15 divgdd: - absolute: 0.0004160007342990252 - relative: 11.11111111133647 + absolute: 1.0164395367051604e-18 + relative: 1.0116858185720617e-09 ptd: absolute: 2.8421709430404007e-13 - relative: 2.005319252332659e-14 + relative: 1.851063925230147e-14 uad: absolute: 1280.0 relative: 4.4531842264012817e-11 ucd: absolute: 4.263256414560601e-13 - relative: 7.166103136574708e-11 + relative: 7.106894435212558e-11 ud: absolute: 1.4210854715202004e-13 relative: 2.2204431850207167e-15 utd: absolute: 3.814697265625e-05 - relative: 4.397821006895427e-11 + relative: 2.5303564622294458e-11 vad: absolute: 1280.0 - relative: 5.04598246392987e-10 + relative: 3.801762325990083e-10 vcd: absolute: 4.263256414560601e-13 - relative: 2.7376954078854966e-11 + relative: 3.969792148639851e-11 vd: absolute: 1.4210854715202004e-13 relative: 2.2204380358035745e-15 vtd: - absolute: 4.76837158203125e-05 - relative: 1.5043808933067975e-10 + absolute: 5.7220458984375e-05 + relative: 1.4144975022713432e-10 wd: absolute: 2.7755575615628914e-16 - relative: 2.220430475548686e-15 + relative: 2.2204304755486847e-15 D_SW-In: - delpcd: - absolute: 3.9335645851679146e-10 - relative: 2.860199948824563e-13 + absolute: 5.7220458984375e-05 + relative: 1.4144975022713432e-10 delpd: absolute: 2.2737367544323206e-12 relative: 2.2200743449158477e-15 divgdd: - absolute: 0.0003315660431181209 - relative: 11.11111111124784 + absolute: 1.0164395367051604e-18 + relative: 1.0116858185720617e-09 mfxd: absolute: 0.0 relative: 0.0 @@ -91,83 +91,83 @@ savepoints: relative: 0.0 ptd: absolute: 2.8421709430404007e-13 - relative: 2.005319252332659e-14 + relative: 1.851063925230147e-14 uad: absolute: 1280.0 relative: 4.4531842264012817e-11 ucd: - absolute: 7.460698725481052e-12 - relative: 2.1928151258000319e-10 + absolute: 9.592326932761353e-13 + relative: 4.014241461643457e-10 ud: absolute: 1.4210854715202004e-13 relative: 2.2204431850207167e-15 vad: absolute: 1280.0 - relative: 5.04598246392987e-10 + relative: 3.801762325990083e-10 vcd: - absolute: 7.815970093361102e-12 - relative: 6.114797082703121e-10 + absolute: 1.0658141036401503e-12 + relative: 9.678516401601087e-10 vd: absolute: 1.4210854715202004e-13 relative: 2.2204380358035745e-15 wd: absolute: 2.7755575615628914e-16 - relative: 2.220430475548686e-15 + relative: 2.2204304755486847e-15 xfxd: - absolute: 135763846936.6738 - relative: 11.111111112205553 + absolute: 0.0 + relative: 0.0 yfxd: - absolute: 204618169353.22342 - relative: 11.11111111132302 + absolute: 0.0 + relative: 0.0 zhd: - absolute: 3926484.407087692 - relative: 11.111111125462843 + absolute: 2.1827872842550278e-10 + relative: 7.04593126878164e-15 D_SW-Out: - delpcd: - absolute: 3.9335645851679146e-10 - relative: 2.860199948824563e-13 + absolute: 2.86102294921875e-05 + relative: 1.0116858185720617e-09 delpd: - absolute: 0.0006273110250276659 - relative: 1.587713695875141e-06 + absolute: 0.0006292948035024892 + relative: 1.6050064191530135e-06 divgdd: - absolute: 6.945670167485263e-19 - relative: 2.0322784771986075e-09 + absolute: 5.590417451878382e-19 + relative: 4.4508165492170205e-09 mfxd: - absolute: 2.3496618687725388e+24 - relative: 16.363636372143787 + absolute: 2.3496618687725375e+24 + relative: 16.3636363730611 mfyd: - absolute: 1.2686334171435037e+24 - relative: 16.36363636398578 + absolute: 1.2686334171435062e+24 + relative: 16.36363636579146 ptd: - absolute: 5.858738205688496e-07 - relative: 1.5276859450659092e-08 + absolute: 5.919210366300831e-07 + relative: 1.543661294125962e-08 uad: absolute: 1280.0 relative: 4.4531842264012817e-11 ucd: - absolute: 7.140954494389007e-12 - relative: 2.2823124124508078e-07 + absolute: 7.105427357601002e-13 + relative: 2.335683747836233e-09 ud: absolute: 2.2351741790771484e-07 - relative: 1.894388781548416e-11 + relative: 1.773082587559174e-11 vad: absolute: 1280.0 - relative: 5.04598246392987e-10 + relative: 3.801762325990083e-10 vcd: - absolute: 7.815970093361102e-12 - relative: 5.733542033757981e-09 + absolute: 1.0658141036401503e-12 + relative: 1.5479001178967714e-09 vd: - absolute: 4.470348358154297e-07 - relative: 3.550699530922532e-11 + absolute: 2.980232238769531e-07 + relative: 1.1344899282993951e-10 wd: - absolute: 3.8847127667440695e-08 - relative: 6.088159060695046e-05 + absolute: 3.881427367014023e-08 + relative: 8.033914185806085e-05 xfxd: - absolute: 0.00110626220703125 - relative: 1.7732558727224426e-09 + absolute: 0.0001537799835205078 + relative: 1.7732558727067202e-09 yfxd: - absolute: 0.0011491775512695312 - relative: 4.3965869300337324e-10 + absolute: 0.000171661376953125 + relative: 8.255143046330278e-10 FVDynamics-In: - delz: absolute: 0.0 @@ -198,29 +198,29 @@ savepoints: relative: 0.0 FVDynamics-Out: - delz: - absolute: 0.0011694489012370468 - relative: 3.8794247824018573e-07 + absolute: 0.001160447809525067 + relative: 3.861710180457495e-07 qvapor: absolute: 3.469446951953614e-17 - relative: 2.220439941711151e-15 + relative: 2.2204399417111508e-15 u: - absolute: 1.3032365657750233e-05 - relative: 0.00027512090893756514 + absolute: 1.5095681362709001e-05 + relative: 4.723330275579375e-05 ua: absolute: 1280.0 - relative: 0.0004849295481220942 + relative: 8.068373919387091e-05 uc: - absolute: 7.140954494389007e-12 - relative: 2.2823124124508078e-07 + absolute: 7.105427357601002e-13 + relative: 2.335683747836233e-09 v: - absolute: 1.4660203575544983e-05 - relative: 0.0006626038916865451 + absolute: 8.262938830938538e-06 + relative: 0.00632014380512243 va: absolute: 1280.0 - relative: 0.0011382692916765906 + relative: 0.00016662659576558862 vc: - absolute: 7.815970093361102e-12 - relative: 5.733542033757981e-09 + absolute: 1.0658141036401503e-12 + relative: 1.5479001178967714e-09 w: - absolute: 1.6963235226485907e-05 - relative: 0.3089630716526554 + absolute: 1.7342969022282606e-05 + relative: 0.025006993840671343 From 6788d770f7e3fb555fd2ba4b581f4278b9eefd12 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 9 Sep 2022 11:14:13 -0700 Subject: [PATCH 03/48] fix var name --- fv3core/pace/fv3core/stencils/dyn_core.py | 4 ++-- tests/mpi/test_checkpoints.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fv3core/pace/fv3core/stencils/dyn_core.py b/fv3core/pace/fv3core/stencils/dyn_core.py index 2893acca9..9bfadfedb 100644 --- a/fv3core/pace/fv3core/stencils/dyn_core.py +++ b/fv3core/pace/fv3core/stencils/dyn_core.py @@ -691,7 +691,7 @@ def _checkpoint_dsw_in(self, state: DycoreState): ucd=state.uc, vcd=state.vc, wd=state.w, - delpcd=state.vt, + delpcd=self._vt, delpd=state.delp, ud=state.u, vd=state.v, @@ -713,7 +713,7 @@ def _checkpoint_dsw_out(self, state: DycoreState): ucd=state.uc, vcd=state.vc, wd=state.w, - delpcd=state.vt, + delpcd=self._vt, delpd=state.delp, ud=state.u, vd=state.v, diff --git a/tests/mpi/test_checkpoints.py b/tests/mpi/test_checkpoints.py index 7d85cee47..75e2dd494 100644 --- a/tests/mpi/test_checkpoints.py +++ b/tests/mpi/test_checkpoints.py @@ -1,5 +1,6 @@ import dataclasses import os +from datetime import timedelta from typing import List, Tuple import dacite @@ -137,6 +138,7 @@ def test_fv_dynamics( phis=state.phis, state=state, checkpointer=validation, + timestep=timedelta(seconds=dycore_config.dt_atmos), ) with validation.trial(): dycore.step_dynamics(state) @@ -162,6 +164,7 @@ def _calibrate_thresholds( phis=state.phis, state=state, checkpointer=calibration, + timestep=timedelta(seconds=dycore_config.dt_atmos), ) for i in range(n_trials): print(f"running calibration trial {i}") From 70c044ee21c93bf8af2b3ee56e3208e1da94d8df Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 9 Sep 2022 16:08:07 -0700 Subject: [PATCH 04/48] fvdynamics-out tracers in compute domain only --- fv3core/pace/fv3core/testing/translate_fvdynamics.py | 2 ++ stencils/pace/stencils/testing/test_translate.py | 6 +++++- util/pace/util/checkpointer/validation.py | 4 ---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/fv3core/pace/fv3core/testing/translate_fvdynamics.py b/fv3core/pace/fv3core/testing/translate_fvdynamics.py index 0d1561e8f..fc62458e9 100644 --- a/fv3core/pace/fv3core/testing/translate_fvdynamics.py +++ b/fv3core/pace/fv3core/testing/translate_fvdynamics.py @@ -282,6 +282,8 @@ def __init__( self._base.out_vars.update(fv_dynamics_vars) self._base.out_vars["ps"] = {"kstart": grid.npz - 1, "kend": grid.npz - 1} self._base.out_vars["phis"] = {"kstart": grid.npz - 1, "kend": grid.npz - 1} + self._do_not_subset_tracers = True + self._tracer_names = ADVECTED_TRACER_NAMES self.max_error = 1e-5 diff --git a/stencils/pace/stencils/testing/test_translate.py b/stencils/pace/stencils/testing/test_translate.py index 3e9899e4c..93e7d8e93 100644 --- a/stencils/pace/stencils/testing/test_translate.py +++ b/stencils/pace/stencils/testing/test_translate.py @@ -378,7 +378,11 @@ def test_parallel_savepoint( ref_data[varname] = [] new_ref_data = all_ref_data[varname] if hasattr(case.testobj, "subset_output"): - new_ref_data = case.testobj.subset_output(varname, new_ref_data) + if hasattr(case.testobj, "_do_not_subset_tracers"): + if varname in case.testobj._tracer_names: + pass + else: + new_ref_data = case.testobj.subset_output(varname, new_ref_data) ref_data[varname].append(new_ref_data) ignore_near_zero = case.testobj.ignore_near_zero_errors.get(varname, False) with subtests.test(varname=varname): diff --git a/util/pace/util/checkpointer/validation.py b/util/pace/util/checkpointer/validation.py index 958127bed..469580e88 100644 --- a/util/pace/util/checkpointer/validation.py +++ b/util/pace/util/checkpointer/validation.py @@ -117,11 +117,7 @@ def __call__(self, savepoint_name: str, **kwargs: ArrayLike) -> None: raise ValueError(f"argument {varname} not in netCDF file {nc_file}") expected = ds[varname][n_calls, self._rank].values - expected[expected == 1.0e40] = 0.0 output = _clip_pace_array_to_target(cast_to_ndarray(array), expected.shape) - if savepoint_name == "FVDynamics-Out": - expected = expected[3:-3, 3:-3, :] - output = output[3:-3, 3:-3, :] expected_not_zero = expected != 0 np.testing.assert_allclose( output[expected_not_zero], From deef15c1e8607c21ffbfbecb9ce04576e148bdef Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 9 Sep 2022 16:08:41 -0700 Subject: [PATCH 05/48] update data version --- Makefile.data_download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.data_download b/Makefile.data_download index 5415ee549..1fb48928c 100644 --- a/Makefile.data_download +++ b/Makefile.data_download @@ -2,7 +2,7 @@ REGRESSION_DATA_STORAGE_BUCKET = gs://vcm-fv3gfs-serialized-regression-data ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) EXPERIMENT ?=c12_6ranks_standard -FORTRAN_SERIALIZED_DATA_VERSION=8.1.1 +FORTRAN_SERIALIZED_DATA_VERSION=8.1.2 TARGET ?=dycore TEST_DATA_ROOT ?=$(ROOT_DIR)/test_data/ TEST_DATA_HOST ?=$(TEST_DATA_ROOT)/$(FORTRAN_SERIALIZED_DATA_VERSION)/$(EXPERIMENT)/$(TARGET) From 2c0fbdc9635eaed386dfc821d4b03325cb2ba495 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Mon, 12 Sep 2022 10:08:03 -0700 Subject: [PATCH 06/48] revert comment --- util/pace/util/checkpointer/validation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/pace/util/checkpointer/validation.py b/util/pace/util/checkpointer/validation.py index 469580e88..34059dda5 100644 --- a/util/pace/util/checkpointer/validation.py +++ b/util/pace/util/checkpointer/validation.py @@ -118,6 +118,8 @@ def __call__(self, savepoint_name: str, **kwargs: ArrayLike) -> None: expected = ds[varname][n_calls, self._rank].values output = _clip_pace_array_to_target(cast_to_ndarray(array), expected.shape) + + # cannot use relative threshold when comparing to zero value expected_not_zero = expected != 0 np.testing.assert_allclose( output[expected_not_zero], From f91c22e1eb3a51f9d98e54a9f914cb02c1bfcf92 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Mon, 12 Sep 2022 12:02:36 -0700 Subject: [PATCH 07/48] fix driver subset --- physics/tests/savepoint/translate/translate_driver.py | 1 + stencils/pace/stencils/testing/test_translate.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/physics/tests/savepoint/translate/translate_driver.py b/physics/tests/savepoint/translate/translate_driver.py index 5351956d0..922ffd3c7 100644 --- a/physics/tests/savepoint/translate/translate_driver.py +++ b/physics/tests/savepoint/translate/translate_driver.py @@ -21,6 +21,7 @@ def __init__(self, grid, namelist, stencil_factory): self.namelist: Namelist = namelist self.stencil_factory = stencil_factory self.stencil_config = self.stencil_factory.config + self._do_not_subset_tracers = False def compute_parallel(self, inputs, communicator): dycore_state = self.state_from_inputs(inputs) diff --git a/stencils/pace/stencils/testing/test_translate.py b/stencils/pace/stencils/testing/test_translate.py index 93e7d8e93..75a835d33 100644 --- a/stencils/pace/stencils/testing/test_translate.py +++ b/stencils/pace/stencils/testing/test_translate.py @@ -378,7 +378,10 @@ def test_parallel_savepoint( ref_data[varname] = [] new_ref_data = all_ref_data[varname] if hasattr(case.testobj, "subset_output"): - if hasattr(case.testobj, "_do_not_subset_tracers"): + do_not_subset_tracers = getattr( + case.testobj, "_do_not_subset_tracers", None + ) + if do_not_subset_tracers: if varname in case.testobj._tracer_names: pass else: From 534a887fd709905e12a506e04b0f70f6bb0dcd44 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Tue, 13 Sep 2022 11:35:45 -0700 Subject: [PATCH 08/48] clean up translate test --- .../fv3core/testing/translate_fvdynamics.py | 62 +++---------------- .../savepoint/translate/translate_driver.py | 3 - .../pace/stencils/testing/test_translate.py | 9 +-- 3 files changed, 9 insertions(+), 65 deletions(-) diff --git a/fv3core/pace/fv3core/testing/translate_fvdynamics.py b/fv3core/pace/fv3core/testing/translate_fvdynamics.py index fc62458e9..918c9fc2c 100644 --- a/fv3core/pace/fv3core/testing/translate_fvdynamics.py +++ b/fv3core/pace/fv3core/testing/translate_fvdynamics.py @@ -15,9 +15,6 @@ from pace.util.grid import GridData -ADVECTED_TRACER_NAMES = utils.tracer_variables[: fv_dynamics.NQ] - - class TranslateDycoreFortranData2Py(TranslateFortranData2Py): def __init__( self, @@ -234,14 +231,14 @@ def __init__( "v": grid.x3d_domain_dict(), "w": {}, "delz": {}, - "qvapor": {}, - "qliquid": {}, - "qice": {}, - "qrain": {}, - "qsnow": {}, - "qgraupel": {}, - "qo3mr": {}, - "qsgs_tke": {}, + "qvapor": grid.compute_dict(), + "qliquid": grid.compute_dict(), + "qice": grid.compute_dict(), + "qrain": grid.compute_dict(), + "qsnow": grid.compute_dict(), + "qgraupel": grid.compute_dict(), + "qo3mr": grid.compute_dict(), + "qsgs_tke": grid.compute_dict(), "qcld": {}, "ps": {}, "pe": { @@ -282,8 +279,6 @@ def __init__( self._base.out_vars.update(fv_dynamics_vars) self._base.out_vars["ps"] = {"kstart": grid.npz - 1, "kend": grid.npz - 1} self._base.out_vars["phis"] = {"kstart": grid.npz - 1, "kend": grid.npz - 1} - self._do_not_subset_tracers = True - self._tracer_names = ADVECTED_TRACER_NAMES self.max_error = 1e-5 @@ -342,8 +337,6 @@ def compute_parallel(self, inputs, communicator): ) self.dycore.step_dynamics(state, pace.util.NullTimer()) outputs = self.outputs_from_state(state) - for name, value in outputs.items(): - outputs[name] = self.subset_output(name, value) return outputs def outputs_from_state(self, state: dict): @@ -366,42 +359,3 @@ def compute_sequential(self, *args, **kwargs): f"{self.__class__} only has a mpirun implementation, " "not running in mock-parallel" ) - - def subset_output(self, varname: str, output): - """ - Given an output array, return the slice of the array which we'd - like to validate against reference data - """ - if self.dycore is None: - raise RuntimeError( - "cannot call subset_output before calling compute_parallel " - "to initialize dycore" - ) - if hasattr(self.dycore, "selective_names") and ( - varname in self.dycore.selective_names # type: ignore - ): - return_value = self.dycore.subset_output(varname, output) # type: ignore - - if varname in ADVECTED_TRACER_NAMES: - - def get_compute_domain_k_interfaces( - instance, - ): - try: - origin = instance.grid_indexing.origin_compute() - domain = instance.grid_indexing.domain_compute(add=(0, 0, 1)) - except AttributeError: - origin = instance.grid.compute_origin() - domain = instance.grid.domain_shape_compute(add=(0, 0, 1)) - return origin, domain - - origin, domain = get_compute_domain_k_interfaces( - self.dycore.tracer_advection - ) - self._validation_slice = tuple( - slice(start, start + n) for start, n in zip(origin, domain) - ) - return_value = output[self._validation_slice] - else: - return_value = output - return return_value diff --git a/physics/tests/savepoint/translate/translate_driver.py b/physics/tests/savepoint/translate/translate_driver.py index 922ffd3c7..46f9dced8 100644 --- a/physics/tests/savepoint/translate/translate_driver.py +++ b/physics/tests/savepoint/translate/translate_driver.py @@ -21,7 +21,6 @@ def __init__(self, grid, namelist, stencil_factory): self.namelist: Namelist = namelist self.stencil_factory = stencil_factory self.stencil_config = self.stencil_factory.config - self._do_not_subset_tracers = False def compute_parallel(self, inputs, communicator): dycore_state = self.state_from_inputs(inputs) @@ -76,6 +75,4 @@ def compute_parallel(self, inputs, communicator): self.dycore = driver.dycore outputs = self.outputs_from_state(driver.state.dycore_state) - for name, value in outputs.items(): - outputs[name] = self.subset_output(name, value) return outputs diff --git a/stencils/pace/stencils/testing/test_translate.py b/stencils/pace/stencils/testing/test_translate.py index 75a835d33..3e9899e4c 100644 --- a/stencils/pace/stencils/testing/test_translate.py +++ b/stencils/pace/stencils/testing/test_translate.py @@ -378,14 +378,7 @@ def test_parallel_savepoint( ref_data[varname] = [] new_ref_data = all_ref_data[varname] if hasattr(case.testobj, "subset_output"): - do_not_subset_tracers = getattr( - case.testobj, "_do_not_subset_tracers", None - ) - if do_not_subset_tracers: - if varname in case.testobj._tracer_names: - pass - else: - new_ref_data = case.testobj.subset_output(varname, new_ref_data) + new_ref_data = case.testobj.subset_output(varname, new_ref_data) ref_data[varname].append(new_ref_data) ignore_near_zero = case.testobj.ignore_near_zero_errors.get(varname, False) with subtests.test(varname=varname): From 9154218f6caaa296b479fb133c15ea3b03dc5f3c Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Wed, 14 Sep 2022 14:21:52 -0700 Subject: [PATCH 09/48] add comment --- fv3core/pace/fv3core/stencils/dyn_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fv3core/pace/fv3core/stencils/dyn_core.py b/fv3core/pace/fv3core/stencils/dyn_core.py index 9bfadfedb..eb5c21f42 100644 --- a/fv3core/pace/fv3core/stencils/dyn_core.py +++ b/fv3core/pace/fv3core/stencils/dyn_core.py @@ -691,6 +691,7 @@ def _checkpoint_dsw_in(self, state: DycoreState): ucd=state.uc, vcd=state.vc, wd=state.w, + # delpc is a temporary and not a variable in D_SW savepoint delpcd=self._vt, delpd=state.delp, ud=state.u, From 3a339765dc309bd302412cd4f1ed2b1c4f0c7a7e Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Thu, 15 Sep 2022 10:16:40 -0700 Subject: [PATCH 10/48] Bump to 8.1.3 --- Makefile.data_download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.data_download b/Makefile.data_download index 1fb48928c..ec5198662 100644 --- a/Makefile.data_download +++ b/Makefile.data_download @@ -2,7 +2,7 @@ REGRESSION_DATA_STORAGE_BUCKET = gs://vcm-fv3gfs-serialized-regression-data ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) EXPERIMENT ?=c12_6ranks_standard -FORTRAN_SERIALIZED_DATA_VERSION=8.1.2 +FORTRAN_SERIALIZED_DATA_VERSION=8.1.3 TARGET ?=dycore TEST_DATA_ROOT ?=$(ROOT_DIR)/test_data/ TEST_DATA_HOST ?=$(TEST_DATA_ROOT)/$(FORTRAN_SERIALIZED_DATA_VERSION)/$(EXPERIMENT)/$(TARGET) From cd8907b4b7020885f9eb4c04379f26bda2189ca8 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 13:26:49 -0700 Subject: [PATCH 11/48] add physics and driver savepoint tests --- .circleci/config.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 660eabe27..6f565fe9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,10 +97,14 @@ jobs: command: | BUILD_ARGS="--progress=plain" DEV=n make build - run: - name: run tests + name: run dycore tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests no_output_timeout: 3h + - run: + name: run physics tests + command: | + TEST_ARGS="--backend=<> -v -s" DEV=y make physics_savepoint_tests - save_cache: key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} paths: @@ -151,10 +155,20 @@ jobs: command: | BUILD_ARGS="--progress=plain" DEV=n make build - run: - name: run tests + name: run dycore tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests_mpi no_output_timeout: 3h + - run: + name: run physics tests + command: | + TEST_ARGS="--backend=<> -v -s" DEV=y make physics_savepoint_tests_mpi + no_output_timeout: 3h + - run: + name: run driver tests + command: | + TEST_ARGS="--backend=<> -v -s" DEV=y make driver_savepoint_tests_mpi + no_output_timeout: 3h - save_cache: key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} paths: @@ -227,10 +241,20 @@ jobs: paths: - /home/circleci/.cache/pip - run: - name: run tests + name: run dycore tests command: | GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_standard" FV3_PATH=$(pwd)/fv3core make savepoint_tests_mpi no_output_timeout: 3h + - run: + name: run physics tests + command: | + GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make physics_savepoint_tests_mpi + no_output_timeout: 3h + - run: + name: run driver tests + command: | + GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make driver_savepoint_tests_mpi + no_output_timeout: 3h - save_cache: key: v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} paths: From 36573acbc60ac41e94baeb5d165695b16b2fe17d Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 13:46:21 -0700 Subject: [PATCH 12/48] specify experiement name --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f565fe9f..e24a24117 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,7 +104,7 @@ jobs: - run: name: run physics tests command: | - TEST_ARGS="--backend=<> -v -s" DEV=y make physics_savepoint_tests + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests - save_cache: key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} paths: @@ -162,12 +162,12 @@ jobs: - run: name: run physics tests command: | - TEST_ARGS="--backend=<> -v -s" DEV=y make physics_savepoint_tests_mpi + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests_mpi no_output_timeout: 3h - run: name: run driver tests command: | - TEST_ARGS="--backend=<> -v -s" DEV=y make driver_savepoint_tests_mpi + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make driver_savepoint_tests_mpi no_output_timeout: 3h - save_cache: key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} From e0beac89251057a1403dcbe818212cfa3bd8c89a Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 14:05:11 -0700 Subject: [PATCH 13/48] add python version to caching --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e24a24117..f8ed85d80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,9 +17,12 @@ commands: libboost-all-dev \ python3 \ python3-pip \ + - run: + name: save python_version.txt + command: python --version | awk '{print $2;}' > python_version.txt - restore_cache: keys: - - v1-venv-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }} + - v1-venv-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }}-{{ checksum "python_version.txt" }} - run: name: setup python environment command: | @@ -29,7 +32,7 @@ commands: pip3 install -r requirements_dev.txt -c constraints.txt python3 -m gt4py.gt_src_manager install - save_cache: - key: v1-venv-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }} + key: v1-venv-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }}-{{ checksum "python_version.txt" }} paths: - venv - /root/.cache/pip From 7dbb1b9bad3edc5bba69b4baa78f0b30f8f8c6c3 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 14:53:33 -0700 Subject: [PATCH 14/48] separate test, increase threshold for circleci --- .circleci/config.yml | 327 +++++++++++------- .../translate/overrides/baroclinic.yaml | 4 +- 2 files changed, 203 insertions(+), 128 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8ed85d80..fe01ee5fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,105 @@ commands: paths: - venv - /root/.cache/pip - + setup_savepoints: + description: "Setup savepoints" + steps: + - gcp-cli/install: + version: 323.0.0 + - run: + name: gcloud auth + command: | + echo $ENCODED_GCR_KEY + echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS + gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + gcloud auth configure-docker + - checkout + - run: + name: Update Submodules + command: git submodule update --init + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt + - run: + name: build image + command: | + BUILD_ARGS="--progress=plain" DEV=n make build + setup_54ranks_mpi: + description: "Install libaries for 54 ranks run" + steps: + - run: | + sudo apt-get update && sudo apt-get install -y make \ + software-properties-common \ + libopenmpi3 \ + libopenmpi-dev \ + libboost-all-dev \ + python3 \ + python3-pip \ + - run: + name: set up environment + command: | + pip3 install --upgrade setuptools wheel + pip3 install -r requirements_dev.txt -c constraints.txt + python3 -m gt4py.gt_src_manager install + restore_cache_savepoints: + description: "Restore cache for serial savepoints" + steps: + - restore_cache: + keys: + - v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} + - restore_cache: + keys: + - v1-savepoints-{{ checksum "Makefile.data_download" }} + save_cache_savepoints: + description: "Save cache for serial savepoints" + steps: + - save_cache: + key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} + paths: + - .gt_cache + - .gt_cache_000000 + - save_cache: + key: v1-savepoints-{{ checksum "Makefile.data_download" }} + paths: + - test_data + restore_cache_savepoints_mpi: + description: "Restore cache for mpi savepoints" + steps: + - restore_cache: + keys: + - v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} + - restore_cache: + keys: + - v1-savepoints-{{ checksum "Makefile.data_download" }} + save_cache_savepoints_mpi: + description: "Save cache for mpi savepoints" + steps: + - save_cache: + key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} + paths: + - .gt_cache + - .gt_cache_000000 + - .gt_cache_000001 + - .gt_cache_000002 + - .gt_cache_000003 + - .gt_cache_000004 + - .gt_cache_000005 + - save_cache: + key: v1-savepoints-{{ checksum "Makefile.data_download" }} + paths: + - test_data + restore_cache_54ranks_savepoints_mpi: + save_cache_54ranks_savepoints_mpi: + description: "Save cache for 54 ranks mpi savepoints" + steps: + - save_cache: + key: v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} + paths: + - .gt_cache + - save_cache: + key: v2-savepoints-c12_54ranks_standard-{{ checksum "Makefile.data_download" }} + paths: + - test_data jobs: lint: @@ -60,7 +158,7 @@ jobs: - ~/.cache/pre-commit - venv - savepoints: + dycore_savepoints: machine: image: ubuntu-2004:202111-02 resource_class: large @@ -73,52 +171,19 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - gcp-cli/install: - version: 323.0.0 - - run: - name: gcloud auth - command: | - echo $ENCODED_GCR_KEY - echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS - gcloud auth configure-docker - - checkout - - run: - name: Update Submodules - command: git submodule update --init + - setup_savepoints - run: name: save gt4py_version.txt command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache: - keys: - - v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} - - restore_cache: - keys: - - v1-savepoints-{{ checksum "Makefile.data_download" }} - - run: - name: build image - command: | - BUILD_ARGS="--progress=plain" DEV=n make build + - restore_cache_savepoints - run: name: run dycore tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests no_output_timeout: 3h - - run: - name: run physics tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests - - save_cache: - key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} - paths: - - .gt_cache - - .gt_cache_000000 - - save_cache: - key: v1-savepoints-{{ checksum "Makefile.data_download" }} - paths: - - test_data + - save_cache_savepoints - savepoints_mpi: + physics_savepoints: machine: image: ubuntu-2004:202111-02 resource_class: large @@ -131,63 +196,84 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - gcp-cli/install: - version: 323.0.0 - - run: - name: gcloud auth - command: | - echo $ENCODED_GCR_KEY - echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS - gcloud auth configure-docker - - checkout - - run: - name: Update Submodules - command: git submodule update --init + - setup_savepoints - run: name: save gt4py_version.txt command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache: - keys: - - v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} - - restore_cache: - keys: - - v1-savepoints-{{ checksum "Makefile.data_download" }} + - restore_cache_savepoints - run: - name: build image + name: run physics tests command: | - BUILD_ARGS="--progress=plain" DEV=n make build + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests + - save_cache_savepoints + + dycore_savepoints_mpi: + machine: + image: ubuntu-2004:202111-02 + resource_class: large + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - setup_savepoints + - restore_cache_savepoints_mpi - run: name: run dycore tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests_mpi no_output_timeout: 3h + - save_cache_savepoints_mpi + + physics_savepoints_mpi: + machine: + image: ubuntu-2004:202111-02 + resource_class: large + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - setup_savepoints + - restore_cache_savepoints_mpi - run: name: run physics tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests_mpi no_output_timeout: 3h + - save_cache_savepoints_mpi + + driver_savepoints_mpi: + machine: + image: ubuntu-2004:202111-02 + resource_class: large + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - setup_savepoints + - restore_cache_savepoints_mpi - run: name: run driver tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make driver_savepoint_tests_mpi no_output_timeout: 3h - - save_cache: - key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} - paths: - - .gt_cache - - .gt_cache_000000 - - .gt_cache_000001 - - .gt_cache_000002 - - .gt_cache_000003 - - .gt_cache_000004 - - .gt_cache_000005 - - save_cache: - key: v1-savepoints-{{ checksum "Makefile.data_download" }} - paths: - - test_data + - save_cache_savepoints_mpi - savepoints_mpi_54rank: + dycore_savepoints_mpi_54rank: machine: image: ubuntu-2004:202111-02 resource_class: xlarge @@ -200,72 +286,61 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - run: | - sudo apt-get update && sudo apt-get install -y make \ - software-properties-common \ - libopenmpi3 \ - libopenmpi-dev \ - libboost-all-dev \ - python3 \ - python3-pip \ - - gcp-cli/install: - version: 323.0.0 - - run: - name: gcloud auth - command: | - echo $ENCODED_GCR_KEY - echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS - gcloud auth configure-docker - - checkout - - run: - name: Update Submodules - command: git submodule update --init - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache: - keys: - - v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} - - restore_cache: - keys: - - v2-savepoints-c12_54ranks_standard-{{ checksum "Makefile.data_download" }} - - restore_cache: - keys: - - v4-pip_cache-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }} - - run: - name: set up environment - command: | - pip3 install --upgrade setuptools wheel - pip3 install -r requirements_dev.txt -c constraints.txt - python3 -m gt4py.gt_src_manager install - - save_cache: - key: v4-pip_cache-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }} - paths: - - /home/circleci/.cache/pip + - setup_54ranks_mpi + - setup_savepoints + - restore_cache_54ranks_savepoints_mpi - run: name: run dycore tests command: | GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_standard" FV3_PATH=$(pwd)/fv3core make savepoint_tests_mpi no_output_timeout: 3h + - save_cache_54ranks_savepoints_mpi + + physics_savepoints_mpi_54rank: + machine: + image: ubuntu-2004:202111-02 + resource_class: xlarge + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - setup_54ranks_mpi + - setup_savepoints + - restore_cache_54ranks_savepoints_mpi - run: name: run physics tests command: | GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make physics_savepoint_tests_mpi no_output_timeout: 3h + - save_cache_54ranks_savepoints_mpi + + driver_savepoints_mpi_54rank: + machine: + image: ubuntu-2004:202111-02 + resource_class: xlarge + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - setup_54ranks_mpi + - setup_savepoints + - restore_cache_54ranks_savepoints_mpi - run: name: run driver tests command: | GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make driver_savepoint_tests_mpi no_output_timeout: 3h - - save_cache: - key: v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} - paths: - - .gt_cache - - save_cache: - key: v2-savepoints-c12_54ranks_standard-{{ checksum "Makefile.data_download" }} - paths: - - test_data + - save_cache_54ranks_savepoints_mpi test_util: docker: diff --git a/physics/tests/savepoint/translate/overrides/baroclinic.yaml b/physics/tests/savepoint/translate/overrides/baroclinic.yaml index 5ca167e1e..44e09a847 100644 --- a/physics/tests/savepoint/translate/overrides/baroclinic.yaml +++ b/physics/tests/savepoint/translate/overrides/baroclinic.yaml @@ -22,9 +22,9 @@ GFSPhysicsDriver: # For validation we deactivate it (for validation only!) Microph: - backend: numpy - max_error: 1e-10 + max_error: 2e-10 - backend: gt:cpu_ifirst - max_error: 1e-12 + max_error: 6e-11 - backend: cuda cuda_no_fma: true - backend: gt:gpu From ef9a47582a6f88b4504f98a63d36eea5deaa1bd4 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 14:56:29 -0700 Subject: [PATCH 15/48] update test names --- .circleci/config.yml | 59 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe01ee5fb..22a409ea8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -474,7 +474,7 @@ workflows: filters: tags: only: /^v.*/ - - savepoints: + - dycore_savepoints: matrix: parameters: backend: @@ -485,7 +485,7 @@ workflows: filters: tags: only: /^v.*/ - - savepoints_mpi: + - physics_savepoints: matrix: parameters: backend: @@ -496,7 +496,60 @@ workflows: filters: tags: only: /^v.*/ - - savepoints_mpi_54rank: + - dycore_savepoints_mpi: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - physics_savepoints_mpi: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - driver_savepoints_mpi: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - dycore_savepoints_mpi_54rank: + matrix: + parameters: + backend: + - numpy + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - physics_savepoints_mpi_54rank: + matrix: + parameters: + backend: + - numpy + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - driver_savepoints_mpi_54rank: matrix: parameters: backend: From 889e19f1837e611525d7a376164b16271224e7f4 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 16 Sep 2022 15:07:33 -0700 Subject: [PATCH 16/48] update config --- .circleci/config.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22a409ea8..c9d0df9ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,6 +78,11 @@ commands: python3 -m gt4py.gt_src_manager install restore_cache_savepoints: description: "Restore cache for serial savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy steps: - restore_cache: keys: @@ -87,6 +92,11 @@ commands: - v1-savepoints-{{ checksum "Makefile.data_download" }} save_cache_savepoints: description: "Save cache for serial savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy steps: - save_cache: key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} @@ -99,6 +109,11 @@ commands: - test_data restore_cache_savepoints_mpi: description: "Restore cache for mpi savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy steps: - restore_cache: keys: @@ -108,6 +123,11 @@ commands: - v1-savepoints-{{ checksum "Makefile.data_download" }} save_cache_savepoints_mpi: description: "Save cache for mpi savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy steps: - save_cache: key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} @@ -124,8 +144,29 @@ commands: paths: - test_data restore_cache_54ranks_savepoints_mpi: + description: "Restore cache for 54 ranks mpi savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy + steps: + - restore_cache: + keys: + - v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} + - restore_cache: + keys: + - v2-savepoints-c12_54ranks_standard-{{ checksum "Makefile.data_download" }} + - restore_cache: + keys: + - v4-pip_cache-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }} save_cache_54ranks_savepoints_mpi: description: "Save cache for 54 ranks mpi savepoints" + parameters: + backend: + description: GT4Py backend + type: string + default: numpy steps: - save_cache: key: v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} From 04cc70a2d97ebb470dcf4840aed1bd43fd72d739 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 22:49:01 +0000 Subject: [PATCH 17/48] add first attempt at unified savepoints target --- .circleci/config.yml | 220 ++++++++++++++++++++----------------------- Makefile | 3 +- 2 files changed, 104 insertions(+), 119 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe01ee5fb..45c4db25c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ commands: paths: - venv - /root/.cache/pip - setup_savepoints: + build_docker_image: description: "Setup savepoints" steps: - gcp-cli/install: @@ -52,89 +52,32 @@ commands: - run: name: Update Submodules command: git submodule update --init - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - run: name: build image command: | BUILD_ARGS="--progress=plain" DEV=n make build - setup_54ranks_mpi: - description: "Install libaries for 54 ranks run" - steps: - - run: | - sudo apt-get update && sudo apt-get install -y make \ - software-properties-common \ - libopenmpi3 \ - libopenmpi-dev \ - libboost-all-dev \ - python3 \ - python3-pip \ - - run: - name: set up environment - command: | - pip3 install --upgrade setuptools wheel - pip3 install -r requirements_dev.txt -c constraints.txt - python3 -m gt4py.gt_src_manager install - restore_cache_savepoints: - description: "Restore cache for serial savepoints" - steps: - - restore_cache: - keys: - - v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} - - restore_cache: - keys: - - v1-savepoints-{{ checksum "Makefile.data_download" }} - save_cache_savepoints: - description: "Save cache for serial savepoints" - steps: - - save_cache: - key: v3-gt_cache-serial-<>-{{ checksum "gt4py_version.txt" }} - paths: - - .gt_cache - - .gt_cache_000000 - - save_cache: - key: v1-savepoints-{{ checksum "Makefile.data_download" }} - paths: - - test_data - restore_cache_savepoints_mpi: - description: "Restore cache for mpi savepoints" + restore_gt_cache: + description: "Restore .gt_cache" + parameters: + key: + type: string steps: + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache: keys: - - v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} - - restore_cache: - keys: - - v1-savepoints-{{ checksum "Makefile.data_download" }} - save_cache_savepoints_mpi: - description: "Save cache for mpi savepoints" - steps: - - save_cache: - key: v2-gt_cache-<>-{{ checksum "gt4py_version.txt" }} - paths: - - .gt_cache - - .gt_cache_000000 - - .gt_cache_000001 - - .gt_cache_000002 - - .gt_cache_000003 - - .gt_cache_000004 - - .gt_cache_000005 - - save_cache: - key: v1-savepoints-{{ checksum "Makefile.data_download" }} - paths: - - test_data - restore_cache_54ranks_savepoints_mpi: - save_cache_54ranks_savepoints_mpi: - description: "Save cache for 54 ranks mpi savepoints" + - v4-gt_cache-<>-{{ checksum "gt4py_version.txt" }} + save_gt_cache: + description: "Save .gt_cache" + parameters: + key: + type: string steps: - save_cache: - key: v3-gt_cache_54ranks-<>-{{ checksum "gt4py_version.txt" }} + key: v4-gt_cache-<>-{{ checksum "gt4py_version.txt" }} paths: - .gt_cache - - save_cache: - key: v2-savepoints-c12_54ranks_standard-{{ checksum "Makefile.data_download" }} - paths: - - test_data jobs: lint: @@ -158,7 +101,7 @@ jobs: - ~/.cache/pre-commit - venv - dycore_savepoints: + make_savepoints: machine: image: ubuntu-2004:202111-02 resource_class: large @@ -168,20 +111,28 @@ jobs: default: "numpy" type: enum enum: ["numpy", "gt:cpu_ifirst"] + experiment: + description: "experiment data to use" + type: enum + enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + target: + description: "make target to run" + type: enum + enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_savepoints - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_savepoints + - checkout + - build_docker_image + - restore_gt_cache: + key: dycore-serial-<> - run: name: run dycore tests command: | - TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=<> make <> no_output_timeout: 3h - - save_cache_savepoints + - save_gt_cache: + key: dycore-serial-<> physics_savepoints: machine: @@ -196,7 +147,8 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_savepoints + - checkout + - build_docker_image - run: name: save gt4py_version.txt command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt @@ -220,7 +172,10 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_savepoints + - build_docker_image + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache_savepoints_mpi - run: name: run dycore tests @@ -242,7 +197,11 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_savepoints + - checkout + - build_docker_image + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache_savepoints_mpi - run: name: run physics tests @@ -264,7 +223,8 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_savepoints + - checkout + - build_docker_image - restore_cache_savepoints_mpi - run: name: run driver tests @@ -273,7 +233,7 @@ jobs: no_output_timeout: 3h - save_cache_savepoints_mpi - dycore_savepoints_mpi_54rank: + make_mpi_54rank: machine: image: ubuntu-2004:202111-02 resource_class: xlarge @@ -283,16 +243,27 @@ jobs: default: "numpy" type: enum enum: ["numpy", "gt:cpu_ifirst"] + experiment: + description: "experiment data to use" + type: enum + enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + target: + description: "make target to run" + type: enum + enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - setup_54ranks_mpi - - setup_savepoints + - checkout + - build_docker_image + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache_54ranks_savepoints_mpi - run: name: run dycore tests command: | - GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_standard" FV3_PATH=$(pwd)/fv3core make savepoint_tests_mpi + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT="<>" NUM_RANKS=54 make <> no_output_timeout: 3h - save_cache_54ranks_savepoints_mpi @@ -309,13 +280,17 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: + - checkout - setup_54ranks_mpi - - setup_savepoints + - build_docker_image + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache_54ranks_savepoints_mpi - run: name: run physics tests command: | - GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make physics_savepoint_tests_mpi + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" NUM_RANKS=54 make physics_savepoint_tests_mpi no_output_timeout: 3h - save_cache_54ranks_savepoints_mpi @@ -332,8 +307,12 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: + - checkout - setup_54ranks_mpi - - setup_savepoints + - build_docker_image + - run: + name: save gt4py_version.txt + command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - restore_cache_54ranks_savepoints_mpi - run: name: run driver tests @@ -474,38 +453,45 @@ workflows: filters: tags: only: /^v.*/ - - savepoints: - matrix: - parameters: - backend: - - numpy - - gt:cpu_ifirst - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - - savepoints_mpi: - matrix: - parameters: - backend: - - numpy - - gt:cpu_ifirst - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - - savepoints_mpi_54rank: + - make_savepoints: matrix: parameters: - backend: - - numpy + backend: ["numpy", "gt:cpu_ifirst"] + experiment: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + target: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] + exclude: + - experiment: "c12_54ranks_standard" + target: "physics_savepoint_tests_mpi" + - experiment: "c12_54ranks_standard" + target: "driver_savepoint_tests_mpi" + - experiment: "c12_54ranks_baroclinic_dycore_microphysics" + target: "savepoint_tests_mpi" context: - GCLOUD_ENCODED_KEY filters: tags: only: /^v.*/ + # - savepoints_mpi: + # matrix: + # parameters: + # backend: + # - numpy + # - gt:cpu_ifirst + # context: + # - GCLOUD_ENCODED_KEY + # filters: + # tags: + # only: /^v.*/ + # - savepoints_mpi_54rank: + # matrix: + # parameters: + # backend: + # - numpy + # context: + # - GCLOUD_ENCODED_KEY + # filters: + # tags: + # only: /^v.*/ - test_util: filters: tags: diff --git a/Makefile b/Makefile index a917ebfa9..123bc0ba5 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,6 @@ VOLUMES ?= ### Testing variables -FV3=fv3core RUN_FLAGS ?=--rm ifeq ("$(CONTAINER_CMD)","") PACE_PATH?=$(ROOT_DIR) @@ -56,7 +55,7 @@ else VOLUMES += -v $(EXPERIMENT_DATA):$(EXPERIMENT_DATA_RUN) endif ifeq ($(CONTAINER_CMD),docker) - CONTAINER_FLAGS=run $(RUN_FLAGS) $(VOLUMES) $(PACE_IMAGE) + CONTAINER_FLAGS=run $(RUN_FLAGS) $(VOLUMES) --env GT_CACHE_ROOT=/pace/.gt_cache $(PACE_IMAGE) else CONTAINER_FLAGS= endif From 89df5ce351f38e640f1b4b7b1911682e7feee8b0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:00:43 +0000 Subject: [PATCH 18/48] attempt to put most of the test logic in a command --- .circleci/config.yml | 184 ++++++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 62 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45c4db25c..bf37b4a82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,6 +78,37 @@ commands: key: v4-gt_cache-<>-{{ checksum "gt4py_version.txt" }} paths: - .gt_cache + make_savepoints: + description: "Setup and run a savepoints test make target" + parameters: + backend: + description: "gt4py backend" + default: "numpy" + type: enum + enum: ["numpy", "gt:cpu_ifirst"] + experiment: + description: "experiment data to use" + type: enum + enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + target: + description: "make target to run" + type: enum + enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] + num_ranks: + description: "number of ranks required" + type: integer + default: 6 + steps: + - build_docker_image + - restore_gt_cache: + key: dycore-serial-<> + - run: + name: run savepoint tests + command: | + TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=<> NUM_RANKS=<> make <> + no_output_timeout: 3h + - save_gt_cache: + key: dycore-serial-<> jobs: lint: @@ -101,7 +132,7 @@ jobs: - ~/.cache/pre-commit - venv - make_savepoints: + dycore_savepoints: machine: image: ubuntu-2004:202111-02 resource_class: large @@ -111,28 +142,15 @@ jobs: default: "numpy" type: enum enum: ["numpy", "gt:cpu_ifirst"] - experiment: - description: "experiment data to use" - type: enum - enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] - target: - description: "make target to run" - type: enum - enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - build_docker_image - - restore_gt_cache: - key: dycore-serial-<> - - run: - name: run dycore tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=<> make <> - no_output_timeout: 3h - - save_gt_cache: - key: dycore-serial-<> + - make_savepoints: + backend: <> + experiment: c12_54ranks_standard + target: savepoint_tests + physics_savepoints: machine: @@ -148,16 +166,11 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_savepoints - - run: - name: run physics tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests - - save_cache_savepoints + - make_savepoints: + backend: <> + experiment: c12_6ranks_baroclinic_dycore_microphysics + target: physics_savepoint_tests + dycore_savepoints_mpi: machine: @@ -172,17 +185,12 @@ jobs: environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_savepoints_mpi - - run: - name: run dycore tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y make savepoint_tests_mpi - no_output_timeout: 3h - - save_cache_savepoints_mpi + - checkout + - make_savepoints: + backend: <> + experiment: c12_54ranks_standard + target: savepoint_tests_mpi + physics_savepoints_mpi: machine: @@ -198,17 +206,11 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_savepoints_mpi - - run: - name: run physics tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make physics_savepoint_tests_mpi - no_output_timeout: 3h - - save_cache_savepoints_mpi + - make_savepoints: + backend: <> + experiment: c12_6ranks_baroclinic_dycore_microphysics + target: physics_savepoint_tests_mpi + driver_savepoints_mpi: machine: @@ -453,24 +455,82 @@ workflows: filters: tags: only: /^v.*/ - - make_savepoints: + - dycore_savepoints: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - physics_savepoints: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + + - dycore_savepoints_mpi: matrix: parameters: - backend: ["numpy", "gt:cpu_ifirst"] - experiment: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] - target: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] - exclude: - - experiment: "c12_54ranks_standard" - target: "physics_savepoint_tests_mpi" - - experiment: "c12_54ranks_standard" - target: "driver_savepoint_tests_mpi" - - experiment: "c12_54ranks_baroclinic_dycore_microphysics" - target: "savepoint_tests_mpi" + backend: + - numpy + - gt:cpu_ifirst context: - GCLOUD_ENCODED_KEY filters: tags: only: /^v.*/ + + - physics_savepoints_mpi: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + + # - driver_savepoints_mpi: + # matrix: + # parameters: + # backend: + # - numpy + # - gt:cpu_ifirst + # context: + # - GCLOUD_ENCODED_KEY + # filters: + # tags: + # only: /^v.*/ + # - make_savepoints: + # matrix: + # parameters: + # backend: ["numpy", "gt:cpu_ifirst"] + # experiment: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + # target: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] + # exclude: + # - experiment: "c12_54ranks_standard" + # target: "physics_savepoint_tests_mpi" + # - experiment: "c12_54ranks_standard" + # target: "driver_savepoint_tests_mpi" + # - experiment: "c12_54ranks_baroclinic_dycore_microphysics" + # target: "savepoint_tests_mpi" + # context: + # - GCLOUD_ENCODED_KEY + # filters: + # tags: + # only: /^v.*/ # - savepoints_mpi: # matrix: # parameters: From e36a10a01d73236d768b618c8d60cb6b4b18eca2 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:03:30 +0000 Subject: [PATCH 19/48] fix config to validate --- .circleci/config.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf37b4a82..fe371b1bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,11 +89,21 @@ commands: experiment: description: "experiment data to use" type: enum - enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] + enum: + - "c12_6ranks_standard" + - "c12_54ranks_standard" + - "c12_6ranks_baroclinic_dycore_microphysics" + - "c12_54ranks_baroclinic_dycore_microphysics" target: description: "make target to run" type: enum - enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] + enum: + - "savepoint_tests" + - "savepoint_tests_mpi" + - "physics_savepoint_tests" + - "physics_savepoint_tests_mpi" + - "driver_savepoint_tests" + - "driver_savepoint_tests_mpi" num_ranks: description: "number of ranks required" type: integer From 343a1c2dcdf7a35fe59fc0e86a49c924a06f2739 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:10:03 +0000 Subject: [PATCH 20/48] fix test_driver, lint, and dycore_savepoints_mpi --- .circleci/config.yml | 82 ++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 57 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe371b1bb..9c71ab468 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,7 +116,7 @@ commands: name: run savepoint tests command: | TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=<> NUM_RANKS=<> make <> - no_output_timeout: 3h + no_output_timeout: 2h - save_gt_cache: key: dycore-serial-<> jobs: @@ -126,9 +126,12 @@ jobs: - image: cimg/python:3.8 steps: - checkout + - run: + name: save python_version.txt + command: python --version | awk '{print $2;}' > python_version.txt - restore_cache: keys: - - v3-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "constraints.txt" }} + - v3-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "constraints.txt" }}-{{ checksum "python_version.txt" }} - run: name: run linting command: | @@ -137,7 +140,7 @@ jobs: pip3 install pre-commit -c constraints.txt pre-commit run --all-files - save_cache: - key: v3-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "constraints.txt" }} + key: v3-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "constraints.txt" }}-{{ checksum "python_version.txt" }} paths: - ~/.cache/pre-commit - venv @@ -198,7 +201,7 @@ jobs: - checkout - make_savepoints: backend: <> - experiment: c12_54ranks_standard + experiment: c12_6ranks_standard target: savepoint_tests_mpi @@ -236,14 +239,10 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - build_docker_image - - restore_cache_savepoints_mpi - - run: - name: run driver tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=c12_6ranks_baroclinic_dycore_microphysics make driver_savepoint_tests_mpi - no_output_timeout: 3h - - save_cache_savepoints_mpi + - make_savepoints: + backend: <> + experiment: c12_6ranks_baroclinic_dycore_microphysics + target: driver_savepoint_tests_mpi make_mpi_54rank: machine: @@ -293,18 +292,11 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - setup_54ranks_mpi - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_54ranks_savepoints_mpi - - run: - name: run physics tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" NUM_RANKS=54 make physics_savepoint_tests_mpi - no_output_timeout: 3h - - save_cache_54ranks_savepoints_mpi + - make_savepoints: + backend: <> + experiment: c12_54ranks_baroclinic_dycore_microphysics + target: physics_savepoint_tests_mpi + num_ranks: 54 driver_savepoints_mpi_54rank: machine: @@ -320,18 +312,11 @@ jobs: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - setup_54ranks_mpi - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_54ranks_savepoints_mpi - - run: - name: run driver tests - command: | - GT_CACHE_ROOT=".gt_cache" TEST_ARGS="--backend=<> -v -s" CONTAINER_CMD="" NUM_RANKS=54 EXPERIMENT="c12_54ranks_baroclinic_dycore_microphysics" make driver_savepoint_tests_mpi - no_output_timeout: 3h - - save_cache_54ranks_savepoints_mpi + - make_savepoints: + backend: <> + experiment: c12_54ranks_baroclinic_dycore_microphysics + target: driver_savepoint_tests_mpi + num_ranks: 54 test_util: docker: @@ -380,30 +365,13 @@ jobs: - run: name: Install Submodules command: git submodule update --init - - restore_cache: - keys: - - v3-driver-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }}-{{ checksum "driver/setup.py" }} - - run: - name: install packages - command: | - if [ $(python changed_from_main.py driver) != false ]; then - sudo apt-get update - sudo apt-get install libopenmpi3 libopenmpi-dev - cd driver && ./examples/create_venv.sh - fi - - save_cache: - key: v3-driver-{{ checksum "requirements_dev.txt" }}-{{ checksum "constraints.txt" }}-{{ checksum "driver/setup.py" }} - paths: - - driver/venv - - /root/.cache/pip + - setup_environment_mpi - run: name: run tests command: | - if [ $(python changed_from_main.py driver) != false ]; then - cd driver - . venv/bin/activate - MPIRUN_CALL="mpirun -n 6 --mca btl_vader_single_copy_mechanism none" make test_mpi - fi + cd driver + . venv/bin/activate + MPIRUN_CALL="mpirun -n 6 --mca btl_vader_single_copy_mechanism none" make test_mpi test_main: docker: From 74d0e1d5796fedaaf533a8974aaeaa5c96d67ab6 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:11:52 +0000 Subject: [PATCH 21/48] add remaining circleci plans --- .circleci/config.yml | 121 +++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c71ab468..53ee8b28a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -224,7 +224,6 @@ jobs: experiment: c12_6ranks_baroclinic_dycore_microphysics target: physics_savepoint_tests_mpi - driver_savepoints_mpi: machine: image: ubuntu-2004:202111-02 @@ -244,7 +243,7 @@ jobs: experiment: c12_6ranks_baroclinic_dycore_microphysics target: driver_savepoint_tests_mpi - make_mpi_54rank: + dycore_savepoints_mpi_54rank: machine: image: ubuntu-2004:202111-02 resource_class: xlarge @@ -254,29 +253,15 @@ jobs: default: "numpy" type: enum enum: ["numpy", "gt:cpu_ifirst"] - experiment: - description: "experiment data to use" - type: enum - enum: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] - target: - description: "make target to run" - type: enum - enum: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - - build_docker_image - - run: - name: save gt4py_version.txt - command: git submodule status external/gt4py | awk '{print $1;}' > gt4py_version.txt - - restore_cache_54ranks_savepoints_mpi - - run: - name: run dycore tests - command: | - TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT="<>" NUM_RANKS=54 make <> - no_output_timeout: 3h - - save_cache_54ranks_savepoints_mpi + - make_savepoints: + backend: <> + experiment: c12_54ranks_standard + target: savepoint_tests_mpi + num_ranks: 54 physics_savepoints_mpi_54rank: machine: @@ -455,7 +440,6 @@ workflows: filters: tags: only: /^v.*/ - - dycore_savepoints_mpi: matrix: parameters: @@ -467,7 +451,6 @@ workflows: filters: tags: only: /^v.*/ - - physics_savepoints_mpi: matrix: parameters: @@ -479,57 +462,47 @@ workflows: filters: tags: only: /^v.*/ - - # - driver_savepoints_mpi: - # matrix: - # parameters: - # backend: - # - numpy - # - gt:cpu_ifirst - # context: - # - GCLOUD_ENCODED_KEY - # filters: - # tags: - # only: /^v.*/ - # - make_savepoints: - # matrix: - # parameters: - # backend: ["numpy", "gt:cpu_ifirst"] - # experiment: ["c12_54ranks_standard", "c12_54ranks_baroclinic_dycore_microphysics"] - # target: ["savepoint_tests_mpi", "physics_savepoint_tests_mpi", "driver_savepoint_tests_mpi"] - # exclude: - # - experiment: "c12_54ranks_standard" - # target: "physics_savepoint_tests_mpi" - # - experiment: "c12_54ranks_standard" - # target: "driver_savepoint_tests_mpi" - # - experiment: "c12_54ranks_baroclinic_dycore_microphysics" - # target: "savepoint_tests_mpi" - # context: - # - GCLOUD_ENCODED_KEY - # filters: - # tags: - # only: /^v.*/ - # - savepoints_mpi: - # matrix: - # parameters: - # backend: - # - numpy - # - gt:cpu_ifirst - # context: - # - GCLOUD_ENCODED_KEY - # filters: - # tags: - # only: /^v.*/ - # - savepoints_mpi_54rank: - # matrix: - # parameters: - # backend: - # - numpy - # context: - # - GCLOUD_ENCODED_KEY - # filters: - # tags: - # only: /^v.*/ + - driver_savepoints_mpi: + matrix: + parameters: + backend: + - numpy + - gt:cpu_ifirst + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - dycore_savepoints_mpi_54rank: + matrix: + parameters: + backend: + - numpy + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - physics_savepoints_mpi_54rank: + matrix: + parameters: + backend: + - numpy + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ + - driver_savepoints_mpi_54rank: + matrix: + parameters: + backend: + - numpy + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ - test_util: filters: tags: From 194147608047418700813050d9e4583f636126a9 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:15:18 +0000 Subject: [PATCH 22/48] activate venv correctly in driver test --- .circleci/config.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53ee8b28a..3111ad1dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,6 +112,9 @@ commands: - build_docker_image - restore_gt_cache: key: dycore-serial-<> + - restore_cache: + keys: + - v1-test_data-<>-{{ checksum "Makefile.data_download" }} - run: name: run savepoint tests command: | @@ -119,6 +122,11 @@ commands: no_output_timeout: 2h - save_gt_cache: key: dycore-serial-<> + - save_cache: + key: v1-test_data-<>-{{ checksum "Makefile.data_download" }} + paths: + - test_data + jobs: lint: @@ -164,7 +172,6 @@ jobs: experiment: c12_54ranks_standard target: savepoint_tests - physics_savepoints: machine: image: ubuntu-2004:202111-02 @@ -184,7 +191,6 @@ jobs: experiment: c12_6ranks_baroclinic_dycore_microphysics target: physics_savepoint_tests - dycore_savepoints_mpi: machine: image: ubuntu-2004:202111-02 @@ -204,7 +210,6 @@ jobs: experiment: c12_6ranks_standard target: savepoint_tests_mpi - physics_savepoints_mpi: machine: image: ubuntu-2004:202111-02 @@ -354,8 +359,8 @@ jobs: - run: name: run tests command: | - cd driver . venv/bin/activate + cd driver MPIRUN_CALL="mpirun -n 6 --mca btl_vader_single_copy_mechanism none" make test_mpi test_main: From 0ead11597784b19e23dcab398d2158d929d9b7a8 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:17:52 +0000 Subject: [PATCH 23/48] use independent keys for gt caching --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3111ad1dc..33b10cc42 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,7 +111,7 @@ commands: steps: - build_docker_image - restore_gt_cache: - key: dycore-serial-<> + key: <>-<>-<> - restore_cache: keys: - v1-test_data-<>-{{ checksum "Makefile.data_download" }} @@ -121,7 +121,7 @@ commands: TEST_ARGS="--backend=<> -v -s" DEV=y EXPERIMENT=<> NUM_RANKS=<> make <> no_output_timeout: 2h - save_gt_cache: - key: dycore-serial-<> + key: <>-<>-<> - save_cache: key: v1-test_data-<>-{{ checksum "Makefile.data_download" }} paths: From c6011ca66c70900ff385a1ef51543fc43e6eb9d9 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Sep 2022 23:50:12 +0000 Subject: [PATCH 24/48] downgrade serial savepoint tests to medium resource type --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33b10cc42..a590701b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,7 +156,7 @@ jobs: dycore_savepoints: machine: image: ubuntu-2004:202111-02 - resource_class: large + resource_class: medium parameters: backend: description: "gt4py backend" @@ -175,7 +175,7 @@ jobs: physics_savepoints: machine: image: ubuntu-2004:202111-02 - resource_class: large + resource_class: medium parameters: backend: description: "gt4py backend" From 3e9d080558091a22be370b3abb0713d38910bd3c Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Mon, 19 Sep 2022 11:01:45 -0700 Subject: [PATCH 25/48] add h5netcdf to requirement --- requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 6a4386b88..18bb1a9ab 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,6 +4,7 @@ pytest-regressions pytest-profiling pytest-cov mpi4py +h5netcdf xarray zarr netCDF4 From 42cfc9b838e0d794f3cf4eaa8523f02f76746b32 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Mon, 19 Sep 2022 11:56:59 -0700 Subject: [PATCH 26/48] specify xarray engine --- stencils/pace/stencils/testing/conftest.py | 8 ++++---- stencils/pace/stencils/testing/savepoint.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/stencils/pace/stencils/testing/conftest.py b/stencils/pace/stencils/testing/conftest.py index ce9946fc3..c50ff8492 100644 --- a/stencils/pace/stencils/testing/conftest.py +++ b/stencils/pace/stencils/testing/conftest.py @@ -154,9 +154,9 @@ def _savepoint_cases( compute_grid: bool, ): return_list = [] - ds_grid: xr.Dataset = xr.open_dataset(os.path.join(data_path, "Grid-Info.nc")).isel( - savepoint=0 - ) + ds_grid: xr.Dataset = xr.open_dataset( + os.path.join(data_path, "Grid-Info.nc"), engine="h5netcdf" + ).isel(savepoint=0) for rank in ranks: grid = TranslateGrid( dataset_to_dict(ds_grid.isel(rank=rank)), @@ -175,7 +175,7 @@ def _savepoint_cases( test_name, grid, namelist, stencil_factory ) n_calls = xr.open_dataset( - os.path.join(data_path, f"{test_name}-In.nc") + os.path.join(data_path, f"{test_name}-In.nc"), engine="h5netcdf" ).dims["savepoint"] for i_call in range(n_calls): return_list.append( diff --git a/stencils/pace/stencils/testing/savepoint.py b/stencils/pace/stencils/testing/savepoint.py index 4ed195c54..d7ba2cb9d 100644 --- a/stencils/pace/stencils/testing/savepoint.py +++ b/stencils/pace/stencils/testing/savepoint.py @@ -48,7 +48,10 @@ def __str__(self): @property def ds_in(self) -> xr.Dataset: return ( - xr.open_dataset(os.path.join(self.data_dir, f"{self.savepoint_name}-In.nc")) + xr.open_dataset( + os.path.join(self.data_dir, f"{self.savepoint_name}-In.nc"), + engine="h5netcdf", + ) .isel(rank=self.rank) .isel(savepoint=self.i_call) ) @@ -57,7 +60,8 @@ def ds_in(self) -> xr.Dataset: def ds_out(self) -> xr.Dataset: return ( xr.open_dataset( - os.path.join(self.data_dir, f"{self.savepoint_name}-Out.nc") + os.path.join(self.data_dir, f"{self.savepoint_name}-Out.nc"), + engine="h5netcdf", ) .isel(rank=self.rank) .isel(savepoint=self.i_call) From d877737499d2569cece10b05ef71781f4d6f4b82 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:00:58 +0000 Subject: [PATCH 27/48] fix bug in checkpointer test, add more savepoints, update thresholds --- Makefile | 4 + fv3core/pace/fv3core/stencils/remapping.py | 66 ++++++- fv3core/pace/fv3core/stencils/tracer_2d_1l.py | 37 +++- tests/mpi/test_checkpoints.py | 28 +-- tests/mpi/thresholds/fv_dynamics.yaml | 164 +++++++++--------- 5 files changed, 201 insertions(+), 98 deletions(-) diff --git a/Makefile b/Makefile index e17d570f4..a4c909628 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,10 @@ physics_savepoint_tests_mpi: build test_main: build $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "pip3 list && cd $(PACE_PATH) && pytest $(TEST_ARGS) $(PACE_PATH)/tests/main" +test_mpi: + TARGET=dycore $(MAKE) get_test_data + mpirun -n 6 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi --data_path=$(EXPERIMENT_DATA)/dycore/ + test_mpi_54rank: mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank diff --git a/fv3core/pace/fv3core/stencils/remapping.py b/fv3core/pace/fv3core/stencils/remapping.py index 71e9997fd..e7c83271f 100644 --- a/fv3core/pace/fv3core/stencils/remapping.py +++ b/fv3core/pace/fv3core/stencils/remapping.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Optional from gt4py.gtscript import ( __INLINED, @@ -15,6 +15,7 @@ import pace.dsl.gt4py_utils as utils import pace.fv3core.stencils.moist_cv as moist_cv +import pace.util from pace.dsl.dace.orchestration import orchestrate from pace.dsl.stencil import StencilFactory from pace.dsl.typing import FloatField, FloatFieldIJ, FloatFieldK @@ -286,12 +287,17 @@ def __init__( nq, pfull, tracers: Dict[str, Quantity], + checkpointer: Optional[pace.util.Checkpointer] = None, ): orchestrate( obj=self, config=stencil_factory.config.dace_config, dace_compiletime_args=["tracers"], ) + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None grid_indexing = stencil_factory.grid_indexing if config.kord_tm >= 0: raise NotImplementedError("map ppm, untested mode where kord_tm >= 0") @@ -557,6 +563,35 @@ def __call__( Remap the deformed Lagrangian surfaces onto the reference, or "Eulerian", coordinate levels. """ + + if self._call_checkpointer: + self._checkpointer( + "Remapping-In", + pt=pt, + delp=delp, + delz=delz, + peln=peln, + u=u, + v=v, + w=w, + ua=ua, + va=va, + cappa=cappa, + q_con=q_con, + q_cld=q_cld, + pkz=pkz, + pk=pk, + pe=pe, + phis=hs, + te_2d=te0_2d, + ps=ps, + wsd=wsd, + omga=omga, + ak=ak, + bk=bk, + pfull=pfull, + dp1=dp1, + ) # TODO: remove unused arguments (and commented code that references them) # TODO: can we trim ps or make it a temporary self._init_pe(pe, self._pe1, self._pe2, ptop) @@ -692,3 +727,32 @@ def __call__( ) else: self._basic_adjust_divide_stencil(pkz, pt) + + if self._call_checkpointer: + self._checkpointer( + "Remapping-Out", + pt=pt, + delp=delp, + delz=delz, + peln=peln, + u=u, + v=v, + w=w, + ua=ua, + va=va, + cappa=cappa, + q_con=q_con, + q_cld=q_cld, + pkz=pkz, + pk=pk, + pe=pe, + hs=hs, + te0_2d=te0_2d, + ps=ps, + wsd=wsd, + omga=omga, + ak=ak, + bk=bk, + pfull=pfull, + dp1=dp1, + ) diff --git a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py index 3faea6431..0d8666a0f 100644 --- a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py +++ b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py @@ -1,5 +1,5 @@ import math -from typing import Dict +from typing import Dict, Optional import gt4py.gtscript as gtscript from gt4py.gtscript import PARALLEL, computation, horizontal, interval, region @@ -183,12 +183,17 @@ def __init__( grid_data, comm: pace.util.CubedSphereCommunicator, tracers: Dict[str, Quantity], + checkpointer: Optional[pace.util.Checkpointer] = None, ): orchestrate( obj=self, config=stencil_factory.config.dace_config, dace_compiletime_args=["tracers"], ) + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None grid_indexing = stencil_factory.grid_indexing self.grid_indexing = grid_indexing # needed for selective validation self._tracer_count = len(tracers) @@ -276,6 +281,34 @@ def make_storage(): [t for t in tracers.keys()], ) + def _checkpoint_input( + self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd + ): + if self._checkpointer is not None: + self._checkpointer( + "Tracer2D1L-In", + dp1=dp1, + mfxd=mfxd, + mfyd=mfyd, + cxd=cxd, + cyd=cyd, + **tracers, + ) + + def _checkpoint_output( + self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd + ): + if self._checkpointer is not None: + self._checkpointer( + "Tracer2D1L-Out", + dp1=dp1, + mfxd=mfxd, + mfyd=mfyd, + cxd=cxd, + cyd=cyd, + **tracers, + ) + def __call__(self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd, mdt): """ Args: @@ -286,6 +319,7 @@ def __call__(self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd, mdt) cxd (inout): cyd (inout): """ + self._checkpoint_input(tracers, dp1, mfxd, mfyd, cxd, cyd) # TODO: remove unused mdt argument # DaCe parsing issue # if len(tracers) != self._tracer_count: @@ -389,3 +423,4 @@ def __call__(self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd, mdt) self._tracers_halo_updater.update() # use variable assignment to avoid a data copy self._swap_dp(dp1, dp2) + self._checkpoint_output(tracers, dp1, mfxd, mfyd, cxd, cyd) diff --git a/tests/mpi/test_checkpoints.py b/tests/mpi/test_checkpoints.py index 75e2dd494..8b811938d 100644 --- a/tests/mpi/test_checkpoints.py +++ b/tests/mpi/test_checkpoints.py @@ -101,7 +101,6 @@ def test_fv_dynamics( ds, translate, ) - print("stencils initialized") if calibrate_thresholds: thresholds = _calibrate_thresholds( initializer=initializer, @@ -153,23 +152,24 @@ def _calibrate_thresholds( n_trials: int, factor: float, ): - state, grid_data = initializer.new_state() calibration = pace.util.ThresholdCalibrationCheckpointer(factor=factor) - dycore = fv3core.DynamicalCore( - comm=communicator, - grid_data=grid_data, - stencil_factory=stencil_factory, - damping_coefficients=damping_coefficients, - config=dycore_config, - phis=state.phis, - state=state, - checkpointer=calibration, - timestep=timedelta(seconds=dycore_config.dt_atmos), - ) for i in range(n_trials): print(f"running calibration trial {i}") - trial_state, _ = initializer.new_state() + trial_state, grid_data = initializer.new_state() perturb(dycore_state_to_dict(trial_state)) + # we need to initialize new DynamicalCore because halo updates bind + # to a particular state object, currently + dycore = fv3core.DynamicalCore( + comm=communicator, + grid_data=grid_data, + stencil_factory=stencil_factory, + damping_coefficients=damping_coefficients, + config=dycore_config, + phis=trial_state.phis, + state=trial_state, + checkpointer=calibration, + timestep=timedelta(seconds=dycore_config.dt_atmos), + ) with calibration.trial(): dycore.step_dynamics(trial_state) all_thresholds = communicator.comm.allgather(calibration.thresholds) diff --git a/tests/mpi/thresholds/fv_dynamics.yaml b/tests/mpi/thresholds/fv_dynamics.yaml index 6c51bf153..1f467640d 100644 --- a/tests/mpi/thresholds/fv_dynamics.yaml +++ b/tests/mpi/thresholds/fv_dynamics.yaml @@ -2,86 +2,86 @@ savepoints: C_SW-In: - delpd: absolute: 2.2737367544323206e-12 - relative: 2.2200743449158477e-15 + relative: 2.220074344915848e-15 divgdd: absolute: 0.0 relative: 0.0 ptd: - absolute: 2.8421709430404007e-13 - relative: 1.851063925230147e-14 + absolute: 2.1316282072803006e-13 + relative: 1.8510639252301478e-14 uad: absolute: 640.0 - relative: 2.2204251720152353e-15 + relative: 2.220425172015235e-15 ucd: absolute: 1.4210854715202004e-13 relative: 2.2204312519237066e-15 ud: absolute: 1.4210854715202004e-13 - relative: 2.2204431850207167e-15 + relative: 2.220373305506478e-15 utd: absolute: 0.0 relative: 0.0 vad: absolute: 640.0 - relative: 2.2204439912402017e-15 + relative: 2.2204408808181443e-15 vcd: absolute: 1.4210854715202004e-13 relative: 2.220444857175165e-15 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204380358035745e-15 + relative: 2.2204311404975443e-15 vtd: absolute: 0.0 relative: 0.0 wd: absolute: 2.7755575615628914e-16 - relative: 2.2204304755486847e-15 + relative: 2.2203205508872382e-15 C_SW-Out: - delpd: absolute: 2.2737367544323206e-12 - relative: 2.2200743449158477e-15 + relative: 2.220074344915848e-15 divgdd: - absolute: 1.0164395367051604e-18 + absolute: 8.597384414631148e-19 relative: 1.0116858185720617e-09 ptd: - absolute: 2.8421709430404007e-13 - relative: 1.851063925230147e-14 + absolute: 2.1316282072803006e-13 + relative: 1.8510639252301478e-14 uad: absolute: 1280.0 - relative: 4.4531842264012817e-11 + relative: 4.4531842263992984e-11 ucd: absolute: 4.263256414560601e-13 - relative: 7.106894435212558e-11 + relative: 1.9511117370337876e-11 ud: absolute: 1.4210854715202004e-13 - relative: 2.2204431850207167e-15 + relative: 2.220373305506478e-15 utd: absolute: 3.814697265625e-05 - relative: 2.5303564622294458e-11 + relative: 7.072664425264997e-11 vad: absolute: 1280.0 - relative: 3.801762325990083e-10 + relative: 1.2738435085781666e-10 vcd: absolute: 4.263256414560601e-13 - relative: 3.969792148639851e-11 + relative: 2.1220030303501963e-11 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204380358035745e-15 + relative: 2.2204311404975443e-15 vtd: - absolute: 5.7220458984375e-05 - relative: 1.4144975022713432e-10 + absolute: 3.814697265625e-05 + relative: 1.0608819723976712e-10 wd: absolute: 2.7755575615628914e-16 - relative: 2.2204304755486847e-15 + relative: 2.2203205508872382e-15 D_SW-In: - delpcd: - absolute: 5.7220458984375e-05 - relative: 1.4144975022713432e-10 + absolute: 3.814697265625e-05 + relative: 1.0608819723976712e-10 delpd: absolute: 2.2737367544323206e-12 - relative: 2.2200743449158477e-15 + relative: 2.220074344915848e-15 divgdd: - absolute: 1.0164395367051604e-18 + absolute: 8.597384414631148e-19 relative: 1.0116858185720617e-09 mfxd: absolute: 0.0 @@ -90,29 +90,29 @@ savepoints: absolute: 0.0 relative: 0.0 ptd: - absolute: 2.8421709430404007e-13 - relative: 1.851063925230147e-14 + absolute: 2.1316282072803006e-13 + relative: 1.8510639252301478e-14 uad: absolute: 1280.0 - relative: 4.4531842264012817e-11 + relative: 4.4531842263992984e-11 ucd: - absolute: 9.592326932761353e-13 - relative: 4.014241461643457e-10 + absolute: 7.904787935331115e-13 + relative: 9.780251382633263e-10 ud: absolute: 1.4210854715202004e-13 - relative: 2.2204431850207167e-15 + relative: 2.220373305506478e-15 vad: absolute: 1280.0 - relative: 3.801762325990083e-10 + relative: 1.2738435085781666e-10 vcd: - absolute: 1.0658141036401503e-12 - relative: 9.678516401601087e-10 + absolute: 7.904787935331115e-13 + relative: 9.780251382633263e-10 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204380358035745e-15 + relative: 2.2204311404975443e-15 wd: absolute: 2.7755575615628914e-16 - relative: 2.2204304755486847e-15 + relative: 2.2203205508872382e-15 xfxd: absolute: 0.0 relative: 0.0 @@ -120,54 +120,54 @@ savepoints: absolute: 0.0 relative: 0.0 zhd: - absolute: 2.1827872842550278e-10 - relative: 7.04593126878164e-15 + absolute: 1.4551915228366852e-10 + relative: 6.52099891718502e-15 D_SW-Out: - delpcd: - absolute: 2.86102294921875e-05 + absolute: 3.337860107421875e-05 relative: 1.0116858185720617e-09 delpd: - absolute: 0.0006292948035024892 - relative: 1.6050064191530135e-06 + absolute: 6.821210263296962e-12 + relative: 9.349923208080012e-15 divgdd: - absolute: 5.590417451878382e-19 - relative: 4.4508165492170205e-09 + absolute: 8.470329472543003e-19 + relative: 1.1893991752988745e-08 mfxd: - absolute: 2.3496618687725375e+24 - relative: 16.3636363730611 + absolute: 1444413440.0 + relative: 50.0 mfyd: - absolute: 1.2686334171435062e+24 - relative: 16.36363636579146 + absolute: 1264189440.0 + relative: 49.999999999999325 ptd: - absolute: 5.919210366300831e-07 - relative: 1.543661294125962e-08 + absolute: 2.8421709430404007e-13 + relative: 1.9970554019934083e-14 uad: absolute: 1280.0 - relative: 4.4531842264012817e-11 + relative: 4.4531842263992984e-11 ucd: - absolute: 7.105427357601002e-13 - relative: 2.335683747836233e-09 + absolute: 5.789813073420191e-13 + relative: 2.0924792441159182e-07 ud: - absolute: 2.2351741790771484e-07 - relative: 1.773082587559174e-11 + absolute: 1.4901161193847656e-07 + relative: 5.308745703431066e-12 vad: absolute: 1280.0 - relative: 3.801762325990083e-10 + relative: 1.2738435085781666e-10 vcd: - absolute: 1.0658141036401503e-12 - relative: 1.5479001178967714e-09 + absolute: 7.904787935331115e-13 + relative: 5.230529400854598e-10 vd: - absolute: 2.980232238769531e-07 - relative: 1.1344899282993951e-10 + absolute: 2.2351741790771484e-07 + relative: 3.999685149120597e-12 wd: - absolute: 3.881427367014023e-08 - relative: 8.033914185806085e-05 + absolute: 6.245004513516506e-16 + relative: 2.9515782433798676e-12 xfxd: - absolute: 0.0001537799835205078 - relative: 1.7732558727067202e-09 + absolute: 0.00015020370483398438 + relative: 1.0950535166065047e-10 yfxd: - absolute: 0.000171661376953125 - relative: 8.255143046330278e-10 + absolute: 0.00011444091796875 + relative: 1.9713230836005074e-10 FVDynamics-In: - delz: absolute: 0.0 @@ -198,29 +198,29 @@ savepoints: relative: 0.0 FVDynamics-Out: - delz: - absolute: 0.001160447809525067 - relative: 3.861710180457495e-07 + absolute: 1.3272938303998671e-10 + relative: 3.682097278337629e-12 qvapor: - absolute: 3.469446951953614e-17 - relative: 2.2204399417111508e-15 + absolute: 4.182851981449076e-14 + relative: 1.3949257689370736e-11 u: - absolute: 1.5095681362709001e-05 - relative: 4.723330275579375e-05 + absolute: 1.8554047187535616e-11 + relative: 2.4845358044915226e-09 ua: absolute: 1280.0 - relative: 8.068373919387091e-05 + relative: 6.545131190335245e-10 uc: - absolute: 7.105427357601002e-13 - relative: 2.335683747836233e-09 + absolute: 5.789813073420191e-13 + relative: 2.0924792441159182e-07 v: - absolute: 8.262938830938538e-06 - relative: 0.00632014380512243 + absolute: 1.248778858098376e-11 + relative: 2.4845358044915226e-09 va: absolute: 1280.0 - relative: 0.00016662659576558862 + relative: 6.175467803387584e-10 vc: - absolute: 1.0658141036401503e-12 - relative: 1.5479001178967714e-09 + absolute: 7.904787935331115e-13 + relative: 5.230529400854598e-10 w: - absolute: 1.7342969022282606e-05 - relative: 0.025006993840671343 + absolute: 1.4737343290160965e-12 + relative: 2.105284830080243e-06 From 2d207f0c1ab55e73215410bf3bed7ec443b1e385 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:22:12 +0000 Subject: [PATCH 28/48] add circleci execution of checkpoint tests, label as savepoint tests, run in docker --- .circleci/config.yml | 19 +++++++++++++++++++ Makefile | 6 +++--- tests/{mpi => savepoints}/conftest.py | 0 tests/{mpi => savepoints}/test_checkpoints.py | 0 .../thresholds/fv_dynamics.yaml | 0 5 files changed, 22 insertions(+), 3 deletions(-) rename tests/{mpi => savepoints}/conftest.py (100%) rename tests/{mpi => savepoints}/test_checkpoints.py (100%) rename tests/{mpi => savepoints}/thresholds/fv_dynamics.yaml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index a590701b9..c5b2bd565 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -396,6 +396,19 @@ jobs: . venv/bin/activate mpirun -n 54 --oversubscribe --mca btl_vader_single_copy_mechanism none python3 -m mpi4py -m pytest tests/mpi_54rank + test_savepoints: + docker: + - image: cimg/python:3.8 + resource_class: large + working_directory: ~/repo + steps: + - checkout + - make_savepoints: + backend: gt:cpu_ifirst + experiment: c12_6ranks_standard + target: test_mpi + num_ranks: 6 + test_notebooks: docker: - image: gcr.io/vcm-ml/pace_notebook_examples @@ -508,6 +521,12 @@ workflows: filters: tags: only: /^v.*/ + - test_savepoints: + context: + - GCLOUD_ENCODED_KEY + filters: + tags: + only: /^v.*/ - test_util: filters: tags: diff --git a/Makefile b/Makefile index 03411d208..b07001b01 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ test_util: $(MAKE) -C util test; \ fi -savepoint_tests: build +savepoint_tests: build ## dycore-only savepoint tests TARGET=dycore $(MAKE) get_test_data $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(FV3CORE_THRESH_ARGS) $(PACE_PATH)/fv3core/tests/savepoint" @@ -128,9 +128,9 @@ physics_savepoint_tests_mpi: build test_main: build $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest $(TEST_ARGS) $(PACE_PATH)/tests/main" -test_mpi: +test_savepoints: ## top level savepoint tests TARGET=dycore $(MAKE) get_test_data - mpirun -n 6 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi --data_path=$(EXPERIMENT_DATA)/dycore/ + $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/mpi" test_mpi_54rank: mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank diff --git a/tests/mpi/conftest.py b/tests/savepoints/conftest.py similarity index 100% rename from tests/mpi/conftest.py rename to tests/savepoints/conftest.py diff --git a/tests/mpi/test_checkpoints.py b/tests/savepoints/test_checkpoints.py similarity index 100% rename from tests/mpi/test_checkpoints.py rename to tests/savepoints/test_checkpoints.py diff --git a/tests/mpi/thresholds/fv_dynamics.yaml b/tests/savepoints/thresholds/fv_dynamics.yaml similarity index 100% rename from tests/mpi/thresholds/fv_dynamics.yaml rename to tests/savepoints/thresholds/fv_dynamics.yaml From 859d2a5515c09d5234687f7864b080ed9cd55434 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:24:16 +0000 Subject: [PATCH 29/48] fix config validation --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5b2bd565..d5940db8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,7 @@ commands: description: "make target to run" type: enum enum: + - "test_savepoints" - "savepoint_tests" - "savepoint_tests_mpi" - "physics_savepoint_tests" @@ -406,7 +407,7 @@ jobs: - make_savepoints: backend: gt:cpu_ifirst experiment: c12_6ranks_standard - target: test_mpi + target: test_savepoints num_ranks: 6 test_notebooks: From e743e624da29ac9e54c513a5147c45062ba44088 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:31:00 +0000 Subject: [PATCH 30/48] define google application credentials env var for top level savepoints tests --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5940db8e..3932feda2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -402,10 +402,12 @@ jobs: - image: cimg/python:3.8 resource_class: large working_directory: ~/repo + environment: + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - checkout - make_savepoints: - backend: gt:cpu_ifirst + backend: numpy experiment: c12_6ranks_standard target: test_savepoints num_ranks: 6 From 915e6e3d145725cee7fbfb79e32c70220d1528ec Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:37:55 +0000 Subject: [PATCH 31/48] use machine worker for test_savepoints --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3932feda2..bf32c5d5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -398,10 +398,9 @@ jobs: mpirun -n 54 --oversubscribe --mca btl_vader_single_copy_mechanism none python3 -m mpi4py -m pytest tests/mpi_54rank test_savepoints: - docker: - - image: cimg/python:3.8 - resource_class: large - working_directory: ~/repo + machine: + image: ubuntu-2004:202111-02 + resource_class: large environment: GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: From c79eec4c0878c098f7b8031c7df8314a22ef6a35 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:47:32 +0000 Subject: [PATCH 32/48] run the 54rank test on an xlarge worker --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 199e7c778..b37e22aa8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -382,7 +382,7 @@ jobs: test_mpi_54rank: docker: - image: cimg/python:3.8 - resource_class: large + resource_class: xlarge working_directory: ~/repo steps: - checkout From f96cb359223280de0f7a5affd588f931881ad515 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:53:56 +0000 Subject: [PATCH 33/48] delete duplicated test definitions --- .circleci/config.yml | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b37e22aa8..9f8f4d2d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -523,47 +523,6 @@ workflows: filters: tags: only: /^v.*/ - - driver_savepoints_mpi: - matrix: - parameters: - backend: - - numpy - - gt:cpu_ifirst - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - - dycore_savepoints_mpi_54rank: - matrix: - parameters: - backend: - - numpy - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - - physics_savepoints_mpi_54rank: - matrix: - parameters: - backend: - - numpy - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - - driver_savepoints_mpi_54rank: - matrix: - parameters: - backend: - - numpy - context: - - GCLOUD_ENCODED_KEY - filters: - tags: - only: /^v.*/ - test_savepoints: context: - GCLOUD_ENCODED_KEY From e0e6e79d68949a920de3c1c5bd3efeff335cddd5 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:55:27 +0000 Subject: [PATCH 34/48] fix path for savepoint tests --- .circleci/config.yml | 8 ++++---- Makefile | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f8f4d2d7..11a47e665 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,7 @@ commands: description: "make target to run" type: enum enum: - - "test_savepoints" + - "test_savepoint" - "savepoint_tests" - "savepoint_tests_mpi" - "physics_savepoint_tests" @@ -396,7 +396,7 @@ jobs: . venv/bin/activate mpirun -n 54 --oversubscribe --mca btl_vader_single_copy_mechanism none python3 -m mpi4py -m pytest tests/mpi_54rank - test_savepoints: + test_savepoint: machine: image: ubuntu-2004:202111-02 resource_class: large @@ -407,7 +407,7 @@ jobs: - make_savepoints: backend: numpy experiment: c12_6ranks_standard - target: test_savepoints + target: test_savepoint num_ranks: 6 test_notebooks: @@ -523,7 +523,7 @@ workflows: filters: tags: only: /^v.*/ - - test_savepoints: + - test_savepoint: context: - GCLOUD_ENCODED_KEY filters: diff --git a/Makefile b/Makefile index f7f638128..ca8d2e10e 100644 --- a/Makefile +++ b/Makefile @@ -130,9 +130,9 @@ physics_savepoint_tests_mpi: build test_main: build $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest $(TEST_ARGS) $(PACE_PATH)/tests/main" -test_savepoints: ## top level savepoint tests +test_savepoint: ## top level savepoint tests TARGET=dycore $(MAKE) get_test_data - $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/mpi" + $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/savepoint" test_mpi_54rank: mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank From 6f88f0bcdca7f1e58b91dc6d60cc9c2995234ceb Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Sep 2022 23:57:42 +0000 Subject: [PATCH 35/48] move directory to match moved path in last commit --- tests/{savepoints => savepoint}/conftest.py | 0 tests/{savepoints => savepoint}/test_checkpoints.py | 0 tests/{savepoints => savepoint}/thresholds/fv_dynamics.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{savepoints => savepoint}/conftest.py (100%) rename tests/{savepoints => savepoint}/test_checkpoints.py (100%) rename tests/{savepoints => savepoint}/thresholds/fv_dynamics.yaml (100%) diff --git a/tests/savepoints/conftest.py b/tests/savepoint/conftest.py similarity index 100% rename from tests/savepoints/conftest.py rename to tests/savepoint/conftest.py diff --git a/tests/savepoints/test_checkpoints.py b/tests/savepoint/test_checkpoints.py similarity index 100% rename from tests/savepoints/test_checkpoints.py rename to tests/savepoint/test_checkpoints.py diff --git a/tests/savepoints/thresholds/fv_dynamics.yaml b/tests/savepoint/thresholds/fv_dynamics.yaml similarity index 100% rename from tests/savepoints/thresholds/fv_dynamics.yaml rename to tests/savepoint/thresholds/fv_dynamics.yaml From 4becd03bb29bd5771768148e7956f91bc82461b6 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 20 Sep 2022 15:53:05 +0000 Subject: [PATCH 36/48] remove driver 54rank compiled test from CircleCI --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11a47e665..5855badfd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -517,7 +517,6 @@ workflows: parameters: backend: - numpy - - gt:cpu_ifirst context: - GCLOUD_ENCODED_KEY filters: From 8589707abc27fa32a5735da155805f4f0e0b52d8 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 20 Sep 2022 16:41:20 +0000 Subject: [PATCH 37/48] increase driver_savepoints_mpi resource type to xlarge --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5855badfd..3f1494381 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,7 +232,7 @@ jobs: driver_savepoints_mpi: machine: image: ubuntu-2004:202111-02 - resource_class: large + resource_class: xlarge parameters: backend: description: "gt4py backend" From fcecc1a377d9d51c8838eba8760a5748f2889b41 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Sep 2022 00:15:17 +0000 Subject: [PATCH 38/48] checkpointer test works with remapping and tracer advection --- Makefile | 2 +- fv3core/pace/fv3core/stencils/fv_dynamics.py | 8 +- fv3core/pace/fv3core/stencils/remapping.py | 32 +- fv3core/pace/fv3core/stencils/tracer_2d_1l.py | 2 - requirements_dev.txt | 1 - stencils/pace/stencils/testing/conftest.py | 8 +- stencils/pace/stencils/testing/savepoint.py | 8 +- tests/savepoint/test_checkpoints.py | 1 - tests/savepoint/thresholds/fv_dynamics.yaml | 343 +++++++++++++----- util/pace/util/checkpointer/validation.py | 23 +- 10 files changed, 301 insertions(+), 127 deletions(-) diff --git a/Makefile b/Makefile index ca8d2e10e..0b606e73b 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ test_main: build test_savepoint: ## top level savepoint tests TARGET=dycore $(MAKE) get_test_data - $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/savepoint" + $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/savepoint" test_mpi_54rank: mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank diff --git a/fv3core/pace/fv3core/stencils/fv_dynamics.py b/fv3core/pace/fv3core/stencils/fv_dynamics.py index efacf78ef..0a0d93f03 100644 --- a/fv3core/pace/fv3core/stencils/fv_dynamics.py +++ b/fv3core/pace/fv3core/stencils/fv_dynamics.py @@ -234,7 +234,12 @@ def __init__( # Build advection stencils self.tracer_advection = tracer_2d_1l.TracerAdvection( - stencil_factory, tracer_transport, self.grid_data, comm, self.tracers + stencil_factory, + tracer_transport, + self.grid_data, + comm, + self.tracers, + checkpointer=checkpointer, ) self._ak = grid_data.ak self._bk = grid_data.bk @@ -316,6 +321,7 @@ def __init__( NQ, self._pfull, tracers=self.tracers, + checkpointer=checkpointer, ) full_xyz_spec = grid_indexing.get_quantity_halo_spec( diff --git a/fv3core/pace/fv3core/stencils/remapping.py b/fv3core/pace/fv3core/stencils/remapping.py index e7c83271f..95dc71dad 100644 --- a/fv3core/pace/fv3core/stencils/remapping.py +++ b/fv3core/pace/fv3core/stencils/remapping.py @@ -25,7 +25,7 @@ from pace.fv3core.stencils.mapn_tracer import MapNTracer from pace.fv3core.stencils.moist_cv import moist_pt_func, moist_pt_last_step from pace.fv3core.stencils.saturation_adjustment import SatAdjust3d -from pace.util import Quantity +from pace.util import X_DIM, Y_DIM, Z_INTERFACE_DIM, Quantity # TODO: Should this be set here or in global_constants? @@ -570,18 +570,20 @@ def __call__( pt=pt, delp=delp, delz=delz, - peln=peln, + peln=peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data u=u, v=v, w=w, ua=ua, va=va, cappa=cappa, - q_con=q_con, - q_cld=q_cld, pkz=pkz, pk=pk, - pe=pe, + pe=pe.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data phis=hs, te_2d=te0_2d, ps=ps, @@ -734,25 +736,19 @@ def __call__( pt=pt, delp=delp, delz=delz, - peln=peln, + peln=peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data u=u, v=v, w=w, - ua=ua, - va=va, cappa=cappa, - q_con=q_con, - q_cld=q_cld, pkz=pkz, pk=pk, - pe=pe, - hs=hs, - te0_2d=te0_2d, - ps=ps, - wsd=wsd, + pe=pe.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data + te_2d=te0_2d, omga=omga, - ak=ak, - bk=bk, - pfull=pfull, dp1=dp1, ) diff --git a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py index 0d8666a0f..c374b5bc0 100644 --- a/fv3core/pace/fv3core/stencils/tracer_2d_1l.py +++ b/fv3core/pace/fv3core/stencils/tracer_2d_1l.py @@ -292,7 +292,6 @@ def _checkpoint_input( mfyd=mfyd, cxd=cxd, cyd=cyd, - **tracers, ) def _checkpoint_output( @@ -306,7 +305,6 @@ def _checkpoint_output( mfyd=mfyd, cxd=cxd, cyd=cyd, - **tracers, ) def __call__(self, tracers: Dict[str, Quantity], dp1, mfxd, mfyd, cxd, cyd, mdt): diff --git a/requirements_dev.txt b/requirements_dev.txt index 18bb1a9ab..6a4386b88 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,7 +4,6 @@ pytest-regressions pytest-profiling pytest-cov mpi4py -h5netcdf xarray zarr netCDF4 diff --git a/stencils/pace/stencils/testing/conftest.py b/stencils/pace/stencils/testing/conftest.py index c50ff8492..ce9946fc3 100644 --- a/stencils/pace/stencils/testing/conftest.py +++ b/stencils/pace/stencils/testing/conftest.py @@ -154,9 +154,9 @@ def _savepoint_cases( compute_grid: bool, ): return_list = [] - ds_grid: xr.Dataset = xr.open_dataset( - os.path.join(data_path, "Grid-Info.nc"), engine="h5netcdf" - ).isel(savepoint=0) + ds_grid: xr.Dataset = xr.open_dataset(os.path.join(data_path, "Grid-Info.nc")).isel( + savepoint=0 + ) for rank in ranks: grid = TranslateGrid( dataset_to_dict(ds_grid.isel(rank=rank)), @@ -175,7 +175,7 @@ def _savepoint_cases( test_name, grid, namelist, stencil_factory ) n_calls = xr.open_dataset( - os.path.join(data_path, f"{test_name}-In.nc"), engine="h5netcdf" + os.path.join(data_path, f"{test_name}-In.nc") ).dims["savepoint"] for i_call in range(n_calls): return_list.append( diff --git a/stencils/pace/stencils/testing/savepoint.py b/stencils/pace/stencils/testing/savepoint.py index d7ba2cb9d..4ed195c54 100644 --- a/stencils/pace/stencils/testing/savepoint.py +++ b/stencils/pace/stencils/testing/savepoint.py @@ -48,10 +48,7 @@ def __str__(self): @property def ds_in(self) -> xr.Dataset: return ( - xr.open_dataset( - os.path.join(self.data_dir, f"{self.savepoint_name}-In.nc"), - engine="h5netcdf", - ) + xr.open_dataset(os.path.join(self.data_dir, f"{self.savepoint_name}-In.nc")) .isel(rank=self.rank) .isel(savepoint=self.i_call) ) @@ -60,8 +57,7 @@ def ds_in(self) -> xr.Dataset: def ds_out(self) -> xr.Dataset: return ( xr.open_dataset( - os.path.join(self.data_dir, f"{self.savepoint_name}-Out.nc"), - engine="h5netcdf", + os.path.join(self.data_dir, f"{self.savepoint_name}-Out.nc") ) .isel(rank=self.rank) .isel(savepoint=self.i_call) diff --git a/tests/savepoint/test_checkpoints.py b/tests/savepoint/test_checkpoints.py index 8b811938d..396ac5b1f 100644 --- a/tests/savepoint/test_checkpoints.py +++ b/tests/savepoint/test_checkpoints.py @@ -116,7 +116,6 @@ def test_fv_dynamics( with open(threshold_filename, "w") as f: yaml.safe_dump(dataclasses.asdict(thresholds), f) communicator.comm.barrier() - print("thresholds calibrated") with open(threshold_filename, "r") as f: data = yaml.safe_load(f) thresholds = dacite.from_dict( diff --git a/tests/savepoint/thresholds/fv_dynamics.yaml b/tests/savepoint/thresholds/fv_dynamics.yaml index 1f467640d..5f7fb29d9 100644 --- a/tests/savepoint/thresholds/fv_dynamics.yaml +++ b/tests/savepoint/thresholds/fv_dynamics.yaml @@ -7,82 +7,82 @@ savepoints: absolute: 0.0 relative: 0.0 ptd: - absolute: 2.1316282072803006e-13 - relative: 1.8510639252301478e-14 + absolute: 2.8421709430404007e-13 + relative: 1.851063925230147e-14 uad: absolute: 640.0 - relative: 2.220425172015235e-15 + relative: 2.2204251720152353e-15 ucd: absolute: 1.4210854715202004e-13 - relative: 2.2204312519237066e-15 + relative: 2.2204312519237058e-15 ud: absolute: 1.4210854715202004e-13 - relative: 2.220373305506478e-15 + relative: 2.22043357031689e-15 utd: absolute: 0.0 relative: 0.0 vad: absolute: 640.0 - relative: 2.2204408808181443e-15 + relative: 2.2204439912402017e-15 vcd: absolute: 1.4210854715202004e-13 - relative: 2.220444857175165e-15 + relative: 2.2204224462363224e-15 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204311404975443e-15 + relative: 2.2204335703168896e-15 vtd: absolute: 0.0 relative: 0.0 wd: absolute: 2.7755575615628914e-16 - relative: 2.2203205508872382e-15 + relative: 2.22040115563081e-15 C_SW-Out: - delpd: absolute: 2.2737367544323206e-12 relative: 2.220074344915848e-15 divgdd: - absolute: 8.597384414631148e-19 - relative: 1.0116858185720617e-09 + absolute: 9.232659125071874e-19 + relative: 1.0116858185822968e-09 ptd: - absolute: 2.1316282072803006e-13 - relative: 1.8510639252301478e-14 + absolute: 2.8421709430404007e-13 + relative: 1.851063925230147e-14 uad: absolute: 1280.0 - relative: 4.4531842263992984e-11 + relative: 4.45318422640128e-11 ucd: - absolute: 4.263256414560601e-13 - relative: 1.9511117370337876e-11 + absolute: 5.684341886080801e-13 + relative: 7.109669843097896e-11 ud: absolute: 1.4210854715202004e-13 - relative: 2.220373305506478e-15 + relative: 2.22043357031689e-15 utd: absolute: 3.814697265625e-05 - relative: 7.072664425264997e-11 + relative: 1.0608996637874988e-10 vad: absolute: 1280.0 - relative: 1.2738435085781666e-10 + relative: 3.782013412207662e-10 vcd: - absolute: 4.263256414560601e-13 - relative: 2.1220030303501963e-11 + absolute: 5.684341886080801e-13 + relative: 3.832773571041006e-11 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204311404975443e-15 + relative: 2.2204335703168896e-15 vtd: - absolute: 3.814697265625e-05 - relative: 1.0608819723976712e-10 + absolute: 4.76837158203125e-05 + relative: 1.7681484149211697e-10 wd: absolute: 2.7755575615628914e-16 - relative: 2.2203205508872382e-15 + relative: 2.22040115563081e-15 D_SW-In: - delpcd: - absolute: 3.814697265625e-05 - relative: 1.0608819723976712e-10 + absolute: 4.76837158203125e-05 + relative: 1.7681484149211697e-10 delpd: absolute: 2.2737367544323206e-12 relative: 2.220074344915848e-15 divgdd: - absolute: 8.597384414631148e-19 - relative: 1.0116858185720617e-09 + absolute: 9.232659125071874e-19 + relative: 1.0116858185822968e-09 mfxd: absolute: 0.0 relative: 0.0 @@ -90,29 +90,29 @@ savepoints: absolute: 0.0 relative: 0.0 ptd: - absolute: 2.1316282072803006e-13 - relative: 1.8510639252301478e-14 + absolute: 2.8421709430404007e-13 + relative: 1.851063925230147e-14 uad: absolute: 1280.0 - relative: 4.4531842263992984e-11 + relative: 4.45318422640128e-11 ucd: - absolute: 7.904787935331115e-13 - relative: 9.780251382633263e-10 + absolute: 1.0547118733938987e-12 + relative: 9.677693932620674e-10 ud: absolute: 1.4210854715202004e-13 - relative: 2.220373305506478e-15 + relative: 2.22043357031689e-15 vad: absolute: 1280.0 - relative: 1.2738435085781666e-10 + relative: 3.782013412207662e-10 vcd: - absolute: 7.904787935331115e-13 - relative: 9.780251382633263e-10 + absolute: 9.237055564881302e-13 + relative: 9.677693932620674e-10 vd: absolute: 1.4210854715202004e-13 - relative: 2.2204311404975443e-15 + relative: 2.2204335703168896e-15 wd: absolute: 2.7755575615628914e-16 - relative: 2.2203205508872382e-15 + relative: 2.22040115563081e-15 xfxd: absolute: 0.0 relative: 0.0 @@ -121,53 +121,53 @@ savepoints: relative: 0.0 zhd: absolute: 1.4551915228366852e-10 - relative: 6.52099891718502e-15 + relative: 6.276364176949949e-15 D_SW-Out: - delpcd: - absolute: 3.337860107421875e-05 - relative: 1.0116858185720617e-09 + absolute: 2.86102294921875e-05 + relative: 1.0116858185822968e-09 delpd: - absolute: 6.821210263296962e-12 - relative: 9.349923208080012e-15 + absolute: 4.6497916628140956e-10 + relative: 5.94497146377002e-13 divgdd: - absolute: 8.470329472543003e-19 - relative: 1.1893991752988745e-08 + absolute: 7.453889935837843e-19 + relative: 1.0253441564098776e-08 mfxd: - absolute: 1444413440.0 - relative: 50.0 + absolute: 1338900480.0 + relative: 80.48278074484665 mfyd: - absolute: 1264189440.0 - relative: 49.999999999999325 + absolute: 1442611200.0 + relative: 91.35608551238332 ptd: absolute: 2.8421709430404007e-13 - relative: 1.9970554019934083e-14 + relative: 1.9970554019934067e-14 uad: absolute: 1280.0 - relative: 4.4531842263992984e-11 + relative: 4.45318422640128e-11 ucd: - absolute: 5.789813073420191e-13 - relative: 2.0924792441159182e-07 + absolute: 7.283063041541027e-13 + relative: 3.104922812656015e-07 ud: - absolute: 1.4901161193847656e-07 - relative: 5.308745703431066e-12 + absolute: 2.2351741790771484e-07 + relative: 1.1748377439244054e-11 vad: absolute: 1280.0 - relative: 1.2738435085781666e-10 + relative: 3.782013412207662e-10 vcd: - absolute: 7.904787935331115e-13 - relative: 5.230529400854598e-10 + absolute: 9.237055564881302e-13 + relative: 1.1477794086270491e-09 vd: - absolute: 2.2351741790771484e-07 - relative: 3.999685149120597e-12 + absolute: 4.470348358154297e-07 + relative: 9.824464615464877e-12 wd: - absolute: 6.245004513516506e-16 - relative: 2.9515782433798676e-12 + absolute: 2.5049406993105094e-14 + relative: 2.951578243379911e-12 xfxd: - absolute: 0.00015020370483398438 - relative: 1.0950535166065047e-10 + absolute: 0.00020742416381835938 + relative: 1.4211097577339212e-10 yfxd: - absolute: 0.00011444091796875 - relative: 1.9713230836005074e-10 + absolute: 0.00016927719116210938 + relative: 5.779053256771664e-10 FVDynamics-In: - delz: absolute: 0.0 @@ -198,29 +198,204 @@ savepoints: relative: 0.0 FVDynamics-Out: - delz: - absolute: 1.3272938303998671e-10 - relative: 3.682097278337629e-12 + absolute: 1.3415046851150692e-10 + relative: 3.6847792855302725e-12 qvapor: - absolute: 4.182851981449076e-14 - relative: 1.3949257689370736e-11 + absolute: 4.2054033866367746e-14 + relative: 1.4024463640625008e-11 u: - absolute: 1.8554047187535616e-11 - relative: 2.4845358044915226e-09 + absolute: 1.782685110640614e-11 + relative: 2.168064672749613e-09 ua: absolute: 1280.0 - relative: 6.545131190335245e-10 + relative: 6.008645027189788e-10 uc: - absolute: 5.789813073420191e-13 - relative: 2.0924792441159182e-07 + absolute: 7.283063041541027e-13 + relative: 3.104922812656015e-07 v: - absolute: 1.248778858098376e-11 - relative: 2.4845358044915226e-09 + absolute: 1.782685110640614e-11 + relative: 2.168064672749613e-09 va: absolute: 1280.0 - relative: 6.175467803387584e-10 + relative: 1.398287210477588e-09 vc: - absolute: 7.904787935331115e-13 - relative: 5.230529400854598e-10 + absolute: 9.237055564881302e-13 + relative: 1.1477794086270491e-09 + w: + absolute: 1.4878549781105477e-12 + relative: 4.312240920073506e-06 + Remapping-In: + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 1.1102230246251565e-15 + relative: 3.900816513433946e-15 + delp: + absolute: 4.6497916628140956e-10 + relative: 5.94497146377002e-13 + delz: + absolute: 3.3651303965598345e-10 + relative: 3.683661360624291e-12 + dp1: + absolute: 3.0809133022557944e-10 + relative: 3.9389570619896543e-13 + omga: + absolute: 3.4139358007223564e-14 + relative: 8.082949459463172e-12 + pe: + absolute: 7.275957614183426e-10 + relative: 6.087790859263778e-14 + peln: + absolute: 5.3290705182007514e-14 + relative: 5.9240936185654696e-15 + pfull: + absolute: 0.0 + relative: 0.0 + phis: + absolute: 7.275957614183426e-11 + relative: 2.2187960775256474e-15 + pk: + absolute: 2.4868995751603507e-13 + relative: 1.766771798461583e-14 + pkz: + absolute: 3.9541703245049575e-11 + relative: 1.4744501515860232e-12 + ps: + absolute: .nan + relative: 2.202890875795688e-15 + pt: + absolute: 2.8421709430404007e-13 + relative: 1.9970545641996602e-14 + q_cld: + absolute: 1.1102230246251565e-15 + relative: 2.2202284338964917e-15 + q_con: + absolute: 6.5052130349130266e-18 + relative: 2.2547666818283325e-10 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.785738223958333e-11 + relative: 1.0911982773815251e-09 + ua: + absolute: 1280.0 + relative: 4.45318422640128e-11 + v: + absolute: 1.4406253967536031e-11 + relative: 6.898663285834368e-10 + va: + absolute: 1280.0 + relative: 3.782013412207662e-10 w: - absolute: 1.4737343290160965e-12 - relative: 2.105284830080243e-06 + absolute: 1.4880631449276649e-12 + relative: 2.4821772190637415e-06 + wsd: + absolute: 4.042339379894955e-14 + relative: 13.333333333333336 + Remapping-Out: + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 3.941291737419306e-14 + relative: 1.3813134196614175e-13 + delp: + absolute: 4.6497916628140956e-10 + relative: 5.94497146377002e-13 + delz: + absolute: 1.3415046851150692e-10 + relative: 3.6847792855302725e-12 + dp1: + absolute: 3.0809133022557944e-10 + relative: 3.9389570619896543e-13 + omga: + absolute: 3.4139358007223564e-14 + relative: 8.082949459463172e-12 + pe: + absolute: 7.275957614183426e-10 + relative: 6.087790859263778e-14 + peln: + absolute: 1.7763568394002505e-14 + relative: 1.8957631801002588e-15 + pfull: + absolute: 0.0 + relative: 0.0 + pk: + absolute: 2.4868995751603507e-13 + relative: 1.0475949452990515e-14 + pkz: + absolute: 4.565237077258644e-11 + relative: 1.7728870714067408e-12 + ps: + absolute: .nan + relative: 7.109756861194954e-15 + pt: + absolute: 4.4565240386873484e-10 + relative: 1.4884396993752746e-12 + q_cld: + absolute: 8.189615652298698e-11 + relative: 8.394109519297739e-07 + q_con: + absolute: 4.205864172560081e-14 + relative: 100.0 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.782685110640614e-11 + relative: 2.168064672749613e-09 + ua: + absolute: 1280.0 + relative: 4.45318422640128e-11 + v: + absolute: 1.4386269953092778e-11 + relative: 1.0838326966966584e-09 + va: + absolute: 1280.0 + relative: 3.782013412207662e-10 + w: + absolute: 1.4878549781105477e-12 + relative: 4.312240920073506e-06 + wsd: + absolute: 4.042339379894955e-14 + relative: 13.333333333333336 + Tracer2D1L-In: + - cxd: + absolute: 2.8080836267374565e-16 + relative: 1.4211029960774323e-10 + cyd: + absolute: 2.7755575615628914e-16 + relative: 5.779055931960873e-10 + dp1: + absolute: 2.2737367544323206e-12 + relative: 2.220074344915848e-15 + mfxd: + absolute: 1338900480.0 + relative: 80.48278074484665 + mfyd: + absolute: 1442611200.0 + relative: 91.35608551238332 + Tracer2D1L-Out: + - cxd: + absolute: 9.351243737687476e-17 + relative: 1.4210965628361645e-10 + cyd: + absolute: 8.673617379884035e-17 + relative: 5.779062322488002e-10 + dp1: + absolute: 3.0809133022557944e-10 + relative: 3.9389570619896543e-13 + mfxd: + absolute: 446300160.0 + relative: 80.48278074484668 + mfyd: + absolute: 480870400.0 + relative: 91.35608551238332 diff --git a/util/pace/util/checkpointer/validation.py b/util/pace/util/checkpointer/validation.py index 34059dda5..ab250fe17 100644 --- a/util/pace/util/checkpointer/validation.py +++ b/util/pace/util/checkpointer/validation.py @@ -121,13 +121,18 @@ def __call__(self, savepoint_name: str, **kwargs: ArrayLike) -> None: # cannot use relative threshold when comparing to zero value expected_not_zero = expected != 0 - np.testing.assert_allclose( - output[expected_not_zero], - expected[expected_not_zero], - rtol=var_thresholds[varname].relative, - ) - - np.testing.assert_allclose( - output, expected, atol=var_thresholds[varname].absolute - ) + rtol = var_thresholds[varname].relative + atol = var_thresholds[varname].absolute + if not np.isnan(rtol): + np.testing.assert_allclose( + output[expected_not_zero], + expected[expected_not_zero], + rtol=var_thresholds[varname].relative, + atol=0.0, + ) + + if not np.isnan(atol): + np.testing.assert_allclose( + output, expected, atol=var_thresholds[varname].absolute, rtol=0.0 + ) self._n_calls[savepoint_name] += 1 From 5c5989596e51dca8e1e3d3bbb58151edb592d941 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Wed, 28 Sep 2022 13:10:37 -0700 Subject: [PATCH 39/48] physics checkpointer init --- physics/pace/physics/stencils/physics.py | 28 +++- stencils/pace/stencils/fv_update_phys.py | 30 ++++ stencils/pace/stencils/update_atmos_state.py | 7 + tests/savepoint/test_checkpoints.py | 144 ++++++++++++++++++- util/pace/util/testing/perturbation.py | 18 ++- 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 4896d8f26..1b2599188 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -1,3 +1,4 @@ +import typing from typing import List import gt4py.gtscript as gtscript @@ -5,6 +6,7 @@ from typing_extensions import Literal import pace.dsl.gt4py_utils as utils +import pace.util import pace.util.constants as constants from pace.dsl.dace.orchestration import orchestrate from pace.dsl.stencil import StencilFactory @@ -175,7 +177,12 @@ def __init__( grid_data: GridData, namelist: PhysicsConfig, active_packages: List[Literal[PHYSICS_PACKAGES]], + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, @@ -247,7 +254,26 @@ def _setup_statein(self): self._p00 = 1.0e5 def __call__(self, physics_state: PhysicsState, timestep: float): - + if self._call_checkpointer: + self._checkpointer( + "GFSPhysicsDriver-In", + qvapor=physics_state.qvapor, + qliquid=physics_state.qliquid, + qrain=physics_state.qrain, + qsnow=physics_state.qsnow, + qice=physics_state.qice, + qgraupel=physics_state.qgraupel, + qo3mr=physics_state.qo3mr, + qsgs_tke=physics_state.qsgs_tke, + qcld=physics_state.qcld, + pt=physics_state.pt, + delp=physics_state.delp, + delz=physics_state.delz, + ua=physics_state.ua, + va=physics_state.va, + w=physics_state.w, + omga=physics_state.omga, + ) self._atmos_phys_driver_statein( self._prsik, physics_state.phii, diff --git a/stencils/pace/stencils/fv_update_phys.py b/stencils/pace/stencils/fv_update_phys.py index 9da8299f5..3aa5b5d8f 100644 --- a/stencils/pace/stencils/fv_update_phys.py +++ b/stencils/pace/stencils/fv_update_phys.py @@ -1,3 +1,5 @@ +import typing + import gt4py.gtscript as gtscript from gt4py.gtscript import FORWARD, PARALLEL, computation, exp, interval, log @@ -91,7 +93,12 @@ def __init__( state: fv3core.DycoreState, u_dt: pace.util.Quantity, v_dt: pace.util.Quantity, + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, @@ -151,6 +158,29 @@ def __call__( t_dt, dt: float, ): + if self._call_checkpointer: + self._checkpointer( + "FVUpdatePhys-In", + u_dt=u_dt, + v_dt=v_dt, + t_dt=t_dt, + ua=state.ua, + va=state.va, + u=state.u, + v=state.v, + qvapor=state.qvapor, + qliquid=state.qliquid, + qice=state.qice, + qrain=state.qrain, + qsnow=state.qsnow, + qgraupel=state.qgraupel, + peln=state.peln, + delp=state.delp, + pt=state.pt, + ps=state.ps, + pe=state.pe, + pk=state.pk, + ) self._moist_cv( state.qvapor, state.qliquid, diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index f7deb6529..7e3852ac1 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -1,3 +1,4 @@ +import typing from typing import Optional from gt4py.gtscript import BACKWARD, FORWARD, PARALLEL, computation, interval @@ -247,7 +248,12 @@ def __init__( dycore_only: bool, apply_tendencies: bool, tendency_state, + checkpointer: typing.Optional[pace.util.Checkpointer] = None, ): + self._checkpointer = checkpointer + # this is only computed in init because Dace does not yet support + # this operation + self._call_checkpointer = checkpointer is not None orchestrate( obj=self, config=stencil_factory.config.dace_config, @@ -287,6 +293,7 @@ def __init__( state, tendency_state.u_dt, tendency_state.v_dt, + checkpointer=checkpointer, ) self._dycore_only = dycore_only # apply_tendencies when we have run physics or fv_subgridz diff --git a/tests/savepoint/test_checkpoints.py b/tests/savepoint/test_checkpoints.py index 396ac5b1f..6eeeff2f5 100644 --- a/tests/savepoint/test_checkpoints.py +++ b/tests/savepoint/test_checkpoints.py @@ -11,8 +11,10 @@ import pace.dsl import pace.util from pace import fv3core +from pace.driver.state import TendencyState from pace.fv3core.initialization.dycore_state import DycoreState from pace.fv3core.testing.translate_fvdynamics import TranslateFVDynamics +from pace.stencils import update_atmos_state from pace.stencils.testing import TranslateGrid, dataset_to_dict from pace.stencils.testing.grid import Grid from pace.util.checkpointer.thresholds import SavepointThresholds @@ -109,7 +111,7 @@ def test_fv_dynamics( damping_coefficients=grid.damping_coefficients, dycore_config=dycore_config, n_trials=10, - factor=10.0, + factor=12.0, ) print(f"calibrated thresholds: {thresholds}") if communicator.rank == 0: @@ -142,6 +144,93 @@ def test_fv_dynamics( dycore.step_dynamics(state) +def test_driver( + backend: str, + data_path: str, + calibrate_thresholds: bool, + threshold_path: str, +): + print("start test call") + namelist = pace.util.Namelist.from_f90nml( + f90nml.read(os.path.join(data_path, "input.nml")) + ) + threshold_filename = os.path.join(threshold_path, "driver.yaml") + communicator = pace.util.CubedSphereCommunicator( + comm=pace.util.MPIComm(), + partitioner=pace.util.CubedSpherePartitioner( + tile=pace.util.TilePartitioner(layout=namelist.layout) + ), + ) + sizer = pace.util.SubtileGridSizer.from_tile_params( + nx_tile=namelist.npx - 1, + ny_tile=namelist.npy - 1, + nz=namelist.npz, + n_halo=3, + tile_partitioner=communicator.partitioner.tile, + tile_rank=communicator.rank, + extra_dim_lengths={}, + layout=namelist.layout, + ) + stencil_factory = pace.dsl.StencilFactory( + config=pace.dsl.StencilConfig( + compilation_config=pace.dsl.CompilationConfig( + backend=backend, + communicator=communicator, + rebuild=False, + ) + ), + grid_indexing=pace.dsl.GridIndexing.from_sizer_and_communicator( + sizer=sizer, + cube=communicator, + ), + ) + quantity_factory = pace.util.QuantityFactory.from_backend(sizer, backend=backend) + grid = get_grid( + data_path=data_path, + rank=communicator.rank, + layout=namelist.layout, + backend=backend, + ) + driver_grid_data = grid.driver_grid_data + translate = TranslateFVDynamics( + grid=grid, namelist=namelist, stencil_factory=stencil_factory + ) + ds = xr.open_dataset(os.path.join(data_path, "Driver-In.nc")).sel( + savepoint=0, rank=communicator.rank + ) + dycore_config = fv3core.DynamicalCoreConfig.from_namelist(namelist) + physics_config = pace.physics.PhysicsConfig.from_namelist(namelist) + initializer = StateInitializer( + ds, + translate, + ) + if calibrate_thresholds: + thresholds = _calibrate_thresholds( + initializer=initializer, + communicator=communicator, + stencil_factory=stencil_factory, + damping_coefficients=grid.damping_coefficients, + dycore_config=dycore_config, + n_trials=10, + factor=12.0, + physics_config=physics_config, + quantity_factory=quantity_factory, + driver_grid_data=driver_grid_data, + ) + print(f"calibrated thresholds: {thresholds}") + if communicator.rank == 0: + with open(threshold_filename, "w") as f: + yaml.safe_dump(dataclasses.asdict(thresholds), f) + communicator.comm.barrier() + with open(threshold_filename, "r") as f: + data = yaml.safe_load(f) + thresholds = dacite.from_dict( + data_class=pace.util.SavepointThresholds, + data=data, + config=dacite.Config(strict=True), + ) + + def _calibrate_thresholds( initializer: StateInitializer, communicator: pace.util.CubedSphereCommunicator, @@ -150,6 +239,9 @@ def _calibrate_thresholds( dycore_config: fv3core.DynamicalCoreConfig, n_trials: int, factor: float, + physics_config: pace.physics.PhysicsConfig = None, + quantity_factory: pace.util.QuantityFactory = None, + driver_grid_data: pace.util.grid.DriverGridData = None, ): calibration = pace.util.ThresholdCalibrationCheckpointer(factor=factor) for i in range(n_trials): @@ -169,8 +261,58 @@ def _calibrate_thresholds( checkpointer=calibration, timestep=timedelta(seconds=dycore_config.dt_atmos), ) + if physics_config is not None: + physics = pace.physics.Physics( + stencil_factory=stencil_factory, + grid_data=grid_data, + namelist=physics_config, + active_packages=["microphysics"], + checkpointer=calibration, + ) + dycore_to_physics = update_atmos_state.DycoreToPhysics( + stencil_factory=stencil_factory, + dycore_config=dycore_config, + do_dry_convective_adjust=dycore_config.do_dry_convective_adjustment, + dycore_only=False, + ) + tendency_state = TendencyState.init_zeros( + quantity_factory=quantity_factory, + ) + end_of_step_update = update_atmos_state.UpdateAtmosphereState( + stencil_factory=stencil_factory, + grid_data=grid_data, + namelist=physics_config, + comm=communicator, + grid_info=driver_grid_data, + state=trial_state, + quantity_factory=quantity_factory, + dycore_only=False, + apply_tendencies=True, + tendency_state=tendency_state, + checkpointer=calibration, + ) + trial_physics_state = pace.physics.PhysicsState.init_zeros( + quantity_factory=quantity_factory, + active_packages=["microphysics"], + ) with calibration.trial(): dycore.step_dynamics(trial_state) + if physics_config is not None: + dycore_to_physics( + dycore_state=trial_state, + physics_state=trial_physics_state, + tendency_state=tendency_state, + timestep=float(physics_config.dt_atmos), + ) + physics(trial_physics_state, timestep=float(physics_config.dt_atmos)) + end_of_step_update( + dycore_state=trial_state, + phy_state=trial_physics_state, + u_dt=tendency_state.u_dt.storage, + v_dt=tendency_state.v_dt.storage, + pt_dt=tendency_state.pt_dt.storage, + dt=float(physics_config.dt_atmos), + ) all_thresholds = communicator.comm.allgather(calibration.thresholds) thresholds = merge_thresholds(all_thresholds) set_manual_thresholds(thresholds) diff --git a/util/pace/util/testing/perturbation.py b/util/pace/util/testing/perturbation.py index 25e423028..e28a5f0f9 100644 --- a/util/pace/util/testing/perturbation.py +++ b/util/pace/util/testing/perturbation.py @@ -10,10 +10,20 @@ def perturb(input: Mapping[str, np.ndarray]): Will only make changes to float64 or float32 arrays. """ roundoff = 1e-16 - for data in input.values(): + for name, data in input.items(): if isinstance(data, np.ndarray) and data.dtype in (np.float64, np.float32): - not_fill_value = data < 1e30 + valid_points = data < 1e30 + if name in [ + "q_con", + "qcld", + "qice", + "qliquid", + "qrain", + "qsnow", + "qgraupel", + ]: + valid_points = np.where((data < 1e30) & (data > 0)) # multiply data by roundoff-level error - data[not_fill_value] *= 1.0 + np.random.uniform( - low=-roundoff, high=roundoff, size=data[not_fill_value].shape + data[valid_points] *= 1.0 + np.random.uniform( + low=-roundoff, high=roundoff, size=data[valid_points].shape ) From 5ea06df08d20234f4aa4cee0a38b476528ed5a1a Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Wed, 28 Sep 2022 15:43:20 -0700 Subject: [PATCH 40/48] update phys and driver out --- stencils/pace/stencils/update_atmos_state.py | 31 ++ tests/savepoint/thresholds/driver.yaml | 549 +++++++++++++++++++ 2 files changed, 580 insertions(+) create mode 100644 tests/savepoint/thresholds/driver.yaml diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index 7e3852ac1..80ea3be84 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -354,3 +354,34 @@ def __call__( pt_dt, dt=dt, ) + if self._call_checkpointer: + self._checkpointer( + "FVUpdatePhys-Out", + qvapor=dycore_state.qvapor, + qliquid=dycore_state.qliquid, + qrain=dycore_state.qrain, + qsnow=dycore_state.qsnow, + qice=dycore_state.qice, + qgraupel=dycore_state.qgraupel, + pt=dycore_state.pt, + ua=dycore_state.ua, + va=dycore_state.va, + u=dycore_state.u, + v=dycore_state.v, + ) + self._checkpointer( + "Driver-Out", + qvapor=dycore_state.qvapor, + qliquid=dycore_state.qliquid, + qrain=dycore_state.qrain, + qsnow=dycore_state.qsnow, + qice=dycore_state.qice, + qgraupel=dycore_state.qgraupel, + pt=dycore_state.pt, + ua=dycore_state.ua, + va=dycore_state.va, + u=dycore_state.u, + v=dycore_state.v, + delz=dycore_state.delz, + delp=dycore_state.delp, + ) diff --git a/tests/savepoint/thresholds/driver.yaml b/tests/savepoint/thresholds/driver.yaml new file mode 100644 index 000000000..28436d40b --- /dev/null +++ b/tests/savepoint/thresholds/driver.yaml @@ -0,0 +1,549 @@ +savepoints: + C_SW-In: + - delpd: + absolute: 2.7284841053187847e-12 + relative: 2.664392791389508e-15 + divgdd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.3048355003642202e-14 + uad: + absolute: 192.0 + relative: 2.6645228139724058e-15 + ucd: + absolute: 4.263256414560601e-14 + relative: 2.664479239525582e-15 + ud: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + utd: + absolute: 0.0 + relative: 0.0 + vad: + absolute: 192.0 + relative: 2.6645146823245013e-15 + vcd: + absolute: 4.263256414560601e-14 + relative: 2.664530705451025e-15 + vd: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + vtd: + absolute: 0.0 + relative: 0.0 + wd: + absolute: 8.326672684688674e-17 + relative: 2.664506596135002e-15 + C_SW-Out: + - delpd: + absolute: 2.7284841053187847e-12 + relative: 2.664392791389508e-15 + divgdd: + absolute: 4.777265822514254e-19 + relative: 3.925412056700377e-09 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.3048355003642202e-14 + uad: + absolute: 384.0 + relative: 1.792767371402652e-09 + ucd: + absolute: 1.7053025658242404e-13 + relative: 5.929879004969384e-11 + ud: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + utd: + absolute: 2.002716064453125e-05 + relative: 3.7052523913230544e-11 + vad: + absolute: 384.0 + relative: 1.792767371402652e-09 + vcd: + absolute: 1.7053025658242404e-13 + relative: 4.054757865553553e-10 + vd: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + vtd: + absolute: 2.002716064453125e-05 + relative: 1.8398302626266127e-09 + wd: + absolute: 8.326672684688674e-17 + relative: 2.664506596135002e-15 + D_SW-In: + - delpcd: + absolute: 2.002716064453125e-05 + relative: 1.8398302626266127e-09 + delpd: + absolute: 2.7284841053187847e-12 + relative: 2.664392791389508e-15 + divgdd: + absolute: 4.777265822514254e-19 + relative: 3.925412056700377e-09 + mfxd: + absolute: 0.0 + relative: 0.0 + mfyd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.3048355003642202e-14 + uad: + absolute: 384.0 + relative: 1.792767371402652e-09 + ucd: + absolute: 7.354117315117037e-13 + relative: 7.45716960892069e-09 + ud: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + vad: + absolute: 384.0 + relative: 1.792767371402652e-09 + vcd: + absolute: 7.354117315117037e-13 + relative: 2.1527364019830838e-07 + vd: + absolute: 4.263256414560601e-14 + relative: 2.6645224906037465e-15 + wd: + absolute: 8.326672684688674e-17 + relative: 2.664506596135002e-15 + xfxd: + absolute: 0.0 + relative: 0.0 + yfxd: + absolute: 0.0 + relative: 0.0 + zhd: + absolute: 1.7462298274040222e-10 + relative: 8.274629575761722e-15 + D_SW-Out: + - delpcd: + absolute: 1.71661376953125e-05 + relative: 3.925412056700377e-09 + delpd: + absolute: 5.4569682106375694e-12 + relative: 5.29997413501075e-15 + divgdd: + absolute: 3.595654861094505e-19 + relative: 4.5304154135537483e-08 + mfxd: + absolute: 1628995584.0 + relative: 120.0 + mfyd: + absolute: 1628504064.0 + relative: 120.0 + ptd: + absolute: 4.263256414560601e-13 + relative: 2.4821302035387395e-14 + uad: + absolute: 384.0 + relative: 1.792767371402652e-09 + ucd: + absolute: 7.354117315117037e-13 + relative: 2.389973474778064e-07 + ud: + absolute: 1.341104507446289e-07 + relative: 3.788836253752432e-11 + vad: + absolute: 384.0 + relative: 1.792767371402652e-09 + vcd: + absolute: 7.354117315117037e-13 + relative: 2.389973509569915e-07 + vd: + absolute: 1.341104507446289e-07 + relative: 3.718147277266473e-11 + wd: + absolute: 1.6653345369377348e-16 + relative: 1.4341582375196082e-12 + xfxd: + absolute: 0.0001087188720703125 + relative: 7.457167648421754e-09 + yfxd: + absolute: 0.0001087188720703125 + relative: 2.1527364273173495e-07 + Driver-Out: + - delp: + absolute: 1.800799509510398e-10 + relative: 5.022117662087534e-13 + delz: + absolute: 25769803776.0 + relative: 4.420922764818589e-12 + pt: + absolute: 5.368292477214709e-10 + relative: 1.7778722690158857e-12 + qgraupel: + absolute: 4.0925587986673597e-19 + relative: 120.0 + qice: + absolute: 4.124890787417885e-15 + relative: 123.0 + qliquid: + absolute: 4.268655741379579e-14 + relative: 6.6325478948235934e-09 + qrain: + absolute: 2.5347384713619685e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 4.2455622351056377e-14 + relative: 8.685658162172946e-12 + u: + absolute: 1.538769112130467e-12 + relative: 2.718327538654071e-09 + ua: + absolute: 384.0 + relative: 1.792767371402652e-09 + v: + absolute: 1.538769112130467e-12 + relative: 5.041619359370808e-09 + va: + absolute: 384.0 + relative: 6.743131502955869e-08 + FVDynamics-In: + - delz: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 0.0 + relative: 0.0 + u: + absolute: 0.0 + relative: 0.0 + ua: + absolute: 0.0 + relative: 0.0 + uc: + absolute: 0.0 + relative: 0.0 + v: + absolute: 0.0 + relative: 0.0 + va: + absolute: 0.0 + relative: 0.0 + vc: + absolute: 0.0 + relative: 0.0 + w: + absolute: 0.0 + relative: 0.0 + FVDynamics-Out: + - delz: + absolute: 25769803776.0 + relative: 4.420922764818589e-12 + qvapor: + absolute: 4.2289088897362603e-14 + relative: 8.651547729736657e-12 + u: + absolute: 1.5384360452230794e-12 + relative: 2.718327538654071e-09 + ua: + absolute: 384.0 + relative: 1.792767371402652e-09 + uc: + absolute: 7.354117315117037e-13 + relative: 2.389973474778064e-07 + v: + absolute: 1.5384360452230794e-12 + relative: 5.041619359370809e-09 + va: + absolute: 384.0 + relative: 6.743131502955869e-08 + vc: + absolute: 7.354117315117037e-13 + relative: 2.389973509569915e-07 + w: + absolute: 2.0960316815532565e-12 + relative: 4.640618151764116e-06 + FVUpdatePhys-In: + - delp: + absolute: 1.800799509510398e-10 + relative: 5.022117662087534e-13 + pe: + absolute: 5.238689482212067e-10 + relative: 6.9515212013449995e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2349241361126896e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2799614024119294e-14 + ps: + absolute: .nan + relative: 5.365436857202281e-15 + pt: + absolute: 5.368292477214709e-10 + relative: 1.7778722690158857e-12 + qgraupel: + absolute: 4.0925587986673597e-19 + relative: 120.0 + qice: + absolute: 4.124890787417885e-15 + relative: 123.0 + qliquid: + absolute: 4.268655741379579e-14 + relative: 6.6325478948235934e-09 + qrain: + absolute: 2.5347384713619685e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 4.2455622351056377e-14 + relative: 8.685658162172946e-12 + t_dt: + absolute: 4.244309981596417e-14 + relative: 5.047263788555737e-08 + u: + absolute: 1.5384360452230794e-12 + relative: 2.718327538654071e-09 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + ua: + absolute: 384.0 + relative: 1.792767371402652e-09 + v: + absolute: 1.5384360452230794e-12 + relative: 5.041619359370809e-09 + v_dt: + absolute: 4.736951571734001e-17 + relative: 120.0 + va: + absolute: 384.0 + relative: 6.743131502955869e-08 + FVUpdatePhys-Out: + - pt: + absolute: 5.368292477214709e-10 + relative: 1.7778722690158857e-12 + qgraupel: + absolute: 4.0925587986673597e-19 + relative: 120.0 + qice: + absolute: 4.124890787417885e-15 + relative: 123.0 + qliquid: + absolute: 4.268655741379579e-14 + relative: 6.6325478948235934e-09 + qrain: + absolute: 2.5347384713619685e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 4.2455622351056377e-14 + relative: 8.685658162172946e-12 + u: + absolute: 1.538769112130467e-12 + relative: 2.718327538654071e-09 + ua: + absolute: 384.0 + relative: 1.792767371402652e-09 + v: + absolute: 1.538769112130467e-12 + relative: 5.041619359370808e-09 + va: + absolute: 384.0 + relative: 6.743131502955869e-08 + GFSPhysicsDriver-In: + - delp: + absolute: 1.7462298274040222e-10 + relative: 5.022117662087535e-13 + delz: + absolute: 25769803776.0 + relative: 4.420922764818589e-12 + omga: + absolute: 9.002975448479944e-12 + relative: 6.166103011315274e-05 + pt: + absolute: 5.368292477214709e-10 + relative: 1.7778722690158857e-12 + qcld: + absolute: 8.528644457328483e-11 + relative: 4.544796922758549e-08 + qgraupel: + absolute: 5.11554960582394e-19 + relative: 120.0 + qice: + absolute: 9.636630736957605e-15 + relative: 120.02930402930403 + qliquid: + absolute: 4.230372562669116e-14 + relative: 6.613159352359565e-09 + qo3mr: + absolute: 0.0 + relative: 0.0 + qrain: + absolute: 5.234663614031576e-19 + relative: 5.676126564496927e-05 + qsgs_tke: + absolute: 0.0 + relative: 0.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 4.2289088897362603e-14 + relative: 8.651547729736657e-12 + ua: + absolute: 5.238689482212067e-10 + relative: 3.8614993901561815e-10 + va: + absolute: 1.3672396548258803e-12 + relative: 6.743131502955869e-08 + w: + absolute: 2.0960316815532565e-12 + relative: 4.640618151764116e-06 + Remapping-In: + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 1.3322676295501878e-15 + relative: 4.684802402122118e-15 + delp: + absolute: 5.4569682106375694e-12 + relative: 5.29997413501075e-15 + delz: + absolute: 25769803776.0 + relative: 4.417755979333718e-12 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.816113018901345e-15 + omga: + absolute: 2.914335439641036e-16 + relative: 2.69741183575927e-12 + pe: + absolute: 5.238689482212067e-10 + relative: 7.012922165955118e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.642154438487291e-15 + pfull: + absolute: 0.0 + relative: 0.0 + phis: + absolute: 0.0 + relative: 0.0 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2610043551750094e-14 + pkz: + absolute: 4.7066350816749036e-11 + relative: 1.7659895624560728e-12 + ps: + absolute: .nan + relative: 1.7916862372061497e-15 + pt: + absolute: 3.410605131648481e-13 + relative: 2.4821302030748827e-14 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.544597783009749e-12 + relative: 2.3499241013589115e-09 + ua: + absolute: 384.0 + relative: 1.792767371402652e-09 + v: + absolute: 1.0764722446765518e-12 + relative: 5.596522336615699e-09 + va: + absolute: 384.0 + relative: 1.792767371402652e-09 + w: + absolute: 2.095906781462986e-12 + relative: 1.320885985901664e-05 + wsd: + absolute: 0.0 + relative: 0.0 + Remapping-Out: + - cappa: + absolute: 3.863576125695545e-14 + relative: 1.354653976534733e-13 + delp: + absolute: 1.7462298274040222e-10 + relative: 5.022117662087535e-13 + delz: + absolute: 25769803776.0 + relative: 4.420922764818589e-12 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.816113018901345e-15 + omga: + absolute: 2.914335439641036e-16 + relative: 2.69741183575927e-12 + pe: + absolute: 5.238689482212067e-10 + relative: 6.9515212013449995e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2349241361126896e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2799614024119294e-14 + pkz: + absolute: 4.7194248509185854e-11 + relative: 1.7700916431128049e-12 + pt: + absolute: 5.368292477214709e-10 + relative: 1.7778722690158857e-12 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.5384360452230794e-12 + relative: 2.4055998841126902e-09 + v: + absolute: 1.0764722446765518e-12 + relative: 5.041619359370809e-09 + w: + absolute: 2.0960316815532565e-12 + relative: 4.640618151764116e-06 + Tracer2D1L-In: + - cxd: + absolute: 2.3418766925686896e-16 + relative: 7.457167570200806e-09 + cyd: + absolute: 2.3418766925686896e-16 + relative: 2.1527364253231465e-07 + dp1: + absolute: 2.7284841053187847e-12 + relative: 2.664392791389508e-15 + mfxd: + absolute: 1628995584.0 + relative: 120.0 + mfyd: + absolute: 1628504064.0 + relative: 120.0 + Tracer2D1L-Out: + - cxd: + absolute: 7.806255641895632e-17 + relative: 7.45716661757556e-09 + cyd: + absolute: 7.806255641895632e-17 + relative: 2.1527364135800607e-07 + dp1: + absolute: 8.185452315956354e-12 + relative: 7.816113018901345e-15 + mfxd: + absolute: 543006720.0 + relative: 120.0 + mfyd: + absolute: 542834688.0 + relative: 120.0 From 538d94322e559d9bd7260dc516cdca0e1dabfa8c Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Fri, 30 Sep 2022 16:43:24 -0700 Subject: [PATCH 41/48] physics checkpoint working for 1 step --- fv3core/pace/fv3core/stencils/dyn_core.py | 13 ++- physics/pace/physics/stencils/physics.py | 3 + stencils/pace/stencils/fv_update_phys.py | 7 +- stencils/pace/stencils/update_atmos_state.py | 43 ++++---- tests/savepoint/test_checkpoints.py | 107 +++++++++++++++---- 5 files changed, 125 insertions(+), 48 deletions(-) diff --git a/fv3core/pace/fv3core/stencils/dyn_core.py b/fv3core/pace/fv3core/stencils/dyn_core.py index 6cf9a4188..f1e76e4a7 100644 --- a/fv3core/pace/fv3core/stencils/dyn_core.py +++ b/fv3core/pace/fv3core/stencils/dyn_core.py @@ -1,6 +1,5 @@ from typing import Dict, Mapping, Optional, Sequence, Tuple -from dace.frontend.python.interface import nounroll as dace_nounroll from gt4py.gtscript import ( __INLINED, BACKWARD, @@ -24,6 +23,7 @@ import pace.util import pace.util as fv3util import pace.util.constants as constants +from dace.frontend.python.interface import nounroll as dace_nounroll from pace.dsl.dace.orchestration import dace_inhibitor, orchestrate from pace.dsl.dace.wrapped_halo_exchange import WrappedHaloUpdater from pace.dsl.stencil import GridIndexing, StencilFactory @@ -682,6 +682,11 @@ def _get_da_min(self) -> float: def _checkpoint_csw(self, state: DycoreState, tag: str): if self.call_checkpointer: + if tag == "In": + # need this to compare with Fortran + self._ut[:] = 0.0 + self._vt[:] = 0.0 + self._divgd.data[:] = 0.0 self.checkpointer( f"C_SW-{tag}", delpd=state.delp, @@ -700,6 +705,8 @@ def _checkpoint_csw(self, state: DycoreState, tag: str): def _checkpoint_dsw_in(self, state: DycoreState): if self.call_checkpointer: + self._xfx[:] = 0.0 + self._yfx[:] = 0.0 self.checkpointer( "D_SW-In", ucd=state.uc, @@ -713,8 +720,8 @@ def _checkpoint_dsw_in(self, state: DycoreState): ptd=state.pt, uad=state.ua, vad=state.va, - zhd=self._zh, - divgdd=self._divgd, + # zhd=self._zh, + # divgdd=self._divgd, xfxd=self._xfx, yfxd=self._yfx, mfxd=state.mfxd, diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 1b2599188..412b772ad 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -43,6 +43,9 @@ def atmos_phys_driver_statein( ): from __externals__ import nwat, pk0inv, pktop, ptop + with computation(BACKWARD), interval(...): + phii = 0.0 + with computation(BACKWARD), interval(0, -1): phii = phii[0, 0, 1] - delz * constants.GRAV diff --git a/stencils/pace/stencils/fv_update_phys.py b/stencils/pace/stencils/fv_update_phys.py index 3aa5b5d8f..b560d8d7a 100644 --- a/stencils/pace/stencils/fv_update_phys.py +++ b/stencils/pace/stencils/fv_update_phys.py @@ -13,6 +13,7 @@ from pace.dsl.typing import Float, FloatField, FloatFieldIJ from pace.stencils.c2l_ord import CubedToLatLon from pace.stencils.update_dwind_phys import AGrid2DGridPhysics +from pace.util import X_DIM, Y_DIM, Z_INTERFACE_DIM from pace.util.grid import DriverGridData, GridData @@ -174,11 +175,13 @@ def __call__( qrain=state.qrain, qsnow=state.qsnow, qgraupel=state.qgraupel, - peln=state.peln, + peln=state.peln.transpose( + [X_DIM, Z_INTERFACE_DIM, Y_DIM] + ), # [x, z, y] fortran data, delp=state.delp, pt=state.pt, ps=state.ps, - pe=state.pe, + pe=state.pe.transpose([X_DIM, Z_INTERFACE_DIM, Y_DIM]), pk=state.pk, ) self._moist_cv( diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index 80ea3be84..bae4750b4 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -159,15 +159,10 @@ def __init__( config=stencil_factory.config.dace_config, dace_compiletime_args=["dycore_state", "physics_state", "tendency_state"], ) - - self._copy_dycore_to_physics = stencil_factory.from_dims_halo( + self._copy_dycore_to_physics = stencil_factory.from_origin_domain( copy_dycore_to_physics, - compute_dims=[ - pace.util.X_INTERFACE_DIM, - pace.util.Y_INTERFACE_DIM, - pace.util.Z_INTERFACE_DIM, - ], - compute_halos=(0, 0), + origin=stencil_factory.grid_indexing.origin_full(), + domain=stencil_factory.grid_indexing.domain_full(add=(0, 0, 1)), ) self._do_dry_convective_adjustment = do_dry_convective_adjust self._dycore_only = dycore_only @@ -369,19 +364,19 @@ def __call__( u=dycore_state.u, v=dycore_state.v, ) - self._checkpointer( - "Driver-Out", - qvapor=dycore_state.qvapor, - qliquid=dycore_state.qliquid, - qrain=dycore_state.qrain, - qsnow=dycore_state.qsnow, - qice=dycore_state.qice, - qgraupel=dycore_state.qgraupel, - pt=dycore_state.pt, - ua=dycore_state.ua, - va=dycore_state.va, - u=dycore_state.u, - v=dycore_state.v, - delz=dycore_state.delz, - delp=dycore_state.delp, - ) + # self._checkpointer( + # "Driver-Out", + # qvapor=dycore_state.qvapor, + # qliquid=dycore_state.qliquid, + # qrain=dycore_state.qrain, + # qsnow=dycore_state.qsnow, + # qice=dycore_state.qice, + # qgraupel=dycore_state.qgraupel, + # pt=dycore_state.pt, + # ua=dycore_state.ua, + # va=dycore_state.va, + # u=dycore_state.u, + # v=dycore_state.v, + # delz=dycore_state.delz, + # delp=dycore_state.delp, + # ) diff --git a/tests/savepoint/test_checkpoints.py b/tests/savepoint/test_checkpoints.py index 6eeeff2f5..1de85b9c7 100644 --- a/tests/savepoint/test_checkpoints.py +++ b/tests/savepoint/test_checkpoints.py @@ -195,7 +195,7 @@ def test_driver( translate = TranslateFVDynamics( grid=grid, namelist=namelist, stencil_factory=stencil_factory ) - ds = xr.open_dataset(os.path.join(data_path, "Driver-In.nc")).sel( + ds = xr.open_dataset(os.path.join(data_path, "FVDynamics-In.nc")).sel( savepoint=0, rank=communicator.rank ) dycore_config = fv3core.DynamicalCoreConfig.from_namelist(namelist) @@ -229,6 +229,72 @@ def test_driver( data=data, config=dacite.Config(strict=True), ) + validation = pace.util.ValidationCheckpointer( + savepoint_data_path=data_path, thresholds=thresholds, rank=communicator.rank + ) + state, grid_data = initializer.new_state() + dycore = fv3core.DynamicalCore( + comm=communicator, + grid_data=grid_data, + stencil_factory=stencil_factory, + damping_coefficients=grid.damping_coefficients, + config=dycore_config, + phis=state.phis, + state=state, + checkpointer=validation, + timestep=timedelta(seconds=dycore_config.dt_atmos), + ) + physics = pace.physics.Physics( + stencil_factory=stencil_factory, + grid_data=grid_data, + namelist=physics_config, + active_packages=["microphysics"], + checkpointer=validation, + ) + dycore_to_physics = update_atmos_state.DycoreToPhysics( + stencil_factory=stencil_factory, + dycore_config=dycore_config, + do_dry_convective_adjust=dycore_config.do_dry_convective_adjustment, + dycore_only=False, + ) + tendency_state = TendencyState.init_zeros( + quantity_factory=quantity_factory, + ) + end_of_step_update = update_atmos_state.UpdateAtmosphereState( + stencil_factory=stencil_factory, + grid_data=grid_data, + namelist=physics_config, + comm=communicator, + grid_info=driver_grid_data, + state=state, + quantity_factory=quantity_factory, + dycore_only=False, + apply_tendencies=True, + tendency_state=tendency_state, + checkpointer=validation, + ) + physics_state = pace.physics.PhysicsState.init_zeros( + quantity_factory=quantity_factory, + active_packages=["microphysics"], + ) + with validation.trial(): + for i in range(2): + dycore.step_dynamics(state) + dycore_to_physics( + dycore_state=state, + physics_state=physics_state, + tendency_state=tendency_state, + timestep=float(physics_config.dt_atmos), + ) + physics(physics_state, timestep=float(physics_config.dt_atmos)) + end_of_step_update( + dycore_state=state, + phy_state=physics_state, + u_dt=tendency_state.u_dt.storage, + v_dt=tendency_state.v_dt.storage, + pt_dt=tendency_state.pt_dt.storage, + dt=float(physics_config.dt_atmos), + ) def _calibrate_thresholds( @@ -296,23 +362,26 @@ def _calibrate_thresholds( active_packages=["microphysics"], ) with calibration.trial(): - dycore.step_dynamics(trial_state) - if physics_config is not None: - dycore_to_physics( - dycore_state=trial_state, - physics_state=trial_physics_state, - tendency_state=tendency_state, - timestep=float(physics_config.dt_atmos), - ) - physics(trial_physics_state, timestep=float(physics_config.dt_atmos)) - end_of_step_update( - dycore_state=trial_state, - phy_state=trial_physics_state, - u_dt=tendency_state.u_dt.storage, - v_dt=tendency_state.v_dt.storage, - pt_dt=tendency_state.pt_dt.storage, - dt=float(physics_config.dt_atmos), - ) + for i in range(2): + dycore.step_dynamics(trial_state) + if physics_config is not None: + dycore_to_physics( + dycore_state=trial_state, + physics_state=trial_physics_state, + tendency_state=tendency_state, + timestep=float(physics_config.dt_atmos), + ) + physics( + trial_physics_state, timestep=float(physics_config.dt_atmos) + ) + end_of_step_update( + dycore_state=trial_state, + phy_state=trial_physics_state, + u_dt=tendency_state.u_dt.storage, + v_dt=tendency_state.v_dt.storage, + pt_dt=tendency_state.pt_dt.storage, + dt=float(physics_config.dt_atmos), + ) all_thresholds = communicator.comm.allgather(calibration.thresholds) thresholds = merge_thresholds(all_thresholds) set_manual_thresholds(thresholds) @@ -321,7 +390,7 @@ def _calibrate_thresholds( def set_manual_thresholds(thresholds: SavepointThresholds): # all thresholds on the input data are 0 because no computation has happened yet - for entry in thresholds.savepoints["FVDynamics-In"]: + for entry in thresholds.savepoints["FVDynamics-In"][0:1]: for name in entry: entry[name] = pace.util.Threshold(relative=0.0, absolute=0.0) From ddea61182c02f4adc95584c2508fc26262519a23 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Tue, 4 Oct 2022 11:28:09 -0700 Subject: [PATCH 42/48] physics checkpoint working --- physics/pace/physics/stencils/physics.py | 49 ++++++++++++++++++- .../stencils/testing/translate_physics.py | 13 +++++ stencils/pace/stencils/update_atmos_state.py | 26 +++++++++- 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 412b772ad..3e178e018 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -18,7 +18,9 @@ from pace.util.grid import GridData from .._config import PhysicsConfig - +from pace.stencils.testing.translate_physics import ( + reshape_pace_variable_to_fortran_format, +) PHYSICS_PACKAGES = Literal["microphysics"] @@ -110,6 +112,16 @@ def prepare_microphysics( qvapor: FloatField, pt: FloatField, delp: FloatField, + u_dt: FloatField, + v_dt: FloatField, + pt_dt: FloatField, + qv_dt: FloatField, + ql_dt: FloatField, + qr_dt: FloatField, + qi_dt: FloatField, + qs_dt: FloatField, + qg_dt: FloatField, + qa_dt: FloatField, ): with computation(BACKWARD), interval(...): dz = (phii[0, 0, 1] - phii[0, 0, 0]) * constants.RGRAV @@ -120,6 +132,17 @@ def prepare_microphysics( / delp * (constants.RDGAS * constants.RGRAV) ) + with computation(PARALLEL), interval(...): + u_dt = 0.0 + v_dt = 0.0 + pt_dt = 0.0 + qv_dt = 0.0 + ql_dt = 0.0 + qr_dt = 0.0 + qi_dt = 0.0 + qs_dt = 0.0 + qg_dt = 0.0 + qa_dt = 0.0 @gtscript.function @@ -191,7 +214,7 @@ def __init__( config=stencil_factory.config.dace_config, dace_compiletime_args=["physics_state"], ) - + self.grid_indexing = stencil_factory.grid_indexing grid_indexing = stencil_factory.grid_indexing origin = grid_indexing.origin_compute() shape = grid_indexing.domain_full(add=(1, 1, 1)) @@ -277,6 +300,11 @@ def __call__(self, physics_state: PhysicsState, timestep: float): w=physics_state.w, omga=physics_state.omga, ) + if self._call_checkpointer: + self._checkpointer( + "AtmosPhysDriverStatein-In", + pt=physics_state.pt, + ) self._atmos_phys_driver_statein( self._prsik, physics_state.phii, @@ -295,6 +323,13 @@ def __call__(self, physics_state: PhysicsState, timestep: float): physics_state.pt, self._dm3d, ) + if self._call_checkpointer: + self._checkpointer( + "AtmosPhysDriverStatein-Out", + IPD_tgrs=reshape_pace_variable_to_fortran_format( + physics_state.pt, self.grid_indexing + ), + ) self._get_prs_fv3( physics_state.phii, physics_state.prsi, @@ -320,6 +355,16 @@ def __call__(self, physics_state: PhysicsState, timestep: float): physics_state.qvapor, physics_state.pt, physics_state.delp, + physics_state.microphysics.udt, + physics_state.microphysics.vdt, + physics_state.microphysics.pt_dt, + physics_state.microphysics.qv_dt, + physics_state.microphysics.ql_dt, + physics_state.microphysics.qr_dt, + physics_state.microphysics.qi_dt, + physics_state.microphysics.qs_dt, + physics_state.microphysics.qg_dt, + physics_state.microphysics.qa_dt, ) self._microphysics(physics_state.microphysics, timestep=timestep) # Fortran uses IPD interface, here we use physics_updated_ to denote diff --git a/stencils/pace/stencils/testing/translate_physics.py b/stencils/pace/stencils/testing/translate_physics.py index 4b2c44d3c..39bf74440 100644 --- a/stencils/pace/stencils/testing/translate_physics.py +++ b/stencils/pace/stencils/testing/translate_physics.py @@ -191,3 +191,16 @@ class ParallelPhysicsTranslate2Py(ParallelTranslate2Py): def __init__(self, rank_grids, namelist, stencil_factory): physics_namelist = PhysicsConfig.from_namelist(namelist) super().__init__(rank_grids, physics_namelist, stencil_factory) + + +def reshape_pace_variable_to_fortran_format(data, grid_indexing): + cn2 = grid_indexing.domain[0] * grid_indexing.domain[1] + return np.reshape( + data[ + grid_indexing.isc : grid_indexing.iec + 1, + grid_indexing.jsc : grid_indexing.jec + 1, + 0 : grid_indexing.domain[2], + ].data, + (cn2, grid_indexing.domain[2]), + order="F", + )[:, ::-1] diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index bae4750b4..3b4262de5 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -11,7 +11,9 @@ from pace.dsl.typing import Float, FloatField from pace.stencils.fv_update_phys import ApplyPhysicsToDycore from pace.util.grid import DriverGridData, GridData - +from pace.stencils.testing.translate_physics import ( + reshape_pace_variable_to_fortran_format, +) # TODO: when this file is not importable from physics or fv3core, import # PhysicsState and DycoreState and use them to type hint below @@ -257,7 +259,7 @@ def __init__( "phy_state", ], ) - + self.grid_indexing = stencil_factory.grid_indexing grid_indexing = stencil_factory.grid_indexing self.namelist = namelist origin = grid_indexing.origin_compute() @@ -315,6 +317,19 @@ def __call__( self._fill_GFS_delp( dycore_state.delp, phy_state.physics_updated_specific_humidity, 1.0e-9 ) + if self._call_checkpointer: + self._checkpointer( + "PhysUpdateTracers-In", + u_dt=u_dt, + v_dt=v_dt, + t_dt=pt_dt, + pt_t1=reshape_pace_variable_to_fortran_format( + phy_state.physics_updated_pt, self.grid_indexing + ), + pt_t0=reshape_pace_variable_to_fortran_format( + phy_state.pt, self.grid_indexing + ), + ) self._prepare_tendencies_and_update_tracers( u_dt, v_dt, @@ -341,6 +356,13 @@ def __call__( dycore_state.delp, self._rdt, ) + if self._call_checkpointer: + self._checkpointer( + "PhysUpdateTracers-Out", + u_dt=u_dt, + v_dt=v_dt, + t_dt=pt_dt, + ) if self._apply_tendencies: self._apply_physics_to_dycore( dycore_state, From 19b9944864f4539c9e35cacd372d0484c0993d7c Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Tue, 4 Oct 2022 11:28:30 -0700 Subject: [PATCH 43/48] 2 step driver threshold --- tests/savepoint/thresholds/driver.yaml | 1021 ++++++++++++++++++------ 1 file changed, 764 insertions(+), 257 deletions(-) diff --git a/tests/savepoint/thresholds/driver.yaml b/tests/savepoint/thresholds/driver.yaml index 28436d40b..a51181c46 100644 --- a/tests/savepoint/thresholds/driver.yaml +++ b/tests/savepoint/thresholds/driver.yaml @@ -1,88 +1,171 @@ savepoints: + AtmosPhysDriverStatein-In: + - pt: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + - pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + AtmosPhysDriverStatein-Out: + - IPD_tgrs: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + - IPD_tgrs: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 C_SW-In: - delpd: absolute: 2.7284841053187847e-12 - relative: 2.664392791389508e-15 + relative: 2.6632276486803376e-15 divgdd: absolute: 0.0 relative: 0.0 ptd: absolute: 4.263256414560601e-13 - relative: 2.3048355003642202e-14 + relative: 2.4024873582185304e-14 + uad: + absolute: 0.0 + relative: 0.0 + ucd: + absolute: 0.0 + relative: 0.0 + ud: + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 + utd: + absolute: 0.0 + relative: 0.0 + vad: + absolute: 0.0 + relative: 0.0 + vcd: + absolute: 0.0 + relative: 0.0 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + vtd: + absolute: 0.0 + relative: 0.0 + wd: + absolute: 0.0 + relative: 0.0 + - delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + divgdd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 uad: - absolute: 192.0 - relative: 2.6645228139724058e-15 + absolute: 384.0 + relative: 1.924686893998195e-09 ucd: - absolute: 4.263256414560601e-14 - relative: 2.664479239525582e-15 + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 ud: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 utd: absolute: 0.0 relative: 0.0 vad: - absolute: 192.0 - relative: 2.6645146823245013e-15 + absolute: 384.0 + relative: 1.10022939861047e-07 vcd: - absolute: 4.263256414560601e-14 - relative: 2.664530705451025e-15 + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 vd: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 vtd: absolute: 0.0 relative: 0.0 wd: - absolute: 8.326672684688674e-17 - relative: 2.664506596135002e-15 + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 C_SW-Out: - delpd: absolute: 2.7284841053187847e-12 - relative: 2.664392791389508e-15 + relative: 2.6632276486803376e-15 divgdd: - absolute: 4.777265822514254e-19 - relative: 3.925412056700377e-09 + absolute: 4.942437247228842e-19 + relative: 5.152618796428155e-08 ptd: absolute: 4.263256414560601e-13 - relative: 2.3048355003642202e-14 + relative: 2.4024873582185304e-14 uad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 2.8634008989163294e-11 ucd: - absolute: 1.7053025658242404e-13 - relative: 5.929879004969384e-11 + absolute: 3.410605131648481e-13 + relative: 2.813991384236331e-10 ud: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 utd: - absolute: 2.002716064453125e-05 - relative: 3.7052523913230544e-11 + absolute: 2.86102294921875e-05 + relative: 2.26433843291111e-10 vad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 8.422447914909142e-11 + vcd: + absolute: 3.410605131648481e-13 + relative: 1.51267761412336e-10 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + vtd: + absolute: 2.86102294921875e-05 + relative: 1.5363504124939987e-10 + wd: + absolute: 0.0 + relative: 0.0 + - delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + divgdd: + absolute: 5.409006817378153e-18 + relative: 3.7456372136851146e-07 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 9.698908343125368e-13 + relative: 4.31030506277985e-08 + ud: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + utd: + absolute: 7.152557373046875e-05 + relative: 3.916045067374942e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 vcd: - absolute: 1.7053025658242404e-13 - relative: 4.054757865553553e-10 + absolute: 9.592326932761353e-13 + relative: 1.535682918794925e-07 vd: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 vtd: - absolute: 2.002716064453125e-05 - relative: 1.8398302626266127e-09 + absolute: 7.152557373046875e-05 + relative: 4.161008969260313e-08 wd: - absolute: 8.326672684688674e-17 - relative: 2.664506596135002e-15 + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 D_SW-In: - delpcd: - absolute: 2.002716064453125e-05 - relative: 1.8398302626266127e-09 + absolute: 2.86102294921875e-05 + relative: 1.5363504124939987e-10 delpd: absolute: 2.7284841053187847e-12 - relative: 2.664392791389508e-15 - divgdd: - absolute: 4.777265822514254e-19 - relative: 3.925412056700377e-09 + relative: 2.6632276486803376e-15 mfxd: absolute: 0.0 relative: 0.0 @@ -91,123 +174,167 @@ savepoints: relative: 0.0 ptd: absolute: 4.263256414560601e-13 - relative: 2.3048355003642202e-14 + relative: 2.4024873582185304e-14 uad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 2.8634008989163294e-11 ucd: - absolute: 7.354117315117037e-13 - relative: 7.45716960892069e-09 + absolute: 6.288303211476887e-13 + relative: 2.008118044889811e-08 ud: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 8.526512829121202e-14 + relative: 2.664505672560514e-15 vad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 8.422447914909142e-11 + vcd: + absolute: 6.288303211476887e-13 + relative: 1.535682918794925e-07 + vd: + absolute: 8.526512829121202e-14 + relative: 2.6645309432189408e-15 + wd: + absolute: 0.0 + relative: 0.0 + xfxd: + absolute: 0.0 + relative: 0.0 + yfxd: + absolute: 0.0 + relative: 0.0 + - delpcd: + absolute: 7.152557373046875e-05 + relative: 4.161008969260313e-08 + delpd: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + mfxd: + absolute: 0.0 + relative: 0.0 + mfyd: + absolute: 0.0 + relative: 0.0 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 1.262989712813578e-12 + relative: 2.8125989221256106e-08 + ud: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 vcd: - absolute: 7.354117315117037e-13 - relative: 2.1527364019830838e-07 + absolute: 1.262989712813578e-12 + relative: 1.0487218599584194e-07 vd: - absolute: 4.263256414560601e-14 - relative: 2.6645224906037465e-15 + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 wd: - absolute: 8.326672684688674e-17 - relative: 2.664506596135002e-15 + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 xfxd: absolute: 0.0 relative: 0.0 yfxd: absolute: 0.0 relative: 0.0 - zhd: - absolute: 1.7462298274040222e-10 - relative: 8.274629575761722e-15 D_SW-Out: - delpcd: - absolute: 1.71661376953125e-05 - relative: 3.925412056700377e-09 + absolute: 2.86102294921875e-05 + relative: 5.152618796428155e-08 delpd: absolute: 5.4569682106375694e-12 - relative: 5.29997413501075e-15 + relative: 5.320929041303097e-15 divgdd: - absolute: 3.595654861094505e-19 - relative: 4.5304154135537483e-08 + absolute: 4.942437247228842e-19 + relative: 5.152618796428155e-08 mfxd: - absolute: 1628995584.0 + absolute: 1604124672.0 relative: 120.0 mfyd: - absolute: 1628504064.0 + absolute: 1621082112.0 relative: 120.0 ptd: absolute: 4.263256414560601e-13 - relative: 2.4821302035387395e-14 + relative: 2.4024873582185304e-14 uad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 2.8634008989163294e-11 ucd: - absolute: 7.354117315117037e-13 - relative: 2.389973474778064e-07 + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 ud: absolute: 1.341104507446289e-07 - relative: 3.788836253752432e-11 + relative: 1.6737607801318062e-10 vad: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 8.422447914909142e-11 vcd: - absolute: 7.354117315117037e-13 - relative: 2.389973509569915e-07 + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 vd: - absolute: 1.341104507446289e-07 - relative: 3.718147277266473e-11 + absolute: 1.7881393432617188e-07 + relative: 7.469099658749364e-11 wd: - absolute: 1.6653345369377348e-16 - relative: 1.4341582375196082e-12 + absolute: 0.0 + relative: 0.0 xfxd: - absolute: 0.0001087188720703125 - relative: 7.457167648421754e-09 + absolute: 0.00010514259338378906 + relative: 2.0081181638579702e-08 yfxd: - absolute: 0.0001087188720703125 - relative: 2.1527364273173495e-07 - Driver-Out: - - delp: - absolute: 1.800799509510398e-10 - relative: 5.022117662087534e-13 - delz: - absolute: 25769803776.0 - relative: 4.420922764818589e-12 - pt: - absolute: 5.368292477214709e-10 - relative: 1.7778722690158857e-12 - qgraupel: - absolute: 4.0925587986673597e-19 + absolute: 0.00010532140731811523 + relative: 1.5356829033557975e-07 + - delpcd: + absolute: 6.651878356933594e-05 + relative: 3.7456372136851146e-07 + delpd: + absolute: 2.6193447411060333e-10 + relative: 5.022709085608076e-13 + divgdd: + absolute: 5.409006817378153e-18 + relative: 2.3153394949370874e-06 + mfxd: + absolute: 64332103680.0 relative: 120.0 - qice: - absolute: 4.124890787417885e-15 - relative: 123.0 - qliquid: - absolute: 4.268655741379579e-14 - relative: 6.6325478948235934e-09 - qrain: - absolute: 2.5347384713619685e-15 + mfyd: + absolute: 62617780224.0 relative: 120.0 - qsnow: - absolute: 0.0 - relative: 0.0 - qvapor: - absolute: 4.2455622351056377e-14 - relative: 8.685658162172946e-12 - u: - absolute: 1.538769112130467e-12 - relative: 2.718327538654071e-09 - ua: - absolute: 384.0 - relative: 1.792767371402652e-09 - v: - absolute: 1.538769112130467e-12 - relative: 5.041619359370808e-09 - va: - absolute: 384.0 - relative: 6.743131502955869e-08 + ptd: + absolute: 6.544098596350523e-12 + relative: 5.97574138843221e-13 + uad: + absolute: 3093.0 + relative: 9.980260518341024e-09 + ucd: + absolute: 1.262989712813578e-12 + relative: 1.2492728689511802e-05 + ud: + absolute: 7.264316082000732e-07 + relative: 8.742370556648009e-08 + vad: + absolute: 3093.0 + relative: 9.98355958190459e-09 + vcd: + absolute: 1.262989712813578e-12 + relative: 3.6260027791287995e-06 + vd: + absolute: 6.370246410369873e-07 + relative: 1.0448120616784678e-08 + wd: + absolute: 1.859949694260621e-12 + relative: 3.944770141167069e-05 + xfxd: + absolute: 0.00018596649169921875 + relative: 2.8125989468113277e-08 + yfxd: + absolute: 0.00018525123596191406 + relative: 1.0487218572228159e-07 FVDynamics-In: - delz: absolute: 0.0 @@ -236,157 +363,349 @@ savepoints: w: absolute: 0.0 relative: 0.0 + - delz: + absolute: 25769803776.0 + relative: 4.425706291051527e-12 + qvapor: + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 + u: + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 384.0 + relative: 1.924686893998195e-09 + uc: + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 + v: + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 + va: + absolute: 384.0 + relative: 1.10022939861047e-07 + vc: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 FVDynamics-Out: - delz: absolute: 25769803776.0 - relative: 4.420922764818589e-12 + relative: 4.425706291051527e-12 qvapor: - absolute: 4.2289088897362603e-14 - relative: 8.651547729736657e-12 + absolute: 3.289035710452026e-14 + relative: 7.505716682286047e-11 u: - absolute: 1.5384360452230794e-12 - relative: 2.718327538654071e-09 + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 ua: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 1.924686893998194e-09 uc: - absolute: 7.354117315117037e-13 - relative: 2.389973474778064e-07 + absolute: 6.288303211476887e-13 + relative: 2.9122418623312017e-07 v: - absolute: 1.5384360452230794e-12 - relative: 5.041619359370809e-09 + absolute: 1.0444978215673473e-12 + relative: 7.151036709605271e-08 va: absolute: 384.0 - relative: 6.743131502955869e-08 + relative: 1.1002293986118271e-07 + vc: + absolute: 6.288303211476887e-13 + relative: 3.6043670523852133e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + qvapor: + absolute: 3.778227730677486e-14 + relative: 2.8773515355509475e-11 + u: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + uc: + absolute: 1.262989712813578e-12 + relative: 1.2492728689511802e-05 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + va: + absolute: 3093.0 + relative: 1.001009536835503e-07 vc: - absolute: 7.354117315117037e-13 - relative: 2.389973509569915e-07 + absolute: 1.262989712813578e-12 + relative: 3.6260027791287995e-06 w: - absolute: 2.0960316815532565e-12 - relative: 4.640618151764116e-06 + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 FVUpdatePhys-In: - delp: - absolute: 1.800799509510398e-10 - relative: 5.022117662087534e-13 + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 pe: absolute: 5.238689482212067e-10 - relative: 6.9515212013449995e-15 + relative: 7.0048868034443836e-15 peln: absolute: 2.1316282072803006e-14 - relative: 2.2349241361126896e-15 + relative: 2.2542546489769687e-15 pk: absolute: 2.984279490192421e-13 - relative: 1.2799614024119294e-14 + relative: 1.2545561792452565e-14 ps: absolute: .nan - relative: 5.365436857202281e-15 + relative: 5.243476108433941e-15 pt: - absolute: 5.368292477214709e-10 - relative: 1.7778722690158857e-12 + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 qgraupel: - absolute: 4.0925587986673597e-19 - relative: 120.0 + absolute: 0.0 + relative: 0.0 qice: - absolute: 4.124890787417885e-15 - relative: 123.0 + absolute: 4.4986438636356084e-15 + relative: 138.0 qliquid: - absolute: 4.268655741379579e-14 - relative: 6.6325478948235934e-09 + absolute: 3.3320351686128014e-14 + relative: 6.967637350538825e-09 qrain: - absolute: 2.5347384713619685e-15 + absolute: 2.8374926106661258e-15 relative: 120.0 qsnow: absolute: 0.0 relative: 0.0 qvapor: - absolute: 4.2455622351056377e-14 - relative: 8.685658162172946e-12 + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 t_dt: - absolute: 4.244309981596417e-14 - relative: 5.047263788555737e-08 + absolute: 3.637979403402908e-14 + relative: 1.4767081826878412e-08 u: - absolute: 1.5384360452230794e-12 - relative: 2.718327538654071e-09 + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 u_dt: absolute: 3.7895612573872007e-16 relative: 120.0 ua: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 1.924686893998194e-09 v: - absolute: 1.5384360452230794e-12 - relative: 5.041619359370809e-09 + absolute: 1.0444978215673473e-12 + relative: 7.151036709605271e-08 v_dt: - absolute: 4.736951571734001e-17 + absolute: 2.368475785867001e-17 relative: 120.0 va: absolute: 384.0 - relative: 6.743131502955869e-08 + relative: 1.1002293986118271e-07 + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + pe: + absolute: 6.984919309616089e-10 + relative: 8.74747886746669e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2749142151097106e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2798456145608727e-14 + ps: + absolute: .nan + relative: 6.990040335621623e-15 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.484916212417695e-15 + relative: .inf + qliquid: + absolute: 3.800939055685626e-14 + relative: 1.6059488926374826e-07 + qrain: + absolute: 2.8293610943724845e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.778227730677486e-14 + relative: 6.869188271838486e-12 + t_dt: + absolute: 1.5158244126046994e-14 + relative: 3.2931956762020915e-08 + u: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + v_dt: + absolute: 2.3684758432181898e-17 + relative: 120.0 + va: + absolute: 3093.0 + relative: 1.001009536835503e-07 FVUpdatePhys-Out: - pt: - absolute: 5.368292477214709e-10 - relative: 1.7778722690158857e-12 + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 qgraupel: - absolute: 4.0925587986673597e-19 - relative: 120.0 + absolute: 0.0 + relative: 0.0 qice: - absolute: 4.124890787417885e-15 - relative: 123.0 + absolute: 4.4986438636356084e-15 + relative: 138.0 qliquid: - absolute: 4.268655741379579e-14 - relative: 6.6325478948235934e-09 + absolute: 3.3320351686128014e-14 + relative: 6.967637350538825e-09 qrain: - absolute: 2.5347384713619685e-15 + absolute: 2.8374926106661258e-15 relative: 120.0 qsnow: absolute: 0.0 relative: 0.0 qvapor: - absolute: 4.2455622351056377e-14 - relative: 8.685658162172946e-12 + absolute: 3.3056890558214036e-14 + relative: 5.41177166737603e-12 u: - absolute: 1.538769112130467e-12 - relative: 2.718327538654071e-09 + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 ua: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 1.924686893998195e-09 v: - absolute: 1.538769112130467e-12 - relative: 5.041619359370808e-09 + absolute: 1.0444978215673473e-12 + relative: 7.154781261256709e-08 va: absolute: 384.0 - relative: 6.743131502955869e-08 + relative: 1.10022939861047e-07 + - pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qgraupel: + absolute: 0.0 + relative: 0.0 + qice: + absolute: 4.484916212417695e-15 + relative: .inf + qliquid: + absolute: 3.800939055685626e-14 + relative: 1.6059488926374826e-07 + qrain: + absolute: 2.8293610943724845e-15 + relative: 120.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.778227730677486e-14 + relative: 6.869188271838486e-12 + u: + absolute: 1.4788170688007085e-12 + relative: 3.846326345494972e-08 + ua: + absolute: 3093.0 + relative: 3.5108474368372467e-09 + v: + absolute: 1.4788170688007085e-12 + relative: 3.846326345494972e-08 + va: + absolute: 3093.0 + relative: 1.0012980122080339e-07 GFSPhysicsDriver-In: - delp: - absolute: 1.7462298274040222e-10 - relative: 5.022117662087535e-13 + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 delz: absolute: 25769803776.0 - relative: 4.420922764818589e-12 + relative: 4.425706291051527e-12 omga: - absolute: 9.002975448479944e-12 - relative: 6.166103011315274e-05 + absolute: 9.321803745576673e-12 + relative: 0.0001252012141681156 pt: - absolute: 5.368292477214709e-10 - relative: 1.7778722690158857e-12 + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 qcld: - absolute: 8.528644457328483e-11 - relative: 4.544796922758549e-08 + absolute: 8.192313494248538e-11 + relative: 7.636900744509854e-07 qgraupel: - absolute: 5.11554960582394e-19 + absolute: 0.0 + relative: 0.0 + qice: + absolute: 1.1277840081491642e-14 relative: 120.0 + qliquid: + absolute: 3.284547113457936e-14 + relative: 6.485006027709165e-09 + qo3mr: + absolute: 0.0 + relative: 0.0 + qrain: + absolute: 7.623296525288703e-21 + relative: 1.9715033202384307e-05 + qsgs_tke: + absolute: 0.0 + relative: 0.0 + qsnow: + absolute: 0.0 + relative: 0.0 + qvapor: + absolute: 3.289035710452026e-14 + relative: 7.505716682286047e-11 + ua: + absolute: 384.0 + relative: 1.924686893998194e-09 + va: + absolute: 384.0 + relative: 1.1002293986118271e-07 + w: + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + omga: + absolute: 1.762632487883886e-11 + relative: 0.00013614641074803746 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + qcld: + absolute: 8.595690825785596e-11 + relative: 3.228782851361542e-07 + qgraupel: + absolute: 0.0 + relative: 0.0 qice: - absolute: 9.636630736957605e-15 - relative: 120.02930402930403 + absolute: 4.484916212417695e-15 + relative: .inf qliquid: - absolute: 4.230372562669116e-14 - relative: 6.613159352359565e-09 + absolute: 3.7826553412993735e-14 + relative: 1.7425773968564856e-07 qo3mr: absolute: 0.0 relative: 0.0 qrain: - absolute: 5.234663614031576e-19 - relative: 5.676126564496927e-05 + absolute: 2.8293610943724845e-15 + relative: 120.0 qsgs_tke: absolute: 0.0 relative: 0.0 @@ -394,17 +713,67 @@ savepoints: absolute: 0.0 relative: 0.0 qvapor: - absolute: 4.2289088897362603e-14 - relative: 8.651547729736657e-12 + absolute: 3.778227730677486e-14 + relative: 2.8773515355509475e-11 ua: - absolute: 5.238689482212067e-10 - relative: 3.8614993901561815e-10 + absolute: 3093.0 + relative: 3.5108474368372467e-09 va: - absolute: 1.3672396548258803e-12 - relative: 6.743131502955869e-08 + absolute: 3093.0 + relative: 1.001009536835503e-07 w: - absolute: 2.0960316815532565e-12 - relative: 4.640618151764116e-06 + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 + PhysUpdateTracers-In: + - pt_t0: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + pt_t1: + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 + t_dt: + absolute: 0.0 + relative: 0.0 + u_dt: + absolute: 0.0 + relative: 0.0 + v_dt: + absolute: 0.0 + relative: 0.0 + - pt_t0: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + pt_t1: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + t_dt: + absolute: 0.0 + relative: 0.0 + u_dt: + absolute: 0.0 + relative: 0.0 + v_dt: + absolute: 0.0 + relative: 0.0 + PhysUpdateTracers-Out: + - t_dt: + absolute: 3.637979403402908e-14 + relative: 1.4767081826878412e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + v_dt: + absolute: 2.368475785867001e-17 + relative: 120.0 + - t_dt: + absolute: 1.5158244126046994e-14 + relative: 3.2931956762020915e-08 + u_dt: + absolute: 3.7895612573872007e-16 + relative: 120.0 + v_dt: + absolute: 2.3684758432181898e-17 + relative: 120.0 Remapping-In: - ak: absolute: 0.0 @@ -414,25 +783,25 @@ savepoints: relative: 0.0 cappa: absolute: 1.3322676295501878e-15 - relative: 4.684802402122118e-15 + relative: 4.684744757427358e-15 delp: absolute: 5.4569682106375694e-12 - relative: 5.29997413501075e-15 + relative: 5.320929041303097e-15 delz: absolute: 25769803776.0 - relative: 4.417755979333718e-12 + relative: 4.424309879776438e-12 dp1: absolute: 8.185452315956354e-12 - relative: 7.816113018901345e-15 + relative: 7.981060438552416e-15 omga: - absolute: 2.914335439641036e-16 - relative: 2.69741183575927e-12 + absolute: 0.0 + relative: 0.0 pe: absolute: 5.238689482212067e-10 - relative: 7.012922165955118e-15 + relative: 8.133300801432759e-15 peln: absolute: 2.1316282072803006e-14 - relative: 2.642154438487291e-15 + relative: 2.6420226645538662e-15 pfull: absolute: 0.0 relative: 0.0 @@ -441,109 +810,247 @@ savepoints: relative: 0.0 pk: absolute: 2.984279490192421e-13 - relative: 1.2610043551750094e-14 + relative: 1.2798830510581378e-14 pkz: - absolute: 4.7066350816749036e-11 - relative: 1.7659895624560728e-12 + absolute: 4.7279513637477066e-11 + relative: 1.7652850589111428e-12 ps: absolute: .nan - relative: 1.7916862372061497e-15 + relative: 1.7503446900789073e-15 pt: absolute: 3.410605131648481e-13 - relative: 2.4821302030748827e-14 + relative: 2.4024873582185304e-14 te_2d: absolute: 0.0 relative: 0.0 u: - absolute: 1.544597783009749e-12 - relative: 2.3499241013589115e-09 + absolute: 1.0444978215673473e-12 + relative: 4.920683557677236e-08 ua: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 2.8634008989163294e-11 v: - absolute: 1.0764722446765518e-12 - relative: 5.596522336615699e-09 + absolute: 1.0125233984581428e-12 + relative: 5.378610226381813e-08 va: absolute: 384.0 - relative: 1.792767371402652e-09 + relative: 8.422447914909142e-11 w: - absolute: 2.095906781462986e-12 - relative: 1.320885985901664e-05 + absolute: 1.859949694260621e-12 + relative: 5.4023923350536e-05 + wsd: + absolute: 0.0 + relative: 0.0 + - ak: + absolute: 0.0 + relative: 0.0 + bk: + absolute: 0.0 + relative: 0.0 + cappa: + absolute: 3.064215547965432e-14 + relative: 1.0751531623036565e-13 + delp: + absolute: 2.6193447411060333e-10 + relative: 5.022709085608076e-13 + delz: + absolute: 25769803776.0 + relative: 4.4284261446010145e-12 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 + omga: + absolute: 9.321803745576673e-12 + relative: 6.434410522391023e-05 + pe: + absolute: 6.984919309616089e-10 + relative: 1.0616077611929113e-14 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.6420212048418394e-15 + pfull: + absolute: 0.0 + relative: 0.0 + phis: + absolute: 0.0 + relative: 0.0 + pk: + absolute: 2.984279490192421e-13 + relative: 1.3065571699261469e-14 + pkz: + absolute: 4.7407411329913884e-11 + relative: 1.7669600579817248e-12 + ps: + absolute: .nan + relative: 5.243476108433941e-15 + pt: + absolute: 6.480149750132114e-12 + relative: 5.904546306310534e-13 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.3962164757685969e-12 + relative: 4.7436825986745986e-08 + ua: + absolute: 3093.0 + relative: 9.980260518341024e-09 + v: + absolute: 1.4788170688007085e-12 + relative: 3.755502628095818e-08 + va: + absolute: 3093.0 + relative: 9.98355958190459e-09 + w: + absolute: 2.369833496107532e-12 + relative: 0.00013648630486358366 wsd: absolute: 0.0 relative: 0.0 Remapping-Out: - cappa: - absolute: 3.863576125695545e-14 - relative: 1.354653976534733e-13 + absolute: 2.9976021664879227e-14 + relative: 1.0517782687951213e-13 delp: - absolute: 1.7462298274040222e-10 - relative: 5.022117662087535e-13 + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 delz: absolute: 25769803776.0 - relative: 4.420922764818589e-12 + relative: 4.425706291051527e-12 dp1: absolute: 8.185452315956354e-12 - relative: 7.816113018901345e-15 + relative: 7.981060438552416e-15 omga: - absolute: 2.914335439641036e-16 - relative: 2.69741183575927e-12 + absolute: 0.0 + relative: 0.0 pe: absolute: 5.238689482212067e-10 - relative: 6.9515212013449995e-15 + relative: 7.0048868034443836e-15 peln: absolute: 2.1316282072803006e-14 - relative: 2.2349241361126896e-15 + relative: 2.2542546489769687e-15 pk: absolute: 2.984279490192421e-13 - relative: 1.2799614024119294e-14 + relative: 1.2545561792452565e-14 pkz: - absolute: 4.7194248509185854e-11 - relative: 1.7700916431128049e-12 + absolute: 4.732214620162267e-11 + relative: 1.76983385674747e-12 pt: - absolute: 5.368292477214709e-10 - relative: 1.7778722690158857e-12 + absolute: 5.409219738794491e-10 + relative: 1.7822151583002022e-12 te_2d: absolute: 0.0 relative: 0.0 u: - absolute: 1.5384360452230794e-12 - relative: 2.4055998841126902e-09 + absolute: 1.0444978215673473e-12 + relative: 4.7436825986745986e-08 v: - absolute: 1.0764722446765518e-12 - relative: 5.041619359370809e-09 + absolute: 1.0231815394945443e-12 + relative: 7.151036709605271e-08 w: - absolute: 2.0960316815532565e-12 - relative: 4.640618151764116e-06 + absolute: 1.859949694260621e-12 + relative: 5.4022591853302335e-05 + - cappa: + absolute: 3.4638958368304884e-14 + relative: 1.213988092423256e-13 + delp: + absolute: 2.6193447411060333e-10 + relative: 5.043732499872104e-13 + delz: + absolute: 25769803776.0 + relative: 4.426936988419288e-12 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 + omga: + absolute: 9.321803745576673e-12 + relative: 6.434410522391023e-05 + pe: + absolute: 6.984919309616089e-10 + relative: 8.74747886746669e-15 + peln: + absolute: 2.1316282072803006e-14 + relative: 2.2749142151097106e-15 + pk: + absolute: 2.984279490192421e-13 + relative: 1.2798456145608727e-14 + pkz: + absolute: 4.736477876576828e-11 + relative: 1.7694778604229885e-12 + pt: + absolute: 5.361471266951412e-10 + relative: 1.7746932988156208e-12 + te_2d: + absolute: 0.0 + relative: 0.0 + u: + absolute: 1.4175327578413999e-12 + relative: 4.7436825986745986e-08 + v: + absolute: 1.4794832026154836e-12 + relative: 3.8463387345110815e-08 + w: + absolute: 2.369833496107532e-12 + relative: 0.0001361464107479025 Tracer2D1L-In: - cxd: - absolute: 2.3418766925686896e-16 - relative: 7.457167570200806e-09 + absolute: 2.2638141361497333e-16 + relative: 2.0081178953558838e-08 cyd: - absolute: 2.3418766925686896e-16 - relative: 2.1527364253231465e-07 + absolute: 2.2768245622195593e-16 + relative: 1.53568292783154e-07 dp1: absolute: 2.7284841053187847e-12 - relative: 2.664392791389508e-15 + relative: 2.6632276486803376e-15 mfxd: - absolute: 1628995584.0 + absolute: 1604124672.0 relative: 120.0 mfyd: - absolute: 1628504064.0 + absolute: 1621082112.0 + relative: 120.0 + - cxd: + absolute: 4.683753385137379e-16 + relative: 2.8125989101612067e-08 + cyd: + absolute: 4.501607420159814e-16 + relative: 1.0487218492835087e-07 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042543481610996e-13 + mfxd: + absolute: 64332103680.0 + relative: 120.0 + mfyd: + absolute: 62617780224.0 relative: 120.0 Tracer2D1L-Out: - cxd: - absolute: 7.806255641895632e-17 - relative: 7.45716661757556e-09 + absolute: 7.546047120499111e-17 + relative: 2.0081177721573792e-08 cyd: - absolute: 7.806255641895632e-17 - relative: 2.1527364135800607e-07 + absolute: 7.611099250848241e-17 + relative: 1.535682939641504e-07 dp1: absolute: 8.185452315956354e-12 - relative: 7.816113018901345e-15 + relative: 7.981060438552416e-15 + mfxd: + absolute: 534724608.0 + relative: 120.0 + mfyd: + absolute: 540352512.0 + relative: 120.0 + - cxd: + absolute: 1.5612511283791264e-16 + relative: 2.812598910161206e-08 + cyd: + absolute: 1.496198998029996e-16 + relative: 1.0487218492835087e-07 + dp1: + absolute: 2.6193447411060333e-10 + relative: 5.042451815692504e-13 mfxd: - absolute: 543006720.0 + absolute: 21444034560.0 relative: 120.0 mfyd: - absolute: 542834688.0 + absolute: 20872593408.0 relative: 120.0 From 480a16ab41791746f19693087bc0a652d71926d3 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Tue, 4 Oct 2022 16:32:41 -0700 Subject: [PATCH 44/48] delete unused checkpoint --- fv3core/pace/fv3core/stencils/dyn_core.py | 2 +- physics/pace/physics/stencils/physics.py | 7 ++++--- stencils/pace/stencils/update_atmos_state.py | 19 ++----------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/fv3core/pace/fv3core/stencils/dyn_core.py b/fv3core/pace/fv3core/stencils/dyn_core.py index f1e76e4a7..3d3230805 100644 --- a/fv3core/pace/fv3core/stencils/dyn_core.py +++ b/fv3core/pace/fv3core/stencils/dyn_core.py @@ -1,5 +1,6 @@ from typing import Dict, Mapping, Optional, Sequence, Tuple +from dace.frontend.python.interface import nounroll as dace_nounroll from gt4py.gtscript import ( __INLINED, BACKWARD, @@ -23,7 +24,6 @@ import pace.util import pace.util as fv3util import pace.util.constants as constants -from dace.frontend.python.interface import nounroll as dace_nounroll from pace.dsl.dace.orchestration import dace_inhibitor, orchestrate from pace.dsl.dace.wrapped_halo_exchange import WrappedHaloUpdater from pace.dsl.stencil import GridIndexing, StencilFactory diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 3e178e018..52d8e02e2 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -15,12 +15,13 @@ from pace.physics.stencils.get_phi_fv3 import get_phi_fv3 from pace.physics.stencils.get_prs_fv3 import get_prs_fv3 from pace.physics.stencils.microphysics import Microphysics -from pace.util.grid import GridData - -from .._config import PhysicsConfig from pace.stencils.testing.translate_physics import ( reshape_pace_variable_to_fortran_format, ) +from pace.util.grid import GridData + +from .._config import PhysicsConfig + PHYSICS_PACKAGES = Literal["microphysics"] diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index 3b4262de5..098519df4 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -10,10 +10,11 @@ from pace.dsl.stencil import StencilFactory from pace.dsl.typing import Float, FloatField from pace.stencils.fv_update_phys import ApplyPhysicsToDycore -from pace.util.grid import DriverGridData, GridData from pace.stencils.testing.translate_physics import ( reshape_pace_variable_to_fortran_format, ) +from pace.util.grid import DriverGridData, GridData + # TODO: when this file is not importable from physics or fv3core, import # PhysicsState and DycoreState and use them to type hint below @@ -386,19 +387,3 @@ def __call__( u=dycore_state.u, v=dycore_state.v, ) - # self._checkpointer( - # "Driver-Out", - # qvapor=dycore_state.qvapor, - # qliquid=dycore_state.qliquid, - # qrain=dycore_state.qrain, - # qsnow=dycore_state.qsnow, - # qice=dycore_state.qice, - # qgraupel=dycore_state.qgraupel, - # pt=dycore_state.pt, - # ua=dycore_state.ua, - # va=dycore_state.va, - # u=dycore_state.u, - # v=dycore_state.v, - # delz=dycore_state.delz, - # delp=dycore_state.delp, - # ) From 90c26d907f47bbc39d1b0dc13859225bdf5bc926 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Wed, 5 Oct 2022 15:43:32 -0700 Subject: [PATCH 45/48] lint --- driver/pace/driver/driver.py | 2 +- dsl/pace/dsl/dace/build.py | 1 - dsl/pace/dsl/dace/dace_config.py | 1 - dsl/pace/dsl/dace/orchestration.py | 4 ++-- dsl/pace/dsl/dace/utils.py | 2 +- dsl/pace/dsl/stencil.py | 2 +- fv3core/pace/fv3core/stencils/fv_dynamics.py | 2 +- physics/pace/physics/stencils/physics.py | 7 ++++--- stencils/pace/stencils/update_atmos_state.py | 3 ++- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/driver/pace/driver/driver.py b/driver/pace/driver/driver.py index db27890e4..aa7a50843 100644 --- a/driver/pace/driver/driver.py +++ b/driver/pace/driver/driver.py @@ -6,9 +6,9 @@ from math import floor from typing import Any, Dict, List, Tuple, Union -import dace import dacite +import dace import pace.driver import pace.dsl import pace.physics diff --git a/dsl/pace/dsl/dace/build.py b/dsl/pace/dsl/dace/build.py index 3b3f8b0fa..6e6d1635a 100644 --- a/dsl/pace/dsl/dace/build.py +++ b/dsl/pace/dsl/dace/build.py @@ -1,7 +1,6 @@ from typing import List, Optional, Tuple from dace.sdfg import SDFG - from pace.dsl.dace.dace_config import DaceConfig, DaCeOrchestration from pace.util import TilePartitioner diff --git a/dsl/pace/dsl/dace/dace_config.py b/dsl/pace/dsl/dace/dace_config.py index 27c7ae903..a1a165cdd 100644 --- a/dsl/pace/dsl/dace/dace_config.py +++ b/dsl/pace/dsl/dace/dace_config.py @@ -2,7 +2,6 @@ from typing import Any, Dict, Optional import dace.config - from pace.dsl.gt4py_utils import is_gpu_backend from pace.util.communicator import CubedSphereCommunicator diff --git a/dsl/pace/dsl/dace/orchestration.py b/dsl/pace/dsl/dace/orchestration.py index 02f20e28e..aaafd6434 100644 --- a/dsl/pace/dsl/dace/orchestration.py +++ b/dsl/pace/dsl/dace/orchestration.py @@ -1,8 +1,9 @@ import os from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union -import dace import gt4py.storage + +import dace from dace import compiletime as DaceCompiletime from dace.dtypes import DeviceType as DaceDeviceType from dace.dtypes import StorageType as DaceStorageType @@ -10,7 +11,6 @@ from dace.frontend.python.parser import DaceProgram from dace.transformation.auto.auto_optimize import make_transients_persistent from dace.transformation.helpers import get_parent_map - from pace.dsl.dace.build import ( determine_compiling_ranks, get_sdfg_path, diff --git a/dsl/pace/dsl/dace/utils.py b/dsl/pace/dsl/dace/utils.py index e86fb6de0..61d17ad1f 100644 --- a/dsl/pace/dsl/dace/utils.py +++ b/dsl/pace/dsl/dace/utils.py @@ -5,7 +5,6 @@ import dace from dace.transformation.helpers import get_parent_map - from pace.dsl.dace.dace_config import DaceConfig @@ -47,6 +46,7 @@ def sdfg_nan_checker(sdfg: dace.SDFG): import copy import sympy as sp + from dace import data as dt from dace import symbolic from dace.sdfg import graph as gr diff --git a/dsl/pace/dsl/stencil.py b/dsl/pace/dsl/stencil.py index bcfea4735..b80e53d48 100644 --- a/dsl/pace/dsl/stencil.py +++ b/dsl/pace/dsl/stencil.py @@ -16,13 +16,13 @@ cast, ) -import dace import gt4py import numpy as np from gt4py import gtscript from gt4py.storage.storage import Storage from gtc.passes.oir_pipeline import DefaultPipeline, OirPipeline +import dace import pace.dsl.gt4py_utils as gt4py_utils import pace.util from pace.dsl.dace.orchestration import SDFGConvertible diff --git a/fv3core/pace/fv3core/stencils/fv_dynamics.py b/fv3core/pace/fv3core/stencils/fv_dynamics.py index 0a0d93f03..fd97d6f31 100644 --- a/fv3core/pace/fv3core/stencils/fv_dynamics.py +++ b/fv3core/pace/fv3core/stencils/fv_dynamics.py @@ -2,13 +2,13 @@ from datetime import timedelta from typing import Dict, Mapping, Optional -from dace.frontend.python.interface import nounroll as dace_no_unroll from gt4py.gtscript import PARALLEL, computation, interval, log import pace.dsl.gt4py_utils as utils import pace.fv3core.stencils.moist_cv as moist_cv import pace.util import pace.util.constants as constants +from dace.frontend.python.interface import nounroll as dace_no_unroll from pace.dsl.dace.orchestration import dace_inhibitor, orchestrate from pace.dsl.dace.wrapped_halo_exchange import WrappedHaloUpdater from pace.dsl.stencil import StencilFactory diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 3e178e018..52d8e02e2 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -15,12 +15,13 @@ from pace.physics.stencils.get_phi_fv3 import get_phi_fv3 from pace.physics.stencils.get_prs_fv3 import get_prs_fv3 from pace.physics.stencils.microphysics import Microphysics -from pace.util.grid import GridData - -from .._config import PhysicsConfig from pace.stencils.testing.translate_physics import ( reshape_pace_variable_to_fortran_format, ) +from pace.util.grid import GridData + +from .._config import PhysicsConfig + PHYSICS_PACKAGES = Literal["microphysics"] diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index 3b4262de5..115551a5a 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -10,10 +10,11 @@ from pace.dsl.stencil import StencilFactory from pace.dsl.typing import Float, FloatField from pace.stencils.fv_update_phys import ApplyPhysicsToDycore -from pace.util.grid import DriverGridData, GridData from pace.stencils.testing.translate_physics import ( reshape_pace_variable_to_fortran_format, ) +from pace.util.grid import DriverGridData, GridData + # TODO: when this file is not importable from physics or fv3core, import # PhysicsState and DycoreState and use them to type hint below From 4c85deaf554ebea3c8cadec5e64c3f563e001c28 Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Thu, 6 Oct 2022 17:01:29 -0700 Subject: [PATCH 46/48] resolve circular dependency --- Makefile | 2 +- physics/pace/physics/stencils/physics.py | 4 +--- stencils/pace/stencils/testing/translate.py | 13 +++++++++++++ stencils/pace/stencils/testing/translate_physics.py | 13 ------------- stencils/pace/stencils/update_atmos_state.py | 4 +--- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 0b606e73b..ffca07f7e 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ test_main: build test_savepoint: ## top level savepoint tests TARGET=dycore $(MAKE) get_test_data - $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(PACE_PATH)/tests/savepoint" + $(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) -k test_fv_dynamics $(PACE_PATH)/tests/savepoint" test_mpi_54rank: mpirun -n 54 $(MPIRUN_ARGS) python3 -m mpi4py -m pytest tests/mpi_54rank diff --git a/physics/pace/physics/stencils/physics.py b/physics/pace/physics/stencils/physics.py index 52d8e02e2..596a3f40d 100644 --- a/physics/pace/physics/stencils/physics.py +++ b/physics/pace/physics/stencils/physics.py @@ -15,9 +15,7 @@ from pace.physics.stencils.get_phi_fv3 import get_phi_fv3 from pace.physics.stencils.get_prs_fv3 import get_prs_fv3 from pace.physics.stencils.microphysics import Microphysics -from pace.stencils.testing.translate_physics import ( - reshape_pace_variable_to_fortran_format, -) +from pace.stencils.testing.translate import reshape_pace_variable_to_fortran_format from pace.util.grid import GridData from .._config import PhysicsConfig diff --git a/stencils/pace/stencils/testing/translate.py b/stencils/pace/stencils/testing/translate.py index 833249cbd..8542ec593 100644 --- a/stencils/pace/stencils/testing/translate.py +++ b/stencils/pace/stencils/testing/translate.py @@ -396,3 +396,16 @@ def python_grid(self): self.make_grid_storage(pygrid) pygrid.add_data(self.data) return pygrid + + +def reshape_pace_variable_to_fortran_format(data, grid_indexing): + cn2 = grid_indexing.domain[0] * grid_indexing.domain[1] + return np.reshape( + data[ + grid_indexing.isc : grid_indexing.iec + 1, + grid_indexing.jsc : grid_indexing.jec + 1, + 0 : grid_indexing.domain[2], + ].data, + (cn2, grid_indexing.domain[2]), + order="F", + )[:, ::-1] diff --git a/stencils/pace/stencils/testing/translate_physics.py b/stencils/pace/stencils/testing/translate_physics.py index 39bf74440..4b2c44d3c 100644 --- a/stencils/pace/stencils/testing/translate_physics.py +++ b/stencils/pace/stencils/testing/translate_physics.py @@ -191,16 +191,3 @@ class ParallelPhysicsTranslate2Py(ParallelTranslate2Py): def __init__(self, rank_grids, namelist, stencil_factory): physics_namelist = PhysicsConfig.from_namelist(namelist) super().__init__(rank_grids, physics_namelist, stencil_factory) - - -def reshape_pace_variable_to_fortran_format(data, grid_indexing): - cn2 = grid_indexing.domain[0] * grid_indexing.domain[1] - return np.reshape( - data[ - grid_indexing.isc : grid_indexing.iec + 1, - grid_indexing.jsc : grid_indexing.jec + 1, - 0 : grid_indexing.domain[2], - ].data, - (cn2, grid_indexing.domain[2]), - order="F", - )[:, ::-1] diff --git a/stencils/pace/stencils/update_atmos_state.py b/stencils/pace/stencils/update_atmos_state.py index 115551a5a..ccd7c97e7 100644 --- a/stencils/pace/stencils/update_atmos_state.py +++ b/stencils/pace/stencils/update_atmos_state.py @@ -10,9 +10,7 @@ from pace.dsl.stencil import StencilFactory from pace.dsl.typing import Float, FloatField from pace.stencils.fv_update_phys import ApplyPhysicsToDycore -from pace.stencils.testing.translate_physics import ( - reshape_pace_variable_to_fortran_format, -) +from pace.stencils.testing.translate import reshape_pace_variable_to_fortran_format from pace.util.grid import DriverGridData, GridData From 9fdd382628d3ac8b4a44cc71956f5911d48fea9e Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Thu, 6 Oct 2022 17:11:52 -0700 Subject: [PATCH 47/48] lint on the right python version --- driver/pace/driver/driver.py | 2 +- dsl/pace/dsl/dace/build.py | 1 + dsl/pace/dsl/dace/dace_config.py | 1 + dsl/pace/dsl/stencil.py | 2 +- fv3core/pace/fv3core/stencils/fv_dynamics.py | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/driver/pace/driver/driver.py b/driver/pace/driver/driver.py index aa7a50843..db27890e4 100644 --- a/driver/pace/driver/driver.py +++ b/driver/pace/driver/driver.py @@ -6,9 +6,9 @@ from math import floor from typing import Any, Dict, List, Tuple, Union +import dace import dacite -import dace import pace.driver import pace.dsl import pace.physics diff --git a/dsl/pace/dsl/dace/build.py b/dsl/pace/dsl/dace/build.py index 6e6d1635a..3b3f8b0fa 100644 --- a/dsl/pace/dsl/dace/build.py +++ b/dsl/pace/dsl/dace/build.py @@ -1,6 +1,7 @@ from typing import List, Optional, Tuple from dace.sdfg import SDFG + from pace.dsl.dace.dace_config import DaceConfig, DaCeOrchestration from pace.util import TilePartitioner diff --git a/dsl/pace/dsl/dace/dace_config.py b/dsl/pace/dsl/dace/dace_config.py index a1a165cdd..27c7ae903 100644 --- a/dsl/pace/dsl/dace/dace_config.py +++ b/dsl/pace/dsl/dace/dace_config.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Optional import dace.config + from pace.dsl.gt4py_utils import is_gpu_backend from pace.util.communicator import CubedSphereCommunicator diff --git a/dsl/pace/dsl/stencil.py b/dsl/pace/dsl/stencil.py index b80e53d48..bcfea4735 100644 --- a/dsl/pace/dsl/stencil.py +++ b/dsl/pace/dsl/stencil.py @@ -16,13 +16,13 @@ cast, ) +import dace import gt4py import numpy as np from gt4py import gtscript from gt4py.storage.storage import Storage from gtc.passes.oir_pipeline import DefaultPipeline, OirPipeline -import dace import pace.dsl.gt4py_utils as gt4py_utils import pace.util from pace.dsl.dace.orchestration import SDFGConvertible diff --git a/fv3core/pace/fv3core/stencils/fv_dynamics.py b/fv3core/pace/fv3core/stencils/fv_dynamics.py index fd97d6f31..0a0d93f03 100644 --- a/fv3core/pace/fv3core/stencils/fv_dynamics.py +++ b/fv3core/pace/fv3core/stencils/fv_dynamics.py @@ -2,13 +2,13 @@ from datetime import timedelta from typing import Dict, Mapping, Optional +from dace.frontend.python.interface import nounroll as dace_no_unroll from gt4py.gtscript import PARALLEL, computation, interval, log import pace.dsl.gt4py_utils as utils import pace.fv3core.stencils.moist_cv as moist_cv import pace.util import pace.util.constants as constants -from dace.frontend.python.interface import nounroll as dace_no_unroll from pace.dsl.dace.orchestration import dace_inhibitor, orchestrate from pace.dsl.dace.wrapped_halo_exchange import WrappedHaloUpdater from pace.dsl.stencil import StencilFactory From 87f51150a6816cde5d2f38c0aaa4397f98b0c25e Mon Sep 17 00:00:00 2001 From: Elynn Wu Date: Mon, 10 Oct 2022 14:10:49 -0700 Subject: [PATCH 48/48] remove unchecked variable --- fv3core/pace/fv3core/stencils/remapping.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fv3core/pace/fv3core/stencils/remapping.py b/fv3core/pace/fv3core/stencils/remapping.py index 95dc71dad..de44d951b 100644 --- a/fv3core/pace/fv3core/stencils/remapping.py +++ b/fv3core/pace/fv3core/stencils/remapping.py @@ -591,7 +591,6 @@ def __call__( omga=omga, ak=ak, bk=bk, - pfull=pfull, dp1=dp1, ) # TODO: remove unused arguments (and commented code that references them)