Updates to make WW3 thread safe (NOAA-EMC#315)

* making TAUNUX, TAUNUY threadprivate
- update from @JamesAbeles-NOAA
* adding gaae and jet to WW3 cmplr, also adding '.intel' to NCEP
machines for future flexibility of having '*.gnu' options
* adding thread private for w3canomd since it's called from
W3SREF which is called in w3srce which is threaded
* added more saved variables to threadprivate
* fix cmpr.env for typo and add files to gitignore
* removing two files that should  have been removed in a previous PR

.gitignore

@@ -52,8 +52,8 @@ regtests/*/input*/*/*.nc
 regtests/*/input*/partition.ww3
 regtests/*/*.png
 regtests/ww3_tp2.14/input/oasis3-mct/doc
-regtests/ww3_tp2.14/input*/*.nc.OAS*CM
-regtests/ww3_tp2.14/input*/*/*.nc.OAS*CM
+regtests/ww3_tp2.14/input*/*.nc.OAS*CM*
+regtests/ww3_tp2.14/input*/*/*.nc.OAS*CM*
 regtests/ww3_tp2.17/input/inlet.msh
 *.nc
 */*.nc

model/bin/cmplr.env

@@ -109,8 +109,9 @@ fi
 if [ "$cmplr" == "intel" ]          || [ "$cmplr" == "intel_debug" ]    || [ "$cmplr" == "intel_prof" ]    || \
    [ "$cmplr" == "so_intel" ]       || [ "$cmplr" == "so_intel_debug" ] || [ "$cmplr" == "so_intel_prof" ] || \
    [ "$cmplr" == "datarmor_intel" ] || [ "$cmplr" == "datarmor_intel_debug" ] || [ "$cmplr" == "datarmor_intel_prof" ] || \
-   [ "$cmplr" == "wcoss_cray" ]     || [ "$cmplr" == "wcoss_dell_p3" ]  || [ "$cmplr" == "cheyenne" ]      || \
-   [ "$cmplr" == "hera" ] || [ "$cmplr" == "orion" ] || [ "$cmplr" == "stampede" ] ; then
+   [ "$cmplr" == "wcoss_cray" ]     || [ "$cmplr" == "wcoss_dell_p3" ]  || [ "$cmplr" == "cheyenne.intel" ]      || \
+   [ "$cmplr" == "gaea.intel" ]     || [ "$cmplr" == "jet.intel" ]      || \
+   [ "$cmplr" == "hera.intel" ]     || [ "$cmplr" == "orion.intel" ]    || [ "$cmplr" == "stampede.intel" ] ; then
 
 
   # COMPILER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -120,11 +121,11 @@ if [ "$cmplr" == "intel" ]          || [ "$cmplr" == "intel_debug" ]    || [ "$c
   comp_mpi='mpiifort'
 
   # cray compiler
-  if [ ! -z "$(echo $cmplr | grep wcoss_cray)" ] ; then
+  if [ ! -z "$(echo $cmplr | grep wcoss_cray)" ]  || [ "$cmplr" == "gaea.intel" ]; then
     comp_seq='ftn'
     comp_mpi='ftn'
   fi
-
+  
   if [ ! -z "$(echo $cmplr | grep cheyenne)" ] ; then
     comp_seq='ifort'
     comp_mpi='mpif90'
@@ -156,9 +157,10 @@ if [ "$cmplr" == "intel" ]          || [ "$cmplr" == "intel_debug" ]    || [ "$c
   if [ "$list" == 'yes' ] ; then optc="$optc -list"; fi
 
   # omp options
-  if [ ! -z "$(echo $cmplr | grep datarmor)" ] || [ "$cmplr" == "hera" ] || [ "$cmplr" == "orion" ] || \
-     [ "$cmplr" == "wcoss_cray" ] || [ "$cmplr" == "wcoss_dell_p3" ] || [ "$cmplr" == "cheyenne" ] || \
-     [ "$cmplr" == "stampede" ]; then
+  if [ ! -z "$(echo $cmplr | grep datarmor)" ] || [ "$cmplr" == "hera.intel" ] || [ "$cmplr" == "orion.intel" ] || \
+     [ "$cmplr" == "wcoss_cray" ] || [ "$cmplr" == "wcoss_dell_p3" ] || [ "$cmplr" == "cheyenne.intel" ] || \
+     [ "$cmplr" == "gaea.intel" ] || [ "$cmplr" == "jet.intel" ] || \
+     [ "$cmplr" == "stampede.intel" ]; then
      optomp="-qopenmp"
   else
      optomp="-openmp"
@@ -205,6 +207,7 @@ fi
 
 if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || [ "$cmplr" == "gnu_prof" ] || \
    [ "$cmplr" == "so_gnu" ] || [ "$cmplr" == "so_gnu_debug" ] || [ "$cmplr" == "so_gnu_prof" ] || \
+   [ "$cmplr" == "hera.gnu" ] || [ "$cmplr" == "cheyenne.gnu" ] || \
    [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || [ "$cmplr" == "datarmor_gnu_prof" ] || \
    [ "$cmplr" == "ukmo_cray_gnu" ] || [ "$cmplr" == "ukmo_cray_gnu_debug" ] || [ "$cmplr" == "ukmo_cray_gnu_prof" ] ; then
 

model/bin/w3_setup

@@ -414,9 +414,10 @@ then
        [ "$cmplr" == "so_intel" ] || [ "$cmplr" == "so_intel_debug" ]               || \
        [ "$cmplr" == "datarmor_intel" ] || [ "$cmplr" == "datarmor_intel_debug" ]   || \
        [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ]                         || \
-       [ "$cmplr" == "hera" ] || [ "$cmplr" == "orion" ]                            || \
-       [ "$cmplr" == "stampede" ]                                                   || \
-       [ "$cmplr" == "cheyenne" ]                                                   || \
+       [ "$cmplr" == "hera.intel" ] || [ "$cmplr" == "orion.intel" ]                || \
+       [ "$cmplr" == "hera.gnu" ]   || [ "$cmplr" == "jet.intel" ]                  || \
+       [ "$cmplr" == "stampede.intel" ] || [ "$cmplr" == "gaea.intel" ]             || \
+       [ "$cmplr" == "cheyenne.intel" ] || [ "$cmplr" == "cheyenne.gnu" ]           || \
        [ "$cmplr" == "wcoss_cray" ] || [ "$cmplr" == "wcoss_dell_p3" ]              || \
        [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ]       || \
        [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ]                         || \
@@ -441,9 +442,10 @@ then
        [ "$cmplr" == "so_intel" ] || [ "$cmplr" == "so_intel_debug" ]               || \
        [ "$cmplr" == "datarmor_intel" ] || [ "$cmplr" == "datarmor_intel_debug" ]   || \
        [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ]                         || \
-       [ "$cmplr" == "hera" ] || [ "$cmplr" == "orion" ]                            || \
-       [ "$cmplr" == "stampede" ]                                                   || \
-       [ "$cmplr" == "cheyenne" ]                                                   || \
+       [ "$cmplr" == "hera.intel" ] || [ "$cmplr" == "orion.intel" ]                || \
+       [ "$cmplr" == "hera.gnu" ]   || [ "$cmplr" == "jet.intel" ]                  || \
+       [ "$cmplr" == "stampede.intel" ] || [ "$cmplr" == "gaea.intel" ]             || \
+       [ "$cmplr" == "cheyenne.intel" ] || [ "$cmplr" == "cheyenne.gnu" ]           || \
        [ "$cmplr" == "wcoss_cray" ] || [ "$cmplr" == "wcoss_dell_p3" ]              || \
        [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ]       || \
        [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ]                         || \

model/esmf/Makefile

@@ -35,9 +35,11 @@ ifeq ($(WW3_COMP),Portland)
 else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","pgi" "datarmor_pgi" "datarmor_pgi_debug"))
  ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -byteswapio
 # intel
-else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","Intel" "hera" "orion" "cheyenne" "stampede"))
+else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","Intel" "hera.intel" "orion.intel" "jet.intel")) 
  ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
-else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_cray" "wcoss_dell_p3"))
+else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)", "cheyenne.intel" "stampede.intel"))
+ ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
+else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_cray" "wcoss_dell_p3" "gaea.intel"))
  ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
 else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","intel" "datarmor_intel" "datarmor_intel_debug"))
  ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian
@@ -47,6 +49,8 @@ else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","mpt" "datarmor_mpt" "datarmor_m
 # gnu
 else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","gnu" "datarmor_gnu" "datarmor_gnu_debug"))
  ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -fconvert=big-endian
+else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","hera.gnu" "cheyenne.gnu"))
+ ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -fconvert=big-endian
 # error
 else
  $(error Unsupported WW3_COMP: $(WW3_COMP))

model/ftn/w3canomd.ftn

@@ -93,12 +93,16 @@
       INTEGER                  :: NDEPTH
       REAL                     :: DEPTHA         ! first depth in table
       REAL, SAVE , PRIVATE, ALLOCATABLE   :: OMEGA(:)
+!/OMPG/!$omp threadprivate( OMEGA )
       INTEGER, SAVE , PRIVATE             :: COUNTER = 0
+!/OMPG/!$omp threadprivate( COUNTER )
 ! Tables for non-linear coefficients ...   
       REAL, SAVE , PRIVATE, ALLOCATABLE   :: TA(:,:,:,:),TB(:,:,:,:),TC_QL(:,:,:,:),&
                                              TT_4M(:,:,:,:),TT_4P(:,:,:,:),TFAKH(:,:),    &
                                              TFAK(:,:)
+!/OMPG/!$omp threadprivate( TA, TB, TC_QL, TT_4M, TT_4P, TFAKH, TFAK )
       INTEGER, SAVE, PRIVATE, ALLOCATABLE :: IM_P(:,:),IM_M(:,:)
+!/OMPG/!$omp threadprivate( IM_P, IM_M )
 
 
 !
@@ -202,10 +206,13 @@
       REAL              :: CO1, ATOE, DPTH
 !/S      INTEGER, SAVE           :: IENT = 0
       LOGICAL, SAVE     :: FIRST = .TRUE.
+!/OMPG/!$omp threadprivate( FIRST ) 
       REAL, ALLOCATABLE, SAVE  :: FR(:), DFIM(:)
       REAL, ALLOCATABLE, SAVE  :: F1(:,:), F3(:,:)
+!/OMPG/!$omp threadprivate( FR, DFIM, F1, F3 ) 
       INTEGER, SAVE     ::  NFRE, NANG
       INTEGER, SAVE     ::  NFREH, NANGH
+!/OMPG/!$omp threadprivate( NFRE, NANG, NFREH, NANGH )
 !/
 !/ ------------------------------------------------------------------- /
 !/
@@ -342,11 +349,12 @@
 
       INTEGER MDW,M,K, K0,M0,MP,KP,MM,KM,KL,KLL,ML,JD
       INTEGER, SAVE :: MR, MA,NMAX
-
+!/OMPG/!$omp threadprivate( MR, MA, NMAX )
 
 !      PARAMETER (NFREH=32,NANGH=36)
 
       INTEGER, SAVE            :: INDEP
+!/OMPG/!$omp threadprivate( INDEP )
       REAL,ALLOCATABLE         :: PF1(:,:),PF3(:,:)
 
 
@@ -357,7 +365,7 @@
      C2,XM,XK
       REAL, SAVE :: OMSTART
       REAL, SAVE :: XMR,XMA, DELTHH, CO1
-
+!/OMPG/!$omp threadprivate( OMSTART, XMR,XMA, DELTHH, CO1 )
       REAL :: F13(NFREH,NANGH)
       REAL :: SUM0,AKMEAN
       REAL :: DELOM(NFREH),THH(NANGH),DFDTH(NFREH)

model/ftn/w3fld1md.ftn

@@ -74,11 +74,14 @@
       PUBLIC
       ! Tail_Choice: Chose the method to determine the level of the tail
       INTEGER, SAVE :: Tail_Choice
+!/OMPG/!$omp threadprivate(Tail_Choice)
       REAL, SAVE    :: Tail_Level !if Tail_Choice=0, tail is constant
       REAL, SAVE    :: Tail_transition_ratio1! freq/fpi where tail 
                                              !  adjustment begins
       REAL, SAVE    :: Tail_transition_ratio2! freq/fpi where tail 
                                              !  adjustment ends
+!/OMPG/!$omp threadprivate(Tail_Level)
+!/OMPG/!$omp threadprivate(Tail_transition_ratio1,Tail_transition_ratio2)
 !/
       CONTAINS
 !/ ------------------------------------------------------------------- /
@@ -228,6 +231,7 @@
       LOGICAL                 :: FSFL1,FSFL2, CRIT1, CRIT2
       LOGICAL                 :: IT_FLAG1, IT_FLAG2
       LOGICAL, SAVE           :: FIRST = .TRUE.
+!/OMPG/!$omp threadprivate( FIRST)
 !/
 !/ ------------------------------------------------------------------- /
 !/

model/ftn/w3fld2md.ftn

@@ -206,6 +206,7 @@
       INTEGER :: COUNT
 !/S      INTEGER, SAVE           :: IENT = 0
       LOGICAL, SAVE           :: FIRST = .TRUE.
+!/OMPG/!$omp threadprivate( FIRST ) 
 !/
 !/ ------------------------------------------------------------------- /
 !/

model/ftn/w3flxxmd.ftn

@@ -180,6 +180,7 @@
 !/
 !/S      INTEGER, SAVE           :: IENT = 0
       LOGICAL, SAVE           :: FIRST = .TRUE.
+!/OMPG/!$omp threadprivate( FIRST ) 
 !/
 !/ ------------------------------------------------------------------- /
 !/

model/ftn/w3srcemd.ftn

@@ -511,12 +511,14 @@
 !/ST4      REAL                    :: BRLAMBDA(NSPEC)
 !/IS2      DOUBLE PRECISION        :: SCATSPEC(NTH) 
       REAL                    :: FOUT(NK,NTH), SOUT(NK,NTH), DOUT(NK,NTH)
-!/FLD1     REAL, SAVE              :: TAUNUX, TAUNUY
-!/FLD2     REAL, SAVE              :: TAUNUX, TAUNUY
+      REAL, SAVE              :: TAUNUX, TAUNUY
+!/OMPG/!$omp threadprivate( TAUNUX, TAUNUY)
       LOGICAL, SAVE           :: FLTEST = .FALSE., FLAGNN = .TRUE.
+!/OMPG/!$omp threadprivate( FLTEST, FLAGNN )
       LOGICAL                 :: SHAVE
       LOGICAL                 :: LBREAK
       LOGICAL, SAVE           :: FIRST = .TRUE.
+!/OMPG/!$omp threadprivate( FIRST ) 
       LOGICAL                 :: PrintDeltaSmDA
       REAL                    :: eInc1, eInc2
       REAL                    :: DeltaSRC(NSPEC), MAXDAC(NSPEC)