more than a few pylint edits

subtoolCli --> subtool_cli
remove unnecessary pass and add docstrings

.github/workflows/create_test_conda_env.yml

@@ -44,5 +44,5 @@ jobs:
         which python
         python --version
         $CONDA/envs/fre-cli/bin/python --version
-        # run pytest
-        pylint fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."
+        # run pytest, ignore netCDF4 Dataset import error
+        pylint --ignored-modules netCDF4 fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."

fre/app/__init__.py

@@ -1,4 +1,5 @@
+''' for fre.app imports '''
 from .maskAtmosPlevel import maskAtmosPlevel_subtool
-from .freapp import appCli
+from .freapp import app_cli
 
-__all__ = ["maskAtmosPlevel_subtool", "appCli"]
+__all__ = ["maskAtmosPlevel_subtool", "app_cli"]

fre/app/freapp.py

@@ -1,12 +1,19 @@
+#!/usr/bin/env python3
+''' fre app calls '''
+
+import time
+
 import click
+
 from .maskAtmosPlevel import maskAtmosPlevel_subtool
 from .generate_time_averages.generate_time_averages import generate
 
 @click.group(help=click.style(" - access fre app subcommands", fg=(250,154,90)))
-def appCli():
-    pass
+def app_cli():
+    ''' entry point to fre app click commands '''
+
 
-@appCli.command()
+@app_cli.command()
 @click.option("-i", "--infile",
               type=str,
               help="Input NetCDF file containing pressure-level output to be masked",
@@ -20,11 +27,12 @@ def appCli():
               required=True)
 @click.pass_context
 def mask_atmos_plevel(context, infile, outfile, psfile):
+    # pylint: disable=unused-argument
     """Mask out pressure level diagnostic output below land surface"""
     context.forward(maskAtmosPlevel_subtool)
 
 
-@appCli.command()
+@app_cli.command()
 @click.option("-i", "--inf",
               type=str,
               required=True,
@@ -57,13 +65,15 @@ def mask_atmos_plevel(context, infile, outfile, psfile):
               help="Compute standard deviations for time-averages as well")
 @click.pass_context
 def gen_time_averages(context, inf, outf, pkg, var, unwgt, avg_type, stddev_type):
+    # pylint: disable=unused-argument
     """
-    generate time averages for specified set of netCDF files. Example: generate-time-averages.py /path/to/your/files/
+    generate time averages for specified set of netCDF files. 
+    Example: generate-time-averages.py /path/to/your/files/
     """
-    import time
     start_time=time.perf_counter()
     context.forward(generate)
-    print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)') # need to change to a click.echo, not sure if echo supports f strings
+    # need to change to a click.echo, not sure if echo supports f strings
+    print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
 
 if __name__ == "__main__":
-    appCli()
+    app_cli()

fre/app/generate_time_averages/frepytoolsTimeAverager.py

@@ -1,4 +1,9 @@
 ''' class for python-native routine usuing netCDF4 and numpy to crunch time-averages '''
+
+import math
+import numpy
+from netCDF4 import Dataset
+
 from .timeAverager import timeAverager
 
 class frepytoolsTimeAverager(timeAverager):
@@ -9,8 +14,9 @@ class inheriting from abstract base class timeAverager
     '''
 
     def generate_timavg(self, infile=None, outfile=None):
-        ''' frepytools approach in a python-native manner. deliberately avoids pre-packaged routines. '''
-        assert(self.pkg=="fre-python-tools")
+        ''' frepytools approach in a python-native manner.
+        deliberately avoids pre-packaged routines. '''
+        assert self.pkg=="fre-python-tools"
         if __debug__:
             print(locals()) #input argument details
 
@@ -21,10 +27,6 @@ def generate_timavg(self, infile=None, outfile=None):
             print(f'ERROR: avg_type={self.avg_type} not supported at this time.')
             return 1
 
-        import math
-        import numpy
-        from netCDF4 import Dataset
-
         # (TODO) file I/O should be a sep function, no? make tests, extend
         nc_fin = Dataset(infile, 'r')
         if nc_fin.file_format != 'NETCDF4':
@@ -36,8 +38,8 @@ def generate_timavg(self, infile=None, outfile=None):
         # attempt to determine target var w/ bronx convention
         if self.var is not None:
             targ_var = self.var
-        else:
-            targ_var = infile.split('/').pop().split('.')[-2] # this can be replaced w/ a regex search maybe
+        else: # this can be replaced w/ a regex search maybe
+            targ_var = infile.split('/').pop().split('.')[-2]
 
         if __debug__:
             print(f'targ_var={targ_var}')
@@ -67,7 +69,9 @@ def generate_timavg(self, infile=None, outfile=None):
         fin_dims = nc_fin.dimensions
         num_time_bnds = fin_dims['time'].size
         if not self.unwgt: #compute sum of weights
-            wgts = numpy.moveaxis( time_bnds,0,-1)[1][:].copy() - numpy.moveaxis( time_bnds,0,-1)[0][:].copy()
+#            wgts = numpy.moveaxis( time_bnds,0,-1 )[1][:].copy() - numpy.moveaxis( time_bnds,0,-1 )[0][:].copy()
+            wgts = numpy.moveaxis( time_bnds,0,-1 )[1][:].copy() \
+                   - numpy.moveaxis( time_bnds,0,-1 )[0][:].copy()
             wgts_sum=sum(wgts)
             if __debug__:
                 print(f'wgts_sum={wgts_sum}')
@@ -134,9 +138,11 @@ def generate_timavg(self, infile=None, outfile=None):
 
 
 
-        # write output file, (TODO) make this a sep function, make tests, extend, consider compression particular;y for NETCDF file writing
-        # with Dataset( outfile, 'w', format='NETCDF4', persist=True ) as nc_fout: # consider this approach instead
-        #nc_fout= Dataset( outfile, 'w', format='NETCDF4', persist=True ) # how should we handle file formats (HDF5, Zarr, older NETCDF)?
+        # write output file
+        # (TODO) make this a sep function, make tests, extend,
+        # (TODO) consider compression particular;y for NETCDF file writing
+        # consider this approach instead:
+        #     with Dataset( outfile, 'w', format='NETCDF4', persist=True ) as nc_fout:
         nc_fout= Dataset( outfile, 'w', format=nc_fin.file_format, persist=True )
 
         # (TODO) make this a sep function, make tests, extend
@@ -156,7 +162,7 @@ def generate_timavg(self, infile=None, outfile=None):
                     print(f'could not get nc file attribute: {ncattr}. moving on.')
                     unwritten_ncattr_list.append(ncattr)
         if len(unwritten_ncattr_list)>0:
-            print(f'WARNING: Some global attributes ({unwritten_ncattr_list}) were not successfully written.')
+            print(f'WARNING: Some global attributes ({unwritten_ncattr_list}) were not written.')
         print('------- DONE writing output attributes. --------')
         ##
 
@@ -166,34 +172,40 @@ def generate_timavg(self, infile=None, outfile=None):
         unwritten_dims_list=[]
         for key in fin_dims:
             try:
-                if key=='time': # this strongly influences the final data structure shape of the averages.
-                    #nc_fout.createDimension( dimname=key, size=None ) # if set to None, and lets say you try to write
-                    #                                                  # e.g. the original 'time_bnds' (which has 60 time steps)
-                    #                                                  # the array holding the avg. value will suddently have 60 time steps
-                    #                                                  # even though only 1 is needed, 59 time steps will have no data
+                if key=='time':
+                    # this strongly influences the final data structure shape of the averages.
+                    # if set to None, and lets say you try to write
+                    # e.g. the original 'time_bnds' (which has 60 time steps)
+                    # the array holding the avg. value will suddently have 60 time steps
+                    # even though only 1 is needed, 59 time steps will have no data
+                    #nc_fout.createDimension( dimname=key, size=None )
                     nc_fout.createDimension( dimname=key, size=1)
                 else:
                     nc_fout.createDimension( dimname=key, size=fin_dims[key].size )
             except:
                 print(f'problem. cannot read/write dimension {key}')
                 unwritten_dims_list.append(key)
         if len(unwritten_dims_list)>0:
-            print(f'WARNING: Some dimensions ({unwritten_dims_list}) were not successfully written.')
+            print(f'WARNING: Some dimensions ({unwritten_dims_list}) were not written.')
         print('------ DONE writing output dimensions. ------- \n')
         ##
 
 
         # (TODO) make this a sep function, make tests, extend
         # first write the data we care most about- those we computed
+        # copying metadata, not fully correct
+        # but not far from wrong according to CF
+        # cell_methods must be changed TO DO
         print(f'\n------- writing data for data {targ_var} -------- ')
         nc_fout.createVariable(targ_var, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions)
-        nc_fout.variables[targ_var].setncatts(nc_fin[targ_var].__dict__) # copying metadata, not fully correct
-                                                                         # but not far from wrong according to CF
-                                                                         # cell_methods must be changed TO DO
+        nc_fout.variables[targ_var].setncatts(nc_fin[targ_var].__dict__)
+
+
         nc_fout.variables[targ_var][:]=avgvals
         if self.stddev_type is not None:
             stddev_varname=targ_var+'_'+self.stddev_type+'_stddev'
-            nc_fout.createVariable(stddev_varname, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions)
+            nc_fout.createVariable(
+                stddev_varname, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions )
             nc_fout.variables[stddev_varname].setncatts(nc_fin[targ_var].__dict__)
             nc_fout.variables[stddev_varname][:]=stddevs
         print('---------- DONE writing output variables. ---------')
@@ -217,15 +229,15 @@ def generate_timavg(self, infile=None, outfile=None):
                     print(f'nc_fin[var].shape={nc_fin[var].shape}')
                     #print(f'len(nc_fout.variables[{var}])={len(nc_fout.variables[var])}')
                     nc_fout.variables[var][:] = [ nc_fin[var][0] ]
-                    print(f'is it a time variable? {self.var_has_time_units(nc_fin.variables[var])}')
+                    print(f'time variable? {self.var_has_time_units(nc_fin.variables[var])}')
             else:
                 continue
 
         if len(unwritten_var_list)>0:
-            print(f'WARNING: some variables\' data ({unwritten_var_list}) was not successfully written.')
+            print(f'WARNING: some variables\' data ({unwritten_var_list}) was not written.')
         if len(unwritten_var_ncattr_dict)>0:
-            print(f'WARNING: some variables\' metadata was not successfully written.')
-            print(f'WARNING: relevant variable (key) and attribute (value) pairs: \n{unwritten_var_ncattr_dict}')
+            print('WARNING: some variables\' metadata was not successfully written.')
+            print(f'WARNING: relevant variable/attr pairs: \n{unwritten_var_ncattr_dict}')
         print('---------- DONE writing output variables. ---------')
         ##
 

fre/catalog/__init__.py

@@ -1,4 +1,4 @@
 #from .gen_intake_gfdl import build_script
-from .frecatalog import catalogCli
+from .frecatalog import catalog_cli
 
-__all__ = ["catalogCli"]
+__all__ = ["catalog_cli"]

fre/catalog/frecatalog.py

@@ -5,12 +5,12 @@
 
 
 @click.group(help=click.style(" - access fre catalog subcommands", fg=(64,94,213)))
-def catalogCli():
+def catalog_cli():
     ''' entry point for click into fre catalog cli calls '''
-    pass
 
 
-@catalogCli.command()
+
+@catalog_cli.command()
 #TODO arguments dont have help message. So consider changing arguments to options?
 @click.argument('input_path', required = False, nargs = 1)
 #, help = 'The directory path with the datasets to be cataloged. E.g a GFDL PP path till /pp')
@@ -26,17 +26,19 @@ def catalogCli():
 @click.pass_context
 def builder(context, input_path = None, output_path = None, config = None, filter_realm = None,
             filter_freq = None, filter_chunk = None, overwrite = False, append = False):
+    # pylint: disable=unused-argument
     """ - Generate .csv and .json files for catalog """
     context.forward(gen_intake_gfdl.create_catalog_cli)
 
-@catalogCli.command()
+@catalog_cli.command()
 @click.argument('json_path', nargs = 1 , required = True)
 @click.argument('json_template_path', nargs = 1 , required = False)
 @click.option('-tf', '--test-failure', is_flag=True, default = False, help="Errors are only printed. Program will not exit.")
 @click.pass_context
 def validate(context, json_path, json_template_path, test_failure):
+    # pylint: disable=unused-argument
     """ - Validate a catalog against catalog schema """
     context.forward(test_catalog.main)
 
 if __name__ == "__main__":
-    catalogCli()
+    catalog_cli()

fre/check/__init__.py

@@ -1,4 +1,4 @@
 from .frecheckexample import check_test_function
-from .frecheck import checkCli
+from .frecheck import check_cli
 
-__all__ = ["check_test_function", "checkCli"]
+__all__ = ["check_test_function", "check_cli"]

fre/check/frecheck.py

@@ -2,15 +2,15 @@
 from .frecheckexample import check_test_function
 
 @click.group(help=click.style(" - access fre check subcommands", fg=(162,91,232)))
-def checkCli():
-    pass
+def check_cli():
+    ''' entry point to fre check click commands '''
 
-@checkCli.command()
+@check_cli.command()
 @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
 @click.pass_context
 def function(context, uppercase):
     """ - Execute fre check test """
     context.forward(check_test_function)
 
 if __name__ == "__main__":
-    checkCli()
+    check_cli()

fre/cmor/__init__.py

@@ -1,4 +1,4 @@
 from .CMORmixer import cmor_run_subtool
-from .frecmor import cmorCli
+from .frecmor import cmor_cli
 
-__all__ = ["cmor_run_subtool", "cmorCli"]
+__all__ = ["cmor_run_subtool", "cmor_cli"]

fre/cmor/frecmor.py

@@ -2,10 +2,10 @@
 from .CMORmixer import cmor_run_subtool
 
 @click.group(help=click.style(" - access fre cmor subcommands", fg=(232,91,204)))
-def cmorCli():
-    pass
+def cmor_cli():
+    ''' entry point to fre cmor click commands '''
 
-@cmorCli.command()
+@cmor_cli.command()
 @click.option("-d", "--indir",
               type=str,
               help="Input directory",
@@ -32,4 +32,4 @@ def run(context, indir, outdir, varlist, table_config, exp_config):
     context.forward(cmor_run_subtool)
 
 if __name__ == "__main__":
-    cmorCli()
+    cmor_cli()

fre/fre.py

@@ -12,16 +12,16 @@
 
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.ppCli",
-                      "catalog": ".catalog.frecatalog.catalogCli",
-                      "list": ".list.frelist.listCli",
-                      "check": ".check.frecheck.checkCli",
-                      "run": ".run.frerun.runCli",
-                      "test": ".test.fretest.testCli",
-                      "yamltools": ".yamltools.freyamltools.yamltoolsCli",
-                      "make": ".make.fremake.makeCli",
-                      "app": ".app.freapp.appCli",
-                      "cmor": ".cmor.frecmor.cmorCli"                      },
+    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+                       "catalog": ".catalog.frecatalog.catalog_cli",
+                       "list": ".list.frelist.list_cli",
+                       "check": ".check.frecheck.check_cli",
+                       "run": ".run.frerun.run_cli",
+                       "test": ".test.fretest.test_cli",
+                       "yamltools": ".yamltools.freyamltools.yamltools_cli",
+                       "make": ".make.fremake.make_cli",
+                       "app": ".app.freapp.app_cli",
+                       "cmor": ".cmor.frecmor.cmor_cli"                      },
     help = click.style(
         "'fre' is the main CLI click group that houses the other tool groups as lazy subcommands.",
         fg='cyan')
@@ -35,7 +35,7 @@
 
 def fre():
     ''' entry point function to subgroup functions '''
-    pass
+
 
 if __name__ == '__main__':
     fre()

fre/list/__init__.py

@@ -1,4 +1,4 @@
 from .frelistexample import list_test_function
-from .frelist import listCli
+from .frelist import list_cli
 
-__all__ = ["list_test_function", "listCli"]
+__all__ = ["list_test_function", "list_cli"]

fre/list/frelist.py

@@ -2,15 +2,15 @@
 from .frelistexample import list_test_function
 
 @click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91)))
-def listCli():
+def list_cli():
     pass
 
-@listCli.command()
+@list_cli.command()
 @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
 @click.pass_context
 def function(context, uppercase):
     """ - Execute fre list test """
     context.forward(list_test_function)
 
 if __name__ == "__main__":
-    listCli()
+    list_cli()

fre/make/__init__.py

@@ -7,11 +7,11 @@
 from .createDocker import dockerfile_create
 from .createMakefile import makefile_create
 from .runFremake import fremake_run
-from .fremake import makeCli
+from .fremake import make_cli
 
 __all__ = ["checkout_create",
            "compile_create",
            "dockerfile_create", 
            "makefile_create",
            "fremake_run",
-           "makeCli"]
+           "make_cli"]