Merge branch 'main' into ADM-match-dups

bin/make_new_sv

@@ -6,7 +6,7 @@ import numpy as np
 #import warnings
 #warnings.simplefilter('error')
 
-from pkg_resources import resource_filename
+from importlib import resources
 from glob import glob
 from time import time
 start = time()
@@ -19,7 +19,7 @@ ap = ArgumentParser("Make a new sv directory, based on the highest existing sv d
 _ = ap.parse_args()
 
 # ADM the standard root name for a desitarget SV directory.
-svroot = resource_filename('desitarget', 'sv')
+svroot = str(resources.files('desitarget').joinpath('sv'))
 
 # ADM retrieve the highest current sv directory.
 fns = sorted(glob(svroot+'*'))

doc/changes.rst

@@ -5,6 +5,7 @@ desitarget Change Log
 2.7.1 (unreleased)
 ------------------
 
+* Fix `pkg_resources`/`scipy.ndimage`/`pytest` deprecations [`PR #829`_].
 * Change match routines to test input arrays for duplicates [`PR #828`_].
     * Addresses `issue #811`_.
 * Add `GAIA_` prefix to `PHOT_*_MEAN_MAG` column names [`PR #827`_].
@@ -40,6 +41,7 @@ desitarget Change Log
 .. _`PR #826`: https://github.com/desihub/desitarget/pull/826
 .. _`PR #827`: https://github.com/desihub/desitarget/pull/827
 .. _`PR #828`: https://github.com/desihub/desitarget/pull/828
+.. _`PR #829`: https://github.com/desihub/desitarget/pull/829
 
 2.7.0 (2023-12-05)
 ------------------

py/desitarget/cmx/cmx_cuts.py

@@ -23,7 +23,7 @@
 from astropy.coordinates import SkyCoord
 from astropy.table import Table, Row
 
-from pkg_resources import resource_filename
+from importlib import resources
 
 from desitarget import io
 from desitarget.cuts import _psflike, _is_row, _get_colnames, _prepare_gaia
@@ -1008,7 +1008,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, w1flux=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # ADM Use RF trained over DR7
         rf_fileName = pathToRF + '/rf_model_dr7.npz'
         rf_HighZ_fileName = pathToRF + '/rf_model_dr7_HighZ.npz'

py/desitarget/cmx/ms_cuts.py

@@ -11,7 +11,7 @@
 """
 
 import numpy as np
-from pkg_resources import resource_filename
+from importlib import resources
 from desitarget.geomask import imaging_mask
 
 # ADM set up the DESI default logger
@@ -545,7 +545,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, maskbits=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # rf filenames
         rf_DR3_fileName = pathToRF + '/rf_model_dr3.npz'
         rf_DR7_fileName = pathToRF + '/rf_model_dr7.npz'

py/desitarget/cuts.py

@@ -23,7 +23,7 @@
 import fitsio
 import numpy as np
 import healpy as hp
-from pkg_resources import resource_filename
+from importlib import resources
 import numpy.lib.recfunctions as rfn
 from importlib import import_module
 
@@ -1904,7 +1904,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, maskbits=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files.
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # rf filename.
         rf_DR9_fileName = pathToRF + '/rf_model_dr9_final.npz'
 

py/desitarget/io.py

@@ -22,7 +22,7 @@
 import healpy as hp
 from glob import glob, iglob
 from time import time
-from pkg_resources import resource_filename
+from importlib import resources
 import yaml
 import hashlib
 
@@ -435,7 +435,7 @@ def write_with_units(filename, data, extname=None, header=None, ecsv=False):
           conversion is applied to `data`.
     """
     # ADM read the desitarget units yaml file.
-    fn = resource_filename('desitarget', os.path.join('data', 'units.yaml'))
+    fn = resources.files('desitarget').joinpath('data/units.yaml')
     with open(fn) as f:
         unitdict = yaml.safe_load(f)
 

py/desitarget/mock/mockmaker.py

@@ -10,7 +10,7 @@
 import os
 import numpy as np
 from glob import glob
-from pkg_resources import resource_filename
+from importlib import resources
 
 import fitsio
 import healpy as hp
@@ -514,11 +514,11 @@ def read_GMM(self, target=None):
                 pass
                 #return
 
-            gmmdir = resource_filename('desitarget', 'mock/data/dr9')
+            gmmdir = resources.files('desitarget').joinpath('mock/data/dr9')
             if not os.path.isdir:
                 log.warning('DR9 GMM directory {} not found!'.format(gmmdir))
                 raise IOError
-            
+
             fracfile = os.path.join(gmmdir, 'fractype_{}.fits'.format(target.lower()))
             fractype = Table.read(fracfile)
 

py/desitarget/skyfibers.py

@@ -15,9 +15,9 @@
 import photutils
 import healpy as hp
 from glob import glob
-from scipy.ndimage.morphology import binary_dilation, binary_erosion
-from scipy.ndimage.measurements import label, find_objects, center_of_mass
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import binary_dilation, binary_erosion
+from scipy.ndimage import label, find_objects, center_of_mass
+from scipy.ndimage import gaussian_filter
 
 # ADM some utility code taken from legacypipe and astrometry.net.
 from desitarget.skyutilities.astrometry.fits import fits_table

py/desitarget/streams/utilities.py

@@ -13,7 +13,7 @@
 import numpy as np
 import astropy.coordinates as acoo
 import astropy.units as auni
-from pkg_resources import resource_filename
+from importlib import resources
 from scipy.interpolate import UnivariateSpline
 from time import time
 import desitarget.streams.gaia_dr3_parallax_zero_point.zpt as gaia_zpt
@@ -396,7 +396,7 @@ def get_stream_parameters(stream_name):
     stream_name = stream_name.upper()
 
     # ADM open and load the parameter yaml file.
-    fn = resource_filename('desitarget', os.path.join('data', 'streams.yaml'))
+    fn = resources.files('desitarget').joinpath('data/streams.yaml')
     with open(fn) as f:
         stream = yaml.safe_load(f)
 

py/desitarget/sv1/sv1_cuts.py

@@ -16,7 +16,7 @@
 import warnings
 
 from time import time
-from pkg_resources import resource_filename
+from importlib import resources
 import healpy as hp
 import fitsio
 
@@ -780,7 +780,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, w1flux=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # Use RF trained over DR9
         rf_fileName = pathToRF + '/rf_model_dr9.npz'
         rf_HighZ_fileName = pathToRF + '/rf_model_dr9_HighZ.npz'
@@ -911,7 +911,7 @@ def isQSO_highz_faint(gflux=None, rflux=None, zflux=None, w1flux=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files.
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # Use RF trained over DR9.
         rf_fileName = pathToRF + '/rf_model_dr9.npz'
 

py/desitarget/sv2/sv2_cuts.py

@@ -20,7 +20,7 @@
 import fitsio
 import numpy as np
 import healpy as hp
-from pkg_resources import resource_filename
+from importlib import resources
 import numpy.lib.recfunctions as rfn
 from importlib import import_module
 
@@ -1516,7 +1516,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, maskbits=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # rf filenames
         rf_DR3_fileName = pathToRF + '/rf_model_dr3.npz'
         rf_DR7_fileName = pathToRF + '/rf_model_dr7.npz'

py/desitarget/sv3/sv3_cuts.py

@@ -20,7 +20,7 @@
 import fitsio
 import numpy as np
 import healpy as hp
-from pkg_resources import resource_filename
+from importlib import resources
 import numpy.lib.recfunctions as rfn
 from importlib import import_module
 
@@ -1721,7 +1721,7 @@ def isQSO_randomforest(gflux=None, rflux=None, zflux=None, maskbits=None,
         colorsReducedIndex = colorsIndex[preSelection]
 
         # Path to random forest files
-        pathToRF = resource_filename('desitarget', 'data')
+        pathToRF = str(resources.files('desitarget').joinpath('data'))
         # rf filenames
         rf_DR3_fileName = pathToRF + '/rf_model_dr3.npz'
         rf_DR7_fileName = pathToRF + '/rf_model_dr7.npz'

py/desitarget/targetmask.py

@@ -7,7 +7,7 @@
 import os.path
 from desiutil.bitmask import BitMask
 import yaml
-from pkg_resources import resource_filename
+from importlib import resources
 
 
 def load_mask_bits(prefix=""):
@@ -18,7 +18,7 @@ def load_mask_bits(prefix=""):
         us = '_'
     prename = prefix+us
     fn = os.path.join(prefix, "data", "{}targetmask.yaml".format(prename))
-    _filepath = resource_filename('desitarget', fn)
+    _filepath = resources.files('desitarget').joinpath(fn)
     with open(_filepath) as fx:
         bitdefs = yaml.safe_load(fx)
         try:

py/desitarget/test/__init__.py

@@ -1,8 +1,5 @@
-def test_suite():
-    """Returns unittest.TestSuite for this package"""
-    import unittest
-    from os.path import dirname
-    basedir = dirname(dirname(__file__))
-    # print(desispec_dir)
-    return unittest.defaultTestLoader.discover(basedir,
-                                               top_level_dir=dirname(basedir))
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from .desitarget_test_suite import runtests

py/desitarget/test/make_testgaia.py

@@ -7,7 +7,7 @@
     import fitsio
     import numpy as np
     from time import time
-    from pkg_resources import resource_filename
+    from importlib import resources
     from desitarget.gaiamatch import find_gaia_files
     from desitarget.tychomatch import find_tycho_files
     from desitarget.uratmatch import find_urat_files
@@ -17,7 +17,7 @@
 
     # ADM choose the Gaia files to cover the same object
     # ADM locations as the sweeps/tractor files.
-    datadir = resource_filename('desitarget.test', 't')
+    datadir = resources.files('desitarget').joinpath('test/t')
     tractorfiles = sorted(io.list_tractorfiles(datadir))
     sweepfiles = sorted(io.list_sweepfiles(datadir))
 

py/desitarget/test/test_brick_fluctuation.py

@@ -57,15 +57,3 @@ def test_generate_depths(self):
             self.assertTrue(isinstance(self.depth[k], np.ndarray))
             self.assertEqual(len(self.depth[k]), len(self.b['RA']))
             self.assertEqual(len(self.depth[k]), len(self.b['DEC']))
-
-
-if __name__ == '__main__':
-    unittest.main()
-
-
-def test_suite():
-    """Allows testing of only this module with the command:
-
-        python setup.py test -m desitarget.test.test_brick_fluctuation
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)

py/desitarget/test/test_brightmask.py

@@ -3,7 +3,7 @@
 """Test desitarget.brightmask.
 """
 import unittest
-from pkg_resources import resource_filename
+from importlib import resources
 import os
 import sys
 import fitsio
@@ -28,12 +28,12 @@ class TestBRIGHTMASK(unittest.TestCase):
     def setUpClass(cls):
         # ADM set up the necessary environment variables.
         cls.gaiadir_orig = os.getenv("GAIA_DIR")
-        testdir = 'desitarget.test'
-        os.environ["GAIA_DIR"] = resource_filename(testdir, 't4')
+        testdir = resources.files('desitarget').joinpath('test')
+        os.environ["GAIA_DIR"] = os.path.join(testdir, 't4')
         cls.tychodir_orig = os.getenv("TYCHO_DIR")
-        os.environ["TYCHO_DIR"] = resource_filename(testdir, 't4/tycho')
+        os.environ["TYCHO_DIR"] = os.path.join(testdir, 't4', 'tycho')
         cls.uratdir_orig = os.getenv("URAT_DIR")
-        os.environ["URAT_DIR"] = resource_filename(testdir, 't4/urat')
+        os.environ["URAT_DIR"] = os.path.join(testdir, 't4', 'urat')
 
         # ADM a temporary output directory to test writing masks.
         cls.maskdir = tempfile.mkdtemp()
@@ -60,7 +60,7 @@ def setUpClass(cls):
             maglim=cls.maglim, maskepoch=cls.maskepoch)
 
         # ADM read in some targets.
-        targdir = resource_filename(testdir, 't')
+        targdir = os.path.join(testdir, 't')
         fn = os.path.join(targdir, 'sweep-320m005-330p000.fits')
         ts = fitsio.read(fn)
         # ADM targets are really sweeps objects, so add target fields.
@@ -262,15 +262,3 @@ def test_targetid(self):
         brickidset = np.array(
             [int(bintargid[-lmostbit:-rmostbit], 2) for bintargid in bintargids])
         self.assertTrue(np.all(brickidset == bid))
-
-
-if __name__ == '__main__':
-    unittest.main()
-
-
-def test_suite():
-    """Allows testing of only this module with the command:
-
-        python setup.py test -m desitarget.test.test_brightmask
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)

py/desitarget/test/test_cmx.py

@@ -4,7 +4,7 @@
 """
 import unittest
 import sys
-from pkg_resources import resource_filename
+from importlib import resources
 import os.path
 from uuid import uuid4
 import numbers
@@ -26,10 +26,10 @@ class TestCMX(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.datadir = resource_filename('desitarget.test', 't')
+        cls.datadir = resources.files('desitarget').joinpath('test/t')
         cls.tractorfiles = sorted(io.list_tractorfiles(cls.datadir))
         cls.sweepfiles = sorted(io.list_sweepfiles(cls.datadir))
-        cls.cmxdir = resource_filename('desitarget.test', 't3')
+        cls.cmxdir = resources.files('desitarget').joinpath('test/t3')
 
         # ADM find which HEALPixels are covered by test sweeps files.
         cls.nside = 32
@@ -43,7 +43,7 @@ def setUpClass(cls):
 
         # ADM set up the GAIA_DIR environment variable.
         cls.gaiadir_orig = os.getenv("GAIA_DIR")
-        os.environ["GAIA_DIR"] = resource_filename('desitarget.test', 't4')
+        os.environ["GAIA_DIR"] = str(resources.files('desitarget').joinpath('test/t4'))
 
     @classmethod
     def tearDownClass(cls):
@@ -157,14 +157,3 @@ def test_parallel_select(self):
                 self.assertTrue('CMX_TARGET' in targets.dtype.names)
                 self.assertEqual(len(targets), np.count_nonzero(targets['CMX_TARGET']))
 
-
-if __name__ == '__main__':
-    unittest.main()
-
-
-def test_suite():
-    """Allows testing of only this module with the command:
-
-        python setup.py test -m desitarget.test.test_cmx
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)