Merge pull request pypeit#1768 from pypeit/gemini_wvcalib

Gemini wvcalib

doc/help/run_pypeit.rst

@@ -4,7 +4,7 @@
     usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c]
                       pypeit_file
     
-    ##  [1;37;42mPypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev729+g948bfc229.d20240202[0m
+    ##  [1;37;42mPypeIt : The Python Spectroscopic Data Reduction Pipeline v1.15.1.dev0+g7dc53070b.d20240208[0m
     ##  
     ##  Available spectrographs include:
     ##   bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v,

doc/include/dev_suite_readme.rst

@@ -419,6 +419,21 @@ To monitor a test in Nautilus as it is running, the logs can be tailed:
 
 Additionally they can be monitored with the `Nautilus Grafana page <https://grafana.nrp-nautilus.io/?orgId=1>`__.
 
+By default ``gen_kube_devsuite`` creates a job using a default container with PypeIt 
+pre-installed. It also supports running with different python versions by
+selecting a different container. For example:
+
+.. code-block:: console
+
+    $ ./gen_kube_devsuite devsuite-python3.11-job devsuite-python3.11-job.yml --container python3.11
+
+Any of the standard python images in docker hub can be used. To use a different container the full 
+download path must be given. For example:
+
+.. code-block:: console
+
+    $ ./gen_kube_devsuite devsuite-ubuntu-job devsuite-ubuntu-job.yml --container docker.io/library/ubuntu:22.04
+
 
 Additional Options
 ------------------

doc/releases/1.15.1dev.rst

@@ -0,0 +1,31 @@
+
+Version 1.15.1dev
+=================
+
+Installation Changes
+--------------------
+
+Dependency Changes
+------------------
+
+Functionality/Performance Improvements and Additions
+----------------------------------------------------
+
+Instrument-specific Updates
+---------------------------
+
+- Added example of improved wavelength solution for GEMINI/GMOS
+
+Script Changes
+--------------
+
+Datamodel Changes
+-----------------
+
+Under-the-hood Improvements
+---------------------------
+
+Bug Fixes
+---------
+
+

doc/spectrographs/gemini_gmos.rst

@@ -165,3 +165,35 @@ The two files provided must be located either:
  (2) the current working data, and/or
  (3) be named with the full path.
 
+Wavelength calibration
+----------------------
+
+Wavelength calibration for GMOS multi-object data is non trivial due to the
+wavelength solution changing non-linearly as a function of the location of the slit
+on the detector. To
+mitigate this, one can manually generate wavelength archives using
+:ref:`pypeit_identify`. However, this procedure would have to be repeated for each
+setup and sometimes for multiple slits per setup. To reduce tis tedium, we recommend using the ``reidentify`` method in the ``wavelengths`` section of the :ref:`pypeit_file`. This is possible through a compilation of all currently available wavelength solutions tabulated in it.
+
+The fits table should contain a single ``BinaryTable`` with the following columns:
+(1) wave: Each entry is a float array with wavelength in angstroms from the user generated wvarxiv.
+(2) flux: Each entry is an float array with the flux value from the user generated wvarxiv.
+(3) order: Each entry is 0 (int). See the ``pypeit/data/arc_lines/reid_arxiv/gemini_gmos_south_ham_b600_compiled.fits`` file for an example within the PypeIt installation.
+
+If one has a set of wvarxiv solutions from :ref:`pypeit_identify`, one can use the ``pypeit_compile_wvarxiv`` script to compile the fits file as follows:
+
+.. code-block:: bash
+
+    pypeit_compile_wvarxiv <path_to_wvarxiv_files> <instrument> <grating>
+
+Use the ``-h`` flag to see more details regarding usage. This script produces a fits file in the ````pypeit/data/arc_lines/reid_arxiv/`` folder.
+
+To use reidentify, add the following user-level parameters to the :ref:`pypeit_file`:
+
+.. code-block:: ini
+
+    [calibrations]
+        [[wavelengths]]
+            reid_arxiv = gemini_gmos_south_ham_b600_compiled.fits
+
+Currently, this method is only supported for the B600 grating on GMOS-S. If you have MOS data with a different grating, please consider compiling your wvarxiv solutions as described above to expand this feature for other users. Please submit a pull request (or contact the PypeIt team) to the PypeIt repository with the fits file.

doc/whatsnew.rst

@@ -11,6 +11,10 @@ What's New in PypeIt
 
 ----
 
+.. include:: releases/1.15.1dev.rst
+
+----
+
 .. include:: releases/1.15.0.rst
 
 ----

pypeit/scripts/__init__.py

@@ -47,6 +47,7 @@
 from pypeit.scripts import tellfit
 from pypeit.scripts import trace_edges
 from pypeit.scripts import view_fits
+from pypeit.scripts import compile_wvarxiv
 
 
 # Build the list of script classes

pypeit/scripts/compile_wvarxiv.py

@@ -0,0 +1,95 @@
+"""
+This script enables the user to convert a MasterWaveCalib wavelength solution fits file
+into a PypeIt arxiv solution that can be used with the full_template method.
+
+.. include common links, assuming primary doc root is up one directory
+.. include:: ../include/links.rst
+"""
+import time
+from pypeit import msgs
+from pypeit import par
+from pypeit import inputfiles
+from pypeit import utils
+from pypeit.scripts import scriptbase
+
+class CompileWVarxiv(scriptbase.ScriptBase):
+    """
+    A class for compiling a set of wxarxiv solutions from Identify into a single fits file.
+
+    Args:
+        wvarxiv_folder (str): Location of the WVarxiv files.
+        instrument (str): Name of the instrument (e.g., keck_lris_blue, keck_deimos, gemini_gmos_s_ham).
+        grating (str): Instrument grating name (e.g., B600, R400, 600_10000).
+        append (bool, optional): Append to an existing file for this instrument. Defaults to False.
+
+    Methods:
+        get_parser(width=None):
+            Returns an ArgumentParser object with the required command line arguments.
+        
+        main(args):
+            Main method for compiling the wxarxiv solutions into a single fits file.
+
+    Example usage:
+        parser = WvarxivCompile.get_parser()
+        args = parser.parse_args()
+        WvarxivCompile.main(args)
+    """    
+
+    @classmethod
+    def get_parser(cls, width=None):
+        parser = super().get_parser(description='Read in a set of wxarxiv solutions from Identify and compile them into a '
+                                                'single fits file to be used with the reidentify method.', width=width)
+        parser.add_argument('wvarxiv_folder', type = str, default=None, help='Location of the WVarxiv files')
+        parser.add_argument('instrument', type = str, default=None, help='Name of instrument. e.g. keck_lris_blue, keck_deimos, gemini_gmos_south_ham')
+        parser.add_argument('grating', type=str, help="Instrument grating name. E.g. b600, r400, 600_10000.")
+        parser.add_argument('--append', default=False, action='store_true', help='Append to an existing file for this instrument.')
+
+        return parser
+
+    @staticmethod
+    def main(args):
+        from astropy.table import Table, join
+        from importlib_resources import files as imres_files
+        import glob, os
+
+        # Read in the wvarxiv files
+        assert os.path.isdir(args.wvarxiv_folder), 'The wvarxiv_folder does not exist'
+        wavarxiv_files = glob.glob(args.wvarxiv_folder + '/*.fits')
+
+        assert len(wavarxiv_files) > 0, 'No wvarxiv fits files found in the folder.'
+
+        # Generate the fits file
+        array_len = len(Table.read(wavarxiv_files[0])['wave'].data)
+
+        reid_table = Table(names=('wave', 'flux', 'order'), dtype=(f'({array_len},)>f8', f'({array_len},)>f8', '>i8'))
+
+        # Loop over all wvarxiv files and merge them into a single table
+        for fitsfile in wavarxiv_files:
+            tab = Table.read(fitsfile)
+            # Make array length errors clear to the user.
+            assert len(tab['wave'].data) == array_len, 'The wvarxiv arrays are not the same length. Check the fits files and ensure these correspond to the same grating and instrument.'
+
+            # Add as a row to the table
+            reid_table.add_row([tab['wave'].data, tab['flux'].data, 0])
+
+        # Write to file
+        out_path = imres_files('pypeit').joinpath('data', 'arc_lines', 'reid_arxiv').joinpath(f'{args.instrument}_{args.grating}_compiled.fits')
+
+        # Does a file already exist?
+        if out_path.exists() and not args.append:
+            msgs.error(f'File {out_path} already exists. Use --append to overwrite the file and add your new solutions to the existing ones.')
+        # What if user asks to append solutions?
+        elif out_path.exists() and args.append:
+            old_table = Table.read(out_path)
+            old_array_len = len(old_table['wave'][0].data)
+            if old_array_len != array_len:
+                msgs.error(f'The old file has an array length of {old_array_len} while the new files have an array length of {array_len}. Cannot merge these files.')
+            else:
+                reid_table = join(old_table, reid_table)
+                reid_table.write(out_path, format='fits', overwrite=args.append)
+                msgs.info(f'Wrote the compiled wvarxiv file to {out_path}.')
+
+        # If the file does not exist, just write it out
+        else:
+            reid_table.write(out_path, format='fits')
+            msgs.info(f'Wrote the compiled wvarxiv file to {out_path}.')

pypeit/spectrographs/gemini_gmos.py

@@ -941,7 +941,8 @@ def config_specific_par(self, scifile, inp_par=None):
         if self.get_meta_value(scifile, 'dispname')[0:4] == 'R400':
             par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_r400_ham.fits'
         elif self.get_meta_value(scifile, 'dispname')[0:4] == 'B600':
-            par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_b600_ham.fits'
+            par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_south_ham_b600_compiled.fits'
+            par['calibrations']['wavelengths']['method'] = 'reidentify'
 
         # The bad amp needs a larger follow_span for slit edge tracing
         obs_epoch = self.get_meta_value(scifile, 'mjd')

setup.cfg

@@ -133,6 +133,7 @@ console_scripts =
     pypeit_show_wvcalib = pypeit.scripts.show_wvcalib:ShowWvCalib.entry_point
     pypeit_tellfit = pypeit.scripts.tellfit:TellFit.entry_point
     pypeit_trace_edges = pypeit.scripts.trace_edges:TraceEdges.entry_point
+    pypeit_compile_wvarxiv = pypeit.scripts.compile_wvarxiv:CompileWVarxiv.entry_point
 
     # GUI scripts
     # Ideally want to be able to do something like this for the following scripts: