Merge pull request #575 from kevin218/dev/kbs

Increment to v0.10

demos/JWST/S1_nirx_template.ecf

@@ -65,7 +65,7 @@ expand_mask         8
 ignore_low          600
 ignore_hi           None
 
-# Manual reference pixel correction for NIRSpec PRISM
+# Manual reference pixel correction for NIRSpec PRISM when not subtracting BG
 refpix_corr         False
 npix_top            8
 npix_bot            8

demos/JWST/S4_template.ecf

@@ -9,7 +9,7 @@ wave_min        1.5         # Minimum wavelength. Set to None to use the shortes
 wave_max        4.5         # Maximum wavelength. Set to None to use the longest extracted wavelength from Stage 3.
 allapers        False       # Run S4 on all of the apertures considered in S3? Otherwise will use newest output in the inputdir
 
-# Mannualy mask pixel columns by index number
+# Manually mask pixel columns by index number
 # mask_columns  []
 
 # Parameters for drift correction of 1D spectra

docs/media/S1_template.ecf

@@ -62,7 +62,7 @@ expand_mask         8
 ignore_low          600
 ignore_hi           None
 
-# Manual reference pixel correction for NIRSpec PRISM
+# Manual reference pixel correction for NIRSpec PRISM when not subtracting BG
 refpix_corr         False
 npix_top            8
 npix_bot            8

docs/media/S4_template.ecf

@@ -9,7 +9,7 @@ wave_min        1.5         # Minimum wavelength. Set to None to use the shortes
 wave_max        4.5         # Maximum wavelength. Set to None to use the longest extracted wavelength from Stage 3.
 allapers        False       # Run S4 on all of the apertures considered in S3? Otherwise will use newest output in the inputdir
 
-# Mannualy mask pixel columns by index number
+# Manualy mask pixel columns by index number
 # mask_columns  []
 
 # Parameters for drift correction of 1D spectra

docs/source/index.rst

@@ -11,7 +11,7 @@ Welcome to Eureka!'s documentation!
 ===================================
 
 ``Eureka!`` is a data reduction and analysis pipeline for exoplanet time-series observations, with a particular focus on JWST data.
-``Eureka!`` is capable of of reducing JWST time-series data (starting from raw, uncalibrated FITS files) and turning it into precise exoplanet transmission and emission spectra.
+``Eureka!`` is capable of reducing JWST time-series data (starting from raw, uncalibrated FITS files) and turning it into precise exoplanet transmission and emission spectra.
 The package is continually being improved, so be sure to check back often for new updates. The code is not officially associated with JWST or the ERS team.
 
 

docs/source/installation.rst

@@ -6,8 +6,8 @@ Installation methods
 --------------------
 
 In order to have consistent, repeatable results across the ``Eureka!`` user community, we recommend that all general users install
-the most recent stable release of ``Eureka!``, v0.9. The following installation instructions are written with this in mind,
-and the most recent stable release is also available as a zipped archive `here <https://github.com/kevin218/Eureka/releases/tag/v0.9>`_.
+the most recent stable release of ``Eureka!``, v0.10. The following installation instructions are written with this in mind,
+and the most recent stable release is also available as a zipped archive `here <https://github.com/kevin218/Eureka/releases/tag/v0.10>`_.
 Also note that if you are using a macOS device with an M1 processor, you will need to use the ``conda`` environment.yml file
 installation instructions below as the pip dependencies fail to build on the M1 processor.
 
@@ -31,7 +31,7 @@ Once in your new conda environment, you can install ``Eureka!`` directly from so
 
 .. code-block:: bash
 
-	git clone -b v0.9 https://github.com/kevin218/Eureka.git
+	git clone -b v0.10 https://github.com/kevin218/Eureka.git
 	cd Eureka
 	pip install -e '.[jwst]'
 
@@ -49,7 +49,7 @@ Once in your new conda environment, you can install the ``Eureka!`` package with
 
 .. code-block:: bash
 
-	pip install 'eureka[jwst]@git+https://github.com/kevin218/Eureka.git@v0.9'
+	pip install -e 'eureka[jwst]@git+https://github.com/kevin218/Eureka.git@v0.10'
 
 Other specific branches can be installed using:
 
@@ -89,7 +89,7 @@ To install using conda:
 
 .. code-block:: bash
 
-	git clone -b v0.9 https://github.com/kevin218/Eureka.git
+	git clone -b v0.10 https://github.com/kevin218/Eureka.git
 	cd Eureka
 	conda env create --file environment.yml --force
 	conda activate eureka

environment.yml

@@ -10,7 +10,6 @@ dependencies:
     - ccdproc
     - celerite # Needed for GP
     - corner
-    - cython
     - dynesty[version='>1.0'] # Lower limit needed for specific arguments
     - emcee[version='>3.0.0'] # Lower limit needed for specific arguments
     - flake8 # Needed for testing
@@ -40,7 +39,7 @@ dependencies:
         - crds
         - exotic-ld
         - image_registration@git+https://github.com/keflavich/image_registration@master # Need GitHub version to avoid np.float issue
-        - jwst==1.10.2
+        - jwst==1.11.4
         - myst-parser # Needed for documentation
         - setuptools_scm # Needed for version number
         - sphinx-rtd-theme # Needed for documentation

environment_pymc3.yml

@@ -11,7 +11,6 @@ dependencies:
     - ccdproc
     - celerite # Needed for GP
     - corner
-    - cython
     - dynesty[version='>1.0'] # Lower limit needed for specific arguments
     - emcee[version='>3.0.0'] # Lower limit needed for specific arguments
     - flake8 # Needed for testing
@@ -43,7 +42,7 @@ dependencies:
         - exoplanet
         - exotic-ld
         - image_registration@git+https://github.com/keflavich/image_registration@master # Need GitHub version to avoid np.float issue
-        - jwst==1.10.2
+        - jwst==1.11.4
         - myst-parser # Needed for documentation
         - opencv-python<4.8 # Upper limit needed for numpy<1.22
         - opencv-python-headless<4.8 # Upper limit needed for numpy<1.22

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "cython", "setuptools_scm[toml]>=6.2"]
+requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

setup.cfg

@@ -2,11 +2,11 @@
 name = eureka
 author = Eureka! pipeline developers
 author_email = [email protected]
-description = A data reduction and analysis pipeline intended for time-series observations with the JWST.
+description = A data reduction and analysis pipeline intended for time-series observations with JWST.
 long_description = file: README.md
 long_description_content_type = text/markdown
 license = MIT License
-license_file = LICENSE
+license_files = [LICENSE,]
 url = https://github.com/kevin218/Eureka
 project_urls =
     Bug Tracker = https://github.com/kevin218/Eureka/issues
@@ -23,8 +23,6 @@ package_dir =
 packages = find:
 zip_safe = True
 python_requires= >=3.8
-setup_requires =
-    cython
 install_requires =
     astraeus@git+https://github.com/kevin218/Astraeus@main
     astropy
@@ -35,7 +33,6 @@ install_requires =
     celerite # Needed for GP
     corner
     crds
-    cython
     dynesty>1.0 # Lower limit needed for specific arguments
     emcee>3.0.0 # Lower limit needed for specific arguments
     exotic-ld
@@ -60,7 +57,7 @@ where = src
 
 [options.extras_require]
 jwst =
-    jwst==1.10.2
+    jwst==1.11.4
     stcal>=1.0.0 # Lower limit needed for our create_integration_model function
 hst =
     image_registration@git+https://github.com/keflavich/image_registration@master # Need GitHub version to avoid np.float issue
@@ -87,6 +84,7 @@ pymc3 =
     opencv-python<4.8 # Upper limit needed for numpy<1.22
     opencv-python-headless<4.8 # Upper limit needed for numpy<1.22
     scipy>=1.4.0,<1.8.0 # Lower limit needed for scipy.fft, upper limit needed for theano, starry, and pymc3
+    xarray<2023.10.0 # Upper limit needed for numpy<1.22
 jupyter =
     jupyter
     ipykernel

setup.py

@@ -1,10 +1,6 @@
 #!/usr/bin/env python
 from setuptools import setup
-from Cython.Build import cythonize
 
 if __name__ == "__main__":
 
-    setup(
-        ext_modules=cythonize(
-            ["src/eureka/S3_data_reduction/niriss_cython.pyx"]),
-    )
+    setup()

src/eureka/S1_detector_processing/group_level.py

@@ -195,7 +195,7 @@ def custom_ref_pixel(input_model, log, meta):
             intgrp_data[dqmask] = np.nan
 
             if hasattr(meta, 'masktrace') and meta.masktrace:
-                intgrp_data *= meta.trace_mask
+                intgrp_data *= input_model.trace_mask
 
             odd_med = np.nanmedian(intgrp_data[odd_row_ref, :])
             evn_med = np.nanmedian(intgrp_data[evn_row_ref, :])

src/eureka/S1_detector_processing/ramp_fitting.py

@@ -371,7 +371,7 @@ def calc_opt_sums_uniform_weight(rn_sect, gain_sect, data_masked, mask_2d,
 
     # Return 'empty' sums if there is no more data to fit
     if data_masked.size == 0:
-        return np.array([]), np.array([]), np.array([]), np.array([]),\
+        return np.array([]), np.array([]), np.array([]), np.array([]), \
             np.array([]), np.array([])
 
     # get initial group for each good pixel for this semiramp

src/eureka/S3_data_reduction/__init__.py

@@ -5,6 +5,7 @@
 from . import miri
 from . import nircam
 from . import niriss_profiles
+from . import niriss_python
 from . import niriss
 from . import nirspec
 from . import optspex

src/eureka/S3_data_reduction/background.py

@@ -205,9 +205,9 @@ def fitbg(dataim, meta, mask, x1, x2, deg=1, threshold=5, isrotate=False,
 
     # Convert x1 and x2 to array, if need be
     ny, nx = np.shape(dataim)
-    if type(x1) == int or type(x1) == np.int64:
+    if isinstance(x1, (int, np.int64)):
         x1 = np.zeros(ny, dtype=int)+x1
-    if type(x2) == int or type(x2) == np.int64:
+    if isinstance(x2, (int, np.int64)):
         x2 = np.zeros(ny, dtype=int)+x2
 
     if deg < 0:

src/eureka/S3_data_reduction/niriss.py

@@ -21,11 +21,7 @@
 
 from .background import fitbg3
 from .niriss_profiles import *
-
-# some cute cython code
-import pyximport
-pyximport.install()
-from . import niriss_cython
+from . import niriss_python
 
 
 __all__ = ['read', 'simplify_niriss_img', 'image_filtering',
@@ -606,7 +602,7 @@ def construct_guesses(A, B, sig, length=10):
     combos = construct_guesses([0.1,30], [0.1,30], [1,40])
 
     # generates length x length x length number of images and fits to the data
-    img1, sigout1 = niriss_cython.build_image_models(data.median,
+    img1, sigout1 = niriss_python.build_image_models(data.median,
                                                      combos[:,0], combos[:,1], 
                                                      combos[:,2], 
                                                      pos1, pos2)
@@ -619,15 +615,15 @@ def construct_guesses(A, B, sig, length=10):
 
     # generates length x length x length number of images centered around the previous
     #   guess to optimize the image fit
-    img2, sigout2 = niriss_cython.build_image_models(data.median, 
+    img2, sigout2 = niriss_python.build_image_models(data.median, 
                                                      combos[:,0], combos[:,1],
                                                      combos[:,2],
                                                      pos1, pos2)
 
     # creates a 2D image for the first and second orders with the best-fit gaussian
     #    profiles
     final_guess = combos[np.argmin(sigout2)]
-    ord1, ord2, _ = niriss_cython.build_image_models(data.median,
+    ord1, ord2, _ = niriss_python.build_image_models(data.median,
                                                      [final_guess[0]],
                                                      [final_guess[1]],
                                                      [final_guess[2]],
@@ -661,7 +657,7 @@ def residuals(params, data, y1_pos, y2_pos):
         """ Calcualtes residuals for best-fit profile. """
         A, B, sig1 = params
         # Produce the model:   
-        model,_ = niriss_cython.build_image_models(data, [A], [B], [sig1], y1_pos, y2_pos)
+        model,_ = niriss_python.build_image_models(data, [A], [B], [sig1], y1_pos, y2_pos)
         # Calculate residuals:     
         res = (model[0] - data)
         return res.flatten()
@@ -676,7 +672,7 @@ def residuals(params, data, y1_pos, y2_pos):
                                )
 
     # creates the final mask
-    out_img1,out_img2,_= niriss_cython.build_image_models(data.median, 
+    out_img1,out_img2,_= niriss_python.build_image_models(data.median, 
                                                           results.x[0:1], 
                                                           results.x[1:2], 
                                                           results.x[2:3], 

src/eureka/S3_data_reduction/niriss_python.py

@@ -0,0 +1,26 @@
+import numpy as np
+
+def build_image_models(data, A, B, sig, mu1, mu2, return_together=True):
+
+    y = np.transpose(np.full((len(A), data.shape[-1], data.shape[0]),
+                             np.arange(0,data.shape[0],1)), axes=(0,2,1))
+
+    sig = np.full(y.T.shape, sig).T
+
+    A = np.full(y.T.shape, A).T
+    exp = np.exp(-(y-mu1)**2/(2.0*sig**2))
+    gauss = 1.0/(2.0*np.pi*np.sqrt(sig))*exp
+    f1x = mu1*gauss*A
+
+    B = np.full(y.T.shape, B).T
+    exp = np.exp(-(y-mu2)**2/(2.0*sig**2))
+    gauss = 1.0/(2.0*np.pi*np.sqrt(sig))*exp
+    f2x = mu2*gauss*B
+
+    model = f1x+f2x
+    sigma = np.nansum(np.sqrt((model-data)**2.0), axis=(1,2))
+
+    if return_together:
+       return model, sigma
+    else:
+       return f1x, f2x, sigma

src/eureka/S3_data_reduction/plots_s3.py

@@ -533,7 +533,7 @@ def residualBackground(data, meta, m, vmin=-200, vmax=1000):
     cmap = plt.cm.plasma.copy()
     cmap.set_bad('k', 1.)
 
-    plt.figure(3304)
+    plt.figure(3304, figsize=(8, 4))
     plt.clf()
     fig, (a0, a1) = plt.subplots(1, 2, gridspec_kw={'width_ratios': [3, 1]},
                                  num=3304, figsize=(8, 3.5))

src/eureka/S4_generate_lightcurves/generate_LD.py

@@ -64,16 +64,17 @@ def exotic_ld(meta, spec, log, white=False):
 
     # Compute wavelength ranges
     if white:
-        wavelength_range = np.array([meta.wave_min, meta.wave_max])
+        wavelength_range = np.array([meta.wave_min, meta.wave_max],
+                                    dtype=float)
         wavelength_range = np.repeat(wavelength_range[np.newaxis],
                                      meta.nspecchan, axis=0)
     else:
-        wsdata = np.array(meta.wave_low)
+        wsdata = np.array(meta.wave_low, dtype=float)
         wsdata = np.append(wsdata, meta.wave_hi[-1])
         wavelength_range = []
         for i in range(meta.nspecchan):
             wavelength_range.append([wsdata[i], wsdata[i+1]])
-        wavelength_range = np.array(wavelength_range)
+        wavelength_range = np.array(wavelength_range, dtype=float)
 
     # wavelength range needs to be in Angstrom
     if spec.wave_1d.attrs['wave_units'] == 'microns':

src/eureka/S4_generate_lightcurves/s4_genLC.py

@@ -107,6 +107,8 @@ def genlc(eventlabel, ecf_path=None, s3_meta=None, input_meta=None):
 
     # Create directories for Stage 5 outputs
     meta.run_s4 = None
+    if not hasattr(meta, 'expand'):
+        meta.expand = 1
     for spec_hw_val in meta.spec_hw_range:
         for bg_hw_val in meta.bg_hw_range:
             if not isinstance(bg_hw_val, str):

src/eureka/S5_lightcurve_fitting/differentiable_models/PyMC3Models.py

@@ -298,6 +298,11 @@ def __init__(self, components, **kwargs):
             # Add the PyMC3 model to each component
             component.model = self.model
 
+        self.GP = False
+        for component in self.components:
+            if component.modeltype == 'GP':
+                self.GP = True
+
         # Setup PyMC3 model parameters
         with self.model: