From cb247490a8f59807951375220a1416b71c7e970d Mon Sep 17 00:00:00 2001 From: Brett Graham Date: Mon, 27 Jan 2025 16:39:12 -0500 Subject: [PATCH 1/2] Fix minor issues with documentation and unused arguments, etc (#1593) --- 1542.mosaic_pipeline.rst | 1 - changes/1593.exposure_pipeline.rst | 1 + changes/1593.resample.rst | 1 + changes/1593.saturation.rst | 1 + docs/roman/package_index.rst | 1 + docs/roman/pipeline/exposure_pipeline.rst | 23 +++++++++---------- .../references_general/references_general.rst | 4 ++-- docs/roman/resample/arguments.rst | 18 --------------- docs/roman/stpipe/config_asdf.rst | 1 - docs/roman/stpipe/user_pipeline.rst | 1 - romancal/pipeline/exposure_pipeline.py | 1 - romancal/resample/resample.py | 22 ------------------ romancal/resample/resample_step.py | 5 ---- romancal/resample/tests/test_resample.py | 8 ------- 14 files changed, 17 insertions(+), 71 deletions(-) delete mode 100644 1542.mosaic_pipeline.rst create mode 100644 changes/1593.exposure_pipeline.rst create mode 100644 changes/1593.resample.rst create mode 100644 changes/1593.saturation.rst diff --git a/1542.mosaic_pipeline.rst b/1542.mosaic_pipeline.rst deleted file mode 100644 index 2fc68647d..000000000 --- a/1542.mosaic_pipeline.rst +++ /dev/null @@ -1 +0,0 @@ -Change the default suffix for mosaic products from _i2d to _coadd diff --git a/changes/1593.exposure_pipeline.rst b/changes/1593.exposure_pipeline.rst new file mode 100644 index 000000000..4cd0f7a0f --- /dev/null +++ b/changes/1593.exposure_pipeline.rst @@ -0,0 +1 @@ +Fix description of arguments in docs and add description of fully saturated input processing. diff --git a/changes/1593.resample.rst b/changes/1593.resample.rst new file mode 100644 index 000000000..4aed162eb --- /dev/null +++ b/changes/1593.resample.rst @@ -0,0 +1 @@ +Remove unused arguments from step specification. diff --git a/changes/1593.saturation.rst b/changes/1593.saturation.rst new file mode 100644 index 000000000..8ff7766a6 --- /dev/null +++ b/changes/1593.saturation.rst @@ -0,0 +1 @@ +Add saturation step docs to package index. diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst index 34a31b3fd..fc849a1d7 100644 --- a/docs/roman/package_index.rst +++ b/docs/roman/package_index.rst @@ -14,6 +14,7 @@ references_general/index.rst refpix/index.rst resample/index.rst + saturation/index.rst skymatch/index.rst source_catalog/index.rst stpipe/index.rst diff --git a/docs/roman/pipeline/exposure_pipeline.rst b/docs/roman/pipeline/exposure_pipeline.rst index 30f81d863..a48c26774 100644 --- a/docs/roman/pipeline/exposure_pipeline.rst +++ b/docs/roman/pipeline/exposure_pipeline.rst @@ -39,18 +39,7 @@ table below. Arguments --------- -The ``exposure`` pipeline has an optional argument:: - - --use_ramp_jump_detection boolean default=True - -When set to ``True``, the pipeline will perform jump detection as a part of the ramp -fitting step. The data at this stage of the pipeline are still in the form of the original -3D ramps ( ngroups x ncols x nrows ) and have had all of the detector-level -correction steps applied to it, up to but not including the detection and flagging of -Cosmic-Ray (CR) hits within each ramp (integration). For this case the jump detection -module in :ref:`ramp_fitting ` will update the dq array with the CR hits (jumps) that -are identified in the step. - +The ``exposure`` pipeline has no arguments Inputs ------ @@ -78,6 +67,16 @@ extensions. When such a file is loaded into the pipeline, it is immediately converted into a `~romancal.datamodels.RampModel`, and has all additional data arrays for errors and Data Quality flags created and initialized. +When the ``ExposurePipeline`` processes a fully saturated input (all pixels flagged as saturated). +The corresponding output image will: + +- contain all 0 data arrays +- contain all 0 variance arrays +- not be processed by steps beyond saturation + +A single fully saturated input will also cause :ref:`tweakreg ` to be skipped +for all input images. + Outputs ------- diff --git a/docs/roman/references_general/references_general.rst b/docs/roman/references_general/references_general.rst index de4261a53..746b5ffdf 100644 --- a/docs/roman/references_general/references_general.rst +++ b/docs/roman/references_general/references_general.rst @@ -51,8 +51,6 @@ documentation on each reference file. +---------------------------------------------+--------------------------------------------------+ | :ref:`flatfield ` | :ref:`FLAT ` | +---------------------------------------------+--------------------------------------------------+ -| :ref:`GAIN ` | :ref:`ramp_fitting ` | -+---------------------------------------------+--------------------------------------------------+ | :ref:`linearity ` | :ref:`LINEARITY ` | +---------------------------------------------+--------------------------------------------------+ | :ref:`photom ` | :ref:`PHOTOM ` | @@ -74,6 +72,8 @@ documentation on each reference file. +--------------------------------------------------+---------------------------------------------+ | :ref:`FLAT ` | :ref:`flatfield ` | +--------------------------------------------------+---------------------------------------------+ +| :ref:`GAIN ` | :ref:`ramp_fitting ` | ++--------------------------------------------------+---------------------------------------------+ | :ref:`LINEARITY ` | :ref:`linearity ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`MASK ` | :ref:`dq_init ` | diff --git a/docs/roman/resample/arguments.rst b/docs/roman/resample/arguments.rst index b72bb630a..4c0e0a759 100644 --- a/docs/roman/resample/arguments.rst +++ b/docs/roman/resample/arguments.rst @@ -82,24 +82,6 @@ image. If `weight_type=exptime`, the scaling value will be set equal to the exposure time found in the image header. -``--single`` (bool, default=False) - If set to `True`, resample each input image into a separate output. If - `False` (the default), each input is resampled additively (with weights) to - a common output - -``--blendheaders`` (bool, default=True) - Blend metadata from all input images into the resampled output image. - -``--allowed_memory`` (float, default=None) - Specifies the fractional amount of free memory to allow when creating the - resampled image. If ``None``, the environment variable - ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no check is made. If the - resampled image would be larger than specified, an ``OutputTooLargeError`` - exception will be generated. - - For example, if set to ``0.5``, only resampled images that use less than - half the available memory can be created. - ``--in_memory`` (bool, default=True) If set to `False`, write output datamodel to disk. diff --git a/docs/roman/stpipe/config_asdf.rst b/docs/roman/stpipe/config_asdf.rst index 923d517b9..b6fb79c1f 100644 --- a/docs/roman/stpipe/config_asdf.rst +++ b/docs/roman/stpipe/config_asdf.rst @@ -70,7 +70,6 @@ entries for clarity: output_use_model: false post_hooks: [] pre_hooks: [] - save_calibrated_ramp: false save_results: true search_output_file: true skip: false diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index 5c7dde510..619ae15d7 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -63,7 +63,6 @@ class: output_use_model: false post_hooks: [] pre_hooks: [] - save_calibrated_ramp: false save_results: true search_output_file: true skip: false diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 11b19351f..d71460ca2 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -46,7 +46,6 @@ class ExposurePipeline(RomanPipeline): class_alias = "roman_elp" spec = """ - save_calibrated_ramp = boolean(default=False) save_results = boolean(default=False) suffix = string(default="cal") """ diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 0e9ad543c..e320c6104 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -48,7 +48,6 @@ def __init__( input_models, output=None, single=False, - blendheaders=True, pixfrac=1.0, kernel="square", fillval="INDEF", @@ -90,7 +89,6 @@ def __init__( self.output_filename = output self.pscale_ratio = pscale_ratio self.single = single - self.blendheaders = blendheaders self.pixfrac = pixfrac self.kernel = kernel self.fillval = fillval @@ -143,23 +141,6 @@ def __init__( log.debug(f"Output mosaic size: {self.output_wcs.array_shape}") - # NOTE: should we enable memory allocation? - - # can_allocate, required_memory = datamodels.util.check_memory_allocation( - # self.output_wcs.array_shape, - # kwargs['allowed_memory'], - # datamodels.ImageModel - # ) - # if not can_allocate: - # raise OutputTooLargeError( - # f'Combined ImageModel size {self.output_wcs.array_shape} ' - # f'requires {bytes2human(required_memory)}. ' - # f'Model cannot be instantiated.' - # ) - - # NOTE: wait for William to fix bug in datamodels' init and then - # use datamodels.ImageModel(shape=(nx, ny)) instead of mk_datamodel() - # n_images sets the number of context image planes. # This should be 1 to start (not the default of 2). self.blank_output = maker_utils.mk_datamodel( @@ -341,9 +322,6 @@ def resample_many_to_one(self): ) is not None: output_model.meta.asn.table_name = asn_table_name - if self.blendheaders: - log.info("Skipping blendheaders for now.") - # Initialize the output with the wcs driz = gwcs_drizzle.GWCSDrizzle( output_model, diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 8796a52ff..32c418b06 100644 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -66,9 +66,6 @@ class ResampleStep(RomanStep): pixel_scale_ratio = float(default=1.0) # Ratio of input to output pixel scale pixel_scale = float(default=None) # Absolute pixel scale in arcsec output_wcs = string(default='') # Custom output WCS. - single = boolean(default=False) - blendheaders = boolean(default=True) - allowed_memory = float(default=None) # Fraction of memory to use for the combined image. in_memory = boolean(default=True) good_bits = string(default='~DO_NOT_USE+NON_SCIENCE') # The good bits to use for building the resampling mask. """ @@ -80,7 +77,6 @@ def process(self, input): input_models = ModelLibrary([input]) # set output filename from meta.filename found in the first datamodel output = input.meta.filename - self.blendheaders = False elif isinstance(input, str): # either a single asdf filename or an association filename try: @@ -118,7 +114,6 @@ def process(self, input): self.wht_type = self.weight_type self.log.info("Setting drizzle's default parameters...") kwargs = self.set_drizzle_defaults() - kwargs["allowed_memory"] = self.allowed_memory # Issue a warning about the use of exptime weighting if self.wht_type == "exptime": diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 2923a2625..cd6b603b8 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -290,8 +290,6 @@ def test_resampledata_init(exposure_1): """Test that ResampleData can set initial values.""" input_models = ModelLibrary(exposure_1) output = "output.asdf" - single = False - blendheaders = False pixfrac = 0.8 kernel = "turbo" fillval = 0.0 @@ -304,8 +302,6 @@ def test_resampledata_init(exposure_1): resample_data = ResampleData( input_models, output=output, - single=single, - blendheaders=blendheaders, pixfrac=pixfrac, kernel=kernel, fillval=fillval, @@ -320,8 +316,6 @@ def test_resampledata_init(exposure_1): assert resample_data.input_models == input_models assert resample_data.output_filename == output assert resample_data.pscale_ratio == pscale_ratio - assert resample_data.single == single - assert resample_data.blendheaders == blendheaders assert resample_data.pixfrac == pixfrac assert resample_data.kernel == kernel assert resample_data.fillval == fillval @@ -341,8 +335,6 @@ def test_resampledata_init_default(exposure_1): assert resample_data.input_models == input_models assert resample_data.output_filename is None assert resample_data.pscale_ratio == 1.0 - assert not resample_data.single - assert resample_data.blendheaders assert resample_data.pixfrac == 1.0 assert resample_data.kernel == "square" assert resample_data.fillval == "INDEF" From 8422362dc33e0d4b3dc6702274b07c740acda37c Mon Sep 17 00:00:00 2001 From: Jonathan Eisenhamer Date: Tue, 28 Jan 2025 13:41:18 -0500 Subject: [PATCH 2/2] RCAL-930 Roundtrip L3 wcsinfo especially when skycell specifications are used (#1585) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- changes/1585.mosaic_pipeline.rst | 1 + romancal/pipeline/mosaic_pipeline.py | 103 +++++++++++--- .../pipeline/tests/test_mosaic_pipeline.py | 133 ++++++++++++++++++ romancal/regtest/test_mos_pipeline.py | 10 ++ romancal/regtest/test_mos_skycell_pipeline.py | 10 ++ romancal/regtest/util.py | 33 +++++ romancal/resample/resample.py | 21 ++- romancal/resample/tests/test_resample.py | 4 +- 8 files changed, 285 insertions(+), 30 deletions(-) create mode 100644 changes/1585.mosaic_pipeline.rst create mode 100644 romancal/pipeline/tests/test_mosaic_pipeline.py create mode 100644 romancal/regtest/util.py diff --git a/changes/1585.mosaic_pipeline.rst b/changes/1585.mosaic_pipeline.rst new file mode 100644 index 000000000..977888ff6 --- /dev/null +++ b/changes/1585.mosaic_pipeline.rst @@ -0,0 +1 @@ +Roundtrip L3 wcsinfo especially when skycell specifications are used diff --git a/romancal/pipeline/mosaic_pipeline.py b/romancal/pipeline/mosaic_pipeline.py index 7294ef3dd..fb5f2315f 100644 --- a/romancal/pipeline/mosaic_pipeline.py +++ b/romancal/pipeline/mosaic_pipeline.py @@ -13,6 +13,7 @@ from astropy import units as u from astropy.modeling import models from gwcs import WCS, coordinate_frames +from stcal.alignment import util as wcs_util import romancal.datamodels.filetype as filetype from romancal.datamodels import ModelLibrary @@ -114,13 +115,13 @@ def process(self, input): # check to see if there exists a skycell on disk if not create it if not isfile(skycell_file_name): - # extract the wcs info from the record for generate_tan_wcs + # extract the wcs info from the record for skycell_to_wcs log.info( "Creating skycell image at ra: %f dec %f", float(skycell_record["ra_center"]), float(skycell_record["dec_center"]), ) - skycell_wcs = generate_tan_wcs(skycell_record) + skycell_wcs = skycell_to_wcs(skycell_record) # skycell_wcs.bounding_box = bounding_box # For resample to use an external grid we need to pass it the skycell gwcs object @@ -128,6 +129,7 @@ def process(self, input): wcs_tree = {"wcs": skycell_wcs} wcs_file = asdf.AsdfFile(wcs_tree) wcs_file.write_to("skycell_wcs.asdf") + self.resample.output_wcs = "skycell_wcs.asdf" self.resample.output_shape = ( int(skycell_record["nx"]), @@ -162,41 +164,96 @@ def process(self, input): return result -def generate_tan_wcs(skycell_record): - """extract the wcs info from the record for generate_tan_wcs - we need the scale, ra, dec, bounding_box""" +def skycell_to_wcs(skycell_record): + """From a skycell record, generate a GWCS + + Parameters + ---------- + skycell_record : dict + A skycell record, or row, from the skycell patches table. + + Returns + ------- + wcsobj : wcs.GWCS + The GWCS object from the skycell record. + """ + wcsinfo = dict() + + # The scale is given in arcseconds per pixel. Convert to degrees. + wcsinfo["pixel_scale"] = float(skycell_record["pixel_scale"]) / 3600.0 + + # Remaining components of the wcsinfo block + wcsinfo["ra_ref"] = float(skycell_record["ra_projection_center"]) + wcsinfo["dec_ref"] = float(skycell_record["dec_projection_center"]) + wcsinfo["x_ref"] = float(skycell_record["x0_projection"]) + wcsinfo["y_ref"] = float(skycell_record["y0_projection"]) + wcsinfo["orientat"] = float(skycell_record["orientat_projection_center"]) + wcsinfo["rotation_matrix"] = None - scale = float(skycell_record["pixel_scale"]) - ra_center = float(skycell_record["ra_projection_center"]) - dec_center = float(skycell_record["dec_projection_center"]) - shiftx = float(skycell_record["x0_projection"]) - shifty = float(skycell_record["y0_projection"]) + # Bounding box of the skycell. Note that the center of the pixels are at (0.5, 0.5) bounding_box = ( (-0.5, -0.5 + skycell_record["nx"]), (-0.5, -0.5 + skycell_record["ny"]), ) - # components of the model - # shift = models.Shift(shiftx) & models.Shift(shifty) + wcsobj = wcsinfo_to_wcs(wcsinfo, bounding_box=bounding_box) + return wcsobj + + +def wcsinfo_to_wcs(wcsinfo, bounding_box=None, name="wcsinfo"): + """Create a GWCS from the L3 wcsinfo meta + + Parameters + ---------- + wcsinfo : dict or MosaicModel.meta.wcsinfo + The L3 wcsinfo to create a GWCS from. - # select a scale for the skycell image, this will come from INS and may - # be optimized for the different survey programs - scale_x = scale - scale_y = scale - # set the pixelsscale to 0.1 arcsec/pixel - pixelscale = models.Scale(scale_x / 3600.0) & models.Scale(scale_y / 3600.0) + bounding_box : None or 4-tuple + The bounding box in detector/pixel space. Form of input is: + (x_left, x_right, y_bottom, y_top) - pixelshift = models.Shift(-1.0 * shiftx) & models.Shift(-1.0 * shifty) + name : str + Value of the `name` attribute of the GWCS object. + + Returns + ------- + wcs : wcs.GWCS + The GWCS object created. + """ + pixelshift = models.Shift(-wcsinfo["x_ref"], name="crpix1") & models.Shift( + -wcsinfo["y_ref"], name="crpix2" + ) + pixelscale = models.Scale(wcsinfo["pixel_scale"], name="cdelt1") & models.Scale( + wcsinfo["pixel_scale"], name="cdelt2" + ) tangent_projection = models.Pix2Sky_TAN() - celestial_rotation = models.RotateNative2Celestial(ra_center, dec_center, 180.0) - det2sky = pixelshift | pixelscale | tangent_projection | celestial_rotation + celestial_rotation = models.RotateNative2Celestial( + wcsinfo["ra_ref"], wcsinfo["dec_ref"], 180.0 + ) + + matrix = wcsinfo.get("rotation_matrix", None) + if matrix: + matrix = np.array(matrix) + else: + orientat = wcsinfo.get("orientat", 0.0) + matrix = wcs_util.calc_rotation_matrix( + np.deg2rad(orientat), v3i_yangle=0.0, vparity=1 + ) + matrix = np.reshape(matrix, (2, 2)) + rotation = models.AffineTransformation2D(matrix, name="pc_rotation_matrix") + det2sky = ( + pixelshift | rotation | pixelscale | tangent_projection | celestial_rotation + ) + detector_frame = coordinate_frames.Frame2D( name="detector", axes_names=("x", "y"), unit=(u.pix, u.pix) ) sky_frame = coordinate_frames.CelestialFrame( reference_frame=coordinates.ICRS(), name="icrs", unit=(u.deg, u.deg) ) - wcsobj = WCS([(detector_frame, det2sky), (sky_frame, None)]) - wcsobj.bounding_box = bounding_box + wcsobj = WCS([(detector_frame, det2sky), (sky_frame, None)], name=name) + + if bounding_box: + wcsobj.bounding_box = bounding_box return wcsobj diff --git a/romancal/pipeline/tests/test_mosaic_pipeline.py b/romancal/pipeline/tests/test_mosaic_pipeline.py new file mode 100644 index 000000000..79bcd2d9b --- /dev/null +++ b/romancal/pipeline/tests/test_mosaic_pipeline.py @@ -0,0 +1,133 @@ +"""Unit tests for the mosaic pipeline""" + +import numpy as np + +import romancal.pipeline.mosaic_pipeline as mp + + +def test_skycell_to_wcs(): + """Test integrity of skycell_to_wcs""" + + skycell = np.void( + ( + "r274dp63x31y81", + 269.7783307416819, + 66.04965143695566, + 1781.5, + 1781.5, + 355.9788, + 3564, + 3564, + 67715.5, + -110484.5, + 269.6657957545588, + 65.9968687812357, + 269.6483032937494, + 66.09523979539262, + 269.89132874168854, + 66.10234971630734, + 269.9079118635897, + 66.00394719483091, + 0.1, + 274.2857142857143, + 63.0, + 0.0, + 463181, + ), + dtype=[ + ("name", "