diff --git a/CHANGES.rst b/CHANGES.rst index bbe3b6a1..82816b54 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,7 @@ - Allow assignment to or creation of node attributes using dot notation of object instances with validation. [#284] +- Move ``dqflags`` from ``romancal`` to ``roman_datamodels``. [#293] - Bugfix for ``model.meta.filename`` not matching the filename of the file on disk. [#295] diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py new file mode 100644 index 00000000..a0db85f6 --- /dev/null +++ b/src/roman_datamodels/dqflags.py @@ -0,0 +1,76 @@ +""" Roman Data Quality Flags + +The definitions are documented in the Roman RTD: + +[NOTE: Documentation not yet implemented. Fix this URL when completed.] + +https://roman-cal-pipeline.readthedocs.io/en/latest/roman/references_general/references_general.html#data-quality-flags + + +Implementation +------------- + +The flags are implemented as "bit flags": Each flag is assigned a bit position +in a byte, or multi-byte word, of memory. If that bit is set, the flag assigned +to that bit is interpreted as being set or active. + +The data structure that stores bit flags is just the standard Python `int`, +which provides 32 bits. Bits of an integer are most easily referred to using +the formula `2**bit_number` where `bit_number` is the 0-index bit of interest. +""" + +from enum import IntEnum, unique + + +# fmt: off +@unique +class pixel(IntEnum): + """Pixel-specific data quality flags""" + + GOOD = 0 # No bits set, all is good + DO_NOT_USE = 2**0 # Bad pixel. Do not use. + SATURATED = 2**1 # Pixel saturated during exposure + JUMP_DET = 2**2 # Jump detected during exposure + DROPOUT = 2**3 # Data lost in transmission + GW_AFFECTED_DATA = 2**4 # Data affected by the GW read window + PERSISTENCE = 2**5 # High persistence (was RESERVED_2) + AD_FLOOR = 2**6 # Below A/D floor (0 DN, was RESERVED_3) + OUTLIER = 2**7 # Flagged by outlier detection (was RESERVED_4) + UNRELIABLE_ERROR = 2**8 # Uncertainty exceeds quoted error + NON_SCIENCE = 2**9 # Pixel not on science portion of detector + DEAD = 2**10 # Dead pixel + HOT = 2**11 # Hot pixel + WARM = 2**12 # Warm pixel + LOW_QE = 2**13 # Low quantum efficiency + TELEGRAPH = 2**15 # Telegraph pixel + NONLINEAR = 2**16 # Pixel highly nonlinear + BAD_REF_PIXEL = 2**17 # Reference pixel cannot be used + NO_FLAT_FIELD = 2**18 # Flat field cannot be measured + NO_GAIN_VALUE = 2**19 # Gain cannot be measured + NO_LIN_CORR = 2**20 # Linearity correction not available + NO_SAT_CHECK = 2**21 # Saturation check not available + UNRELIABLE_BIAS = 2**22 # Bias variance large + UNRELIABLE_DARK = 2**23 # Dark variance large + UNRELIABLE_SLOPE = 2**24 # Slope variance large (i.e., noisy pixel) + UNRELIABLE_FLAT = 2**25 # Flat variance large + RESERVED_5 = 2**26 # + RESERVED_6 = 2**27 # + UNRELIABLE_RESET = 2**28 # Sensitive to reset anomaly + RESERVED_7 = 2**29 # + OTHER_BAD_PIXEL = 2**30 # A catch-all flag + REFERENCE_PIXEL = 2**31 # Pixel is a reference pixel + + +@unique +class group(IntEnum): + """Group-specific data quality flags + Once groups are combined, these flags are equivalent to the pixel-specific flags. + """ + GOOD = pixel.GOOD + DO_NOT_USE = pixel.DO_NOT_USE + SATURATED = pixel.SATURATED + JUMP_DET = pixel.JUMP_DET + DROPOUT = pixel.DROPOUT + AD_FLOOR = pixel.AD_FLOOR + +# fmt: on diff --git a/tests/test_dqflags.py b/tests/test_dqflags.py new file mode 100644 index 00000000..0b366543 --- /dev/null +++ b/tests/test_dqflags.py @@ -0,0 +1,104 @@ +from math import log10 + +import pytest + +from roman_datamodels import datamodels as rdm +from roman_datamodels import dqflags +from roman_datamodels.maker_utils import mk_datamodel + + +def _is_power_of_two(x): + return (log10(x) / log10(2)) % 1 == 0 + + +def test_pixel_uniqueness(): + """ + Test that there are no duplicate names in dqflags.pixel + + Note: The @unique decorator should ensure that no flag names have the + same value as another in the enum raising an error at first import + of this module. However, this test is just a sanity check on this. + """ + + assert len(dqflags.pixel) == len(dqflags.pixel.__members__) + + +@pytest.mark.parametrize("flag", dqflags.pixel) +def test_pixel_flags(flag): + """Test that each pixel flag follows the defined rules""" + # Test that the pixel flags are dqflags.pixel instances + assert isinstance(flag, dqflags.pixel) + + # Test that the pixel flags are ints + assert isinstance(flag, int) + + # Test that the pixel flags are dict accessible + assert dqflags.pixel[flag.name] is flag + + # Test that the pixel flag is a power of 2 + if flag.name == "GOOD": + # GOOD is the only non-power-of-two flag (it is 0) + assert flag.value == 0 + else: + assert _is_power_of_two(flag.value) + + +@pytest.mark.parametrize("flag", dqflags.pixel) +def test_write_pixel_flags(tmp_path, flag): + filename = tmp_path / "test_dq.asdf" + + ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8)) + + # Set all pixels to the flag value + ramp.pixeldq[...] = flag + + # Check that we can write the model to disk (i.e. the flag validates) + ramp.save(filename) + + # Check that we can read the model back in and the flag is preserved + with rdm.open(filename) as dm: + assert (dm.pixeldq == flag).all() + + +def test_group_uniqueness(): + """ + Test that there are no duplicate names in dqflags.group + + Note: The @unique decorator should ensure that no flag names have the + same value as another in the enum raising an error at first import + of this module. However, this test is just a sanity check on this. + """ + assert len(dqflags.group) == len(dqflags.group.__members__) + + +@pytest.mark.parametrize("flag", dqflags.group) +def test_group_flags(flag): + """Test that each group flag follows the defined rules""" + # Test that the group flags are dqflags.group instances + assert isinstance(flag, dqflags.group) + + # Test that the group flags are ints + assert isinstance(flag, int) + + # Test that the group flags are dict accessible + assert dqflags.group[flag.name] is flag + + # Test that each group flag matches a pixel flag of the same name + assert dqflags.pixel[flag.name] == flag + + +@pytest.mark.parametrize("flag", dqflags.group) +def test_write_group_flags(tmp_path, flag): + filename = tmp_path / "test_dq.asdf" + + ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8)) + + # Set all pixels to the flag value + ramp.groupdq[...] = flag + + # Check that we can write the model to disk (i.e. the flag validates) + ramp.save(filename) + + # Check that we can read the model back in and the flag is preserved + with rdm.open(filename) as dm: + assert (dm.groupdq == flag).all()