Skip to content

Commit

Permalink
Initial commit.
Browse files Browse the repository at this point in the history
  • Loading branch information
PaulHuwe committed Sep 30, 2024
1 parent ce9eb29 commit 77e5100
Show file tree
Hide file tree
Showing 8 changed files with 221 additions and 10 deletions.
12 changes: 6 additions & 6 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ This PR addresses ...

<!-- if you can't perform these tasks due to permissions, please ask a maintainer to do them -->
## Tasks
- [ ] update or add relevant tests
- [ ] update relevant docstrings and / or `docs/` page
- [ ] Does this PR change any API used downstream? (if not, label with `no-changelog-entry-needed`)
- [ ] write news fragment(s) in `changes/`: `echo "changed something" > changes/<PR#>.<changetype>.rst` (see below for change types)
- [ ] [start a `romancal` regression test](https://github.com/spacetelescope/RegressionTests/actions/workflows/romancal.yml) with this branch installed (`"git+https://github.com/<fork>/roman_datamodels@<branch>"`)
- [ ] Update or add relevant `roman_datamodels` tests.
- [ ] Update relevant docstrings and / or `docs/` page.
- [ ] Does this PR change any API used downstream? (If not, label with `no-changelog-entry-needed`.)
- [ ] Write news fragment(s) in `changes/`: `echo "changed something" > changes/<PR#>.<changetype>.rst` (see below for change types).
- [ ] Start a `romancal` regression test (https://github.com/spacetelescope/RegressionTests/actions/workflows/romancal.yml) with this branch installed (`"git+https://github.com/<fork>/rad@<branch>"`).

<details><summary>news fragment change types...</summary>
<details><summary>News fragment change types:</summary>

- ``changes/<PR#>.feature.rst``: new feature
- ``changes/<PR#>.bugfix.rst``: fixes an issue
Expand Down
12 changes: 12 additions & 0 deletions src/roman_datamodels/datamodels/_datamodels.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,14 @@ class FlatRefModel(_DataModel):
_node_type = stnode.FlatRef


class AbvegaoffsetRefModel(_DataModel):
_node_type = stnode.AbvegaoffsetRef


class ApcorrRefModel(_DataModel):
_node_type = stnode.ApcorrRef


class DarkRefModel(_DataModel):
_node_type = stnode.DarkRef

Expand All @@ -235,6 +243,10 @@ class DistortionRefModel(_DataModel):
_node_type = stnode.DistortionRef


class EpsfRefModel(_DataModel):
_node_type = stnode.EpsfRef


class GainRefModel(_DataModel):
_node_type = stnode.GainRef

Expand Down
16 changes: 16 additions & 0 deletions src/roman_datamodels/maker_utils/_common_meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -659,6 +659,22 @@ def mk_ref_dark_meta(**kwargs):

return meta

def mk_ref_epsf_meta(**kwargs):
"""
Create dummy metadata for ePSF reference file instances.
Returns
-------
dict (follows reference_file/ref_common-1.0.0 schema + ePSF reference file metadata)
"""
meta = mk_ref_common("EPSF", **kwargs)
meta["oversample"] = kwargs.get("oversample", NONUM)
meta["effective_temperature"] = kwargs.get("effective_temperature", np.arange(1, 10).tolist())
meta["defocus"] = kwargs.get("defocus", np.arange(1, 10).tolist())
meta["pixel_x"] = kwargs.get("pixel_x", np.arange(1, 10, dtype=np.float32).tolist())
meta["pixel_y"] = kwargs.get("pixel_y", np.arange(1, 10, dtype=np.float32).tolist())

return meta

def mk_ref_distoriton_meta(**kwargs):
"""
Expand Down
142 changes: 138 additions & 4 deletions src/roman_datamodels/maker_utils/_ref_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,24 @@

from roman_datamodels import stnode

from ._base import MESSAGE, save_node
from ._base import MESSAGE, save_node, NONUM
from ._common_meta import (
mk_ref_common,
mk_ref_dark_meta,
mk_ref_distoriton_meta,
mk_ref_epsf_meta,
mk_ref_pixelarea_meta,
mk_ref_readnoise_meta,
mk_ref_units_dn_meta,
)

__all__ = [
"mk_abvegaoffset",
"mk_apcorr",
"mk_flat",
"mk_dark",
"mk_distortion",
"mk_epsf",
"mk_gain",
"mk_ipc",
"mk_linearity",
Expand All @@ -33,6 +37,106 @@
"mk_refpix",
]

OPT_ELEM = ("F062", "F087", "F106", "F129", "F146", "F158", "F184", "F213", "GRISM", "PRISM", "DARK")


def mk_ref_abvegaoffset_data(**kwargs):
"""
Create dummy data for AB Vega Offset reference file instances.
Returns
-------
dict
"""
data = {}
for optical_elem in OPT_ELEM:
data[optical_elem] = {}
data[optical_elem]['abvega_offset'] = kwargs.get(optical_elem,{}).get("abvega_offset", float(NONUM))

return data


def mk_abvegaoffset(*, filepath=None, **kwargs):
"""
Create a dummy AB Vega Offset instance (or file) with valid values
for attributes required by the schema.
Parameters
----------
filepath
(optional, keyword-only) File name and path to write model to.
Returns
-------
roman_datamodels.stnode.AbvegaoffsetRef
"""
abvegaref = stnode.AbvegaoffsetRef()
abvegaref["meta"] = mk_ref_common("ABVEGAOFFSET", **kwargs.get("meta", {}))
abvegaref["data"] = mk_ref_abvegaoffset_data(**kwargs.get("data", {}))

return save_node(abvegaref, filepath=filepath)


def mk_ref_apcorr_data(shape=(10,), **kwargs):
"""
Create dummy data for Aperture Correction reference file instances.
Parameters
----------
shape
(optional, keyword-only) Shape of arrays in the model.
If shape is greater than 1D, the first dimension is used.
Returns
-------
dict
"""
if len(shape) > 1:
shape = shape[:1]

warnings.warn(f"{MESSAGE} assuming the first entry. The remaining are thrown out!", UserWarning)

data = {}
for optical_elem in OPT_ELEM:
data[optical_elem] = {}
data[optical_elem]['ap_corrections'] = kwargs.get(optical_elem,{}).get("ap_corrections", np.zeros(shape, dtype=np.float64))
data[optical_elem]['ee_fractions'] = kwargs.get(optical_elem,{}).get("ee_fractions", np.zeros(shape, dtype=np.float64))
data[optical_elem]['ee_radii'] = kwargs.get(optical_elem,{}).get("ee_radii", np.zeros(shape, dtype=np.float64))
data[optical_elem]['sky_background_rin'] = kwargs.get(optical_elem,{}).get("sky_background_rin", float(NONUM))
data[optical_elem]['sky_background_rout'] = kwargs.get(optical_elem,{}).get("sky_background_rout", float(NONUM))

return data


def mk_apcorr(*, shape=(10,), filepath=None, **kwargs):
"""
Create a dummy Aperture Correction instance (or file) with arrays and valid values
for attributes required by the schema.
Parameters
----------
shape
(optional, keyword-only) Shape of arrays in the model.
If shape is greater than 1D, the first dimension is used.
filepath
(optional, keyword-only) File name and path to write model to.
Returns
-------
roman_datamodels.stnode.ApcorrRef
"""
if len(shape) > 1:
shape = shape[:1]

warnings.warn(f"{MESSAGE} assuming the first entry. The remaining is thrown out!", UserWarning)

apcorrref = stnode.ApcorrRef()
apcorrref["meta"] = mk_ref_common("APCORR", **kwargs.get("meta", {}))
apcorrref["data"] = mk_ref_apcorr_data(shape, **kwargs.get("data", {}))

return save_node(apcorrref, filepath=filepath)


def mk_flat(*, shape=(4096, 4096), filepath=None, **kwargs):
"""
Expand Down Expand Up @@ -128,6 +232,38 @@ def mk_distortion(*, filepath=None, **kwargs):
return save_node(distortionref, filepath=filepath)


def mk_epsf(*, shape=(3, 9, 361, 361), filepath=None, **kwargs):
"""
Create a dummy ePSF instance (or file) with arrays and valid values
for attributes required by the schema.
Parameters
----------
shape
(optional, keyword-only) Shape of arrays in the model.
If shape is greater than 1D, the first dimension is used.
filepath
(optional, keyword-only) File name and path to write model to.
Returns
-------
roman_datamodels.stnode.EpsfRef
"""
if len(shape) != 4:
shape = (3, 9, 361, 361)
warnings.warn("Input shape must be 4D. Defaulting to (3, 9, 361, 361)")

epsfref = stnode.EpsfRef()
epsfref["meta"] = mk_ref_epsf_meta(**kwargs.get("meta", {}))

epsfref['psf'] = kwargs.get("psf", np.zeros(shape, dtype=np.float32))
epsfref['extended_psf'] = kwargs.get("extended_psf", np.zeros(shape[2:], dtype=np.float32))


return save_node(epsfref, filepath=filepath)


def mk_gain(*, shape=(4096, 4096), filepath=None, **kwargs):
"""
Create a dummy Gain instance (or file) with arrays and valid values for
Expand Down Expand Up @@ -340,9 +476,7 @@ def _mk_phot_table(**kwargs):
"""
Create the phot_table for the photom reference file.
"""
entries = ("F062", "F087", "F106", "F129", "F146", "F158", "F184", "F213", "GRISM", "PRISM", "DARK")

return {entry: _mk_phot_table_entry(entry, **kwargs.get(entry, {})) for entry in entries}
return {entry: _mk_phot_table_entry(entry, **kwargs.get(entry, {})) for entry in OPT_ELEM}


def mk_wfi_img_photom(*, filepath=None, **kwargs):
Expand Down
4 changes: 4 additions & 0 deletions tests/test_maker_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ def test_maker_utility_implemented(node_class):

@pytest.mark.parametrize("node_class", stnode.NODE_CLASSES)
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_instance_valid(node_class):
"""
Expand All @@ -39,6 +40,7 @@ def test_instance_valid(node_class):

@pytest.mark.parametrize("node_class", [c for c in stnode.NODE_CLASSES if issubclass(c, stnode.TaggedObjectNode)])
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_no_extra_fields(node_class, manifest):
instance = maker_utils.mk_node(node_class, shape=(8, 8, 8))
Expand Down Expand Up @@ -100,6 +102,7 @@ def test_deprecated():

@pytest.mark.parametrize("model_class", [mdl for mdl in maker_utils.NODE_REGISTRY])
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_datamodel_maker(model_class):
"""
Expand All @@ -120,6 +123,7 @@ def test_datamodel_maker(model_class):

@pytest.mark.parametrize("node_class", [node for node in datamodels.MODEL_REGISTRY])
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_override_data(node_class):
"""
Expand Down
40 changes: 40 additions & 0 deletions tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def test_datamodel_exists(name):

@pytest.mark.parametrize("model", datamodels.MODEL_REGISTRY.values())
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_node_type_matches_model(model):
"""
Expand Down Expand Up @@ -348,6 +349,27 @@ def test_reference_file_model_base(tmp_path):
if not found_common:
raise ValueError("Reference schema does not include ref_common") # pragma: no cover

# AB Vega Offest Correction tests
def test_make_abvegaoffset():
abvegaoffset = utils.mk_abvegaoffset()
assert abvegaoffset.meta.reftype == "ABVEGAOFFSET"
assert isinstance(abvegaoffset.data.GRISM["abvega_offset"], float)

# Test validation
abvegaoffset_model = datamodels.AbvegaoffsetRefModel(abvegaoffset)
assert abvegaoffset_model.validate() is None

# Aperture Correction tests
def test_make_apcorr():
apcorr = utils.mk_apcorr()
assert apcorr.meta.reftype == "APCORR"
assert isinstance(apcorr.data.DARK["sky_background_rin"], float)
assert isinstance(apcorr.data.DARK["ap_corrections"], np.ndarray)
assert isinstance(apcorr.data.DARK["ap_corrections"][0], float)

# Test validation
apcorr_model = datamodels.ApcorrRefModel(apcorr)
assert apcorr_model.validate() is None

# Flat tests
def test_make_flat():
Expand Down Expand Up @@ -421,6 +443,22 @@ def test_make_distortion():
assert distortion_model.validate() is None


# ePSF tests
def test_make_epsf():
epsf = utils.mk_epsf(shape=(2, 4, 8, 8))
assert epsf.meta.reftype == "EPSF"
assert isinstance(epsf.meta['pixel_x'], list)
assert isinstance(epsf.meta['pixel_x'][0], float)
assert epsf['psf'].shape == (2, 4, 8, 8)
print(f"XXX type(epsf['psf'][0,0,0,0]) = {type(epsf['psf'][0,0,0,0])}")
assert isinstance(epsf['psf'][0,0,0,0], (float, np.float32))


# Test validation
epsf_model = datamodels.EpsfRefModel(epsf)
assert epsf_model.validate() is None


# Gain tests
def test_make_gain():
gain = utils.mk_gain(shape=(8, 8))
Expand Down Expand Up @@ -890,6 +928,7 @@ def test_model_validate_without_save():
@pytest.mark.parametrize("node", datamodels.MODEL_REGISTRY.keys())
@pytest.mark.parametrize("correct, model", datamodels.MODEL_REGISTRY.items())
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_model_only_init_with_correct_node(node, correct, model):
"""
Expand Down Expand Up @@ -932,6 +971,7 @@ def test_ramp_from_science_raw():

@pytest.mark.parametrize("model", datamodels.MODEL_REGISTRY.values())
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_datamodel_construct_like_from_like(model):
"""
Expand Down
2 changes: 2 additions & 0 deletions tests/test_open.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ def test_no_memmap(tmp_path, kwargs):

@pytest.mark.parametrize("node_class", [node for node in datamodels.MODEL_REGISTRY])
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_node_round_trip(tmp_path, node_class):
file_path = tmp_path / "test.asdf"
Expand All @@ -195,6 +196,7 @@ def test_node_round_trip(tmp_path, node_class):

@pytest.mark.parametrize("node_class", [node for node in datamodels.MODEL_REGISTRY])
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_opening_model(tmp_path, node_class):
file_path = tmp_path / "test.asdf"
Expand Down
3 changes: 3 additions & 0 deletions tests/test_stnode.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def test_generated_node_classes(tag):

@pytest.mark.parametrize("node_class", stnode.NODE_CLASSES)
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_copy(node_class):
"""Demonstrate nodes can copy themselves, but don't always deepcopy."""
Expand All @@ -49,6 +50,7 @@ def test_copy(node_class):

@pytest.mark.parametrize("node_class", datamodels.MODEL_REGISTRY.keys())
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_deepcopy_model(node_class):
node = maker_utils.mk_node(node_class, shape=(8, 8, 8))
Expand Down Expand Up @@ -90,6 +92,7 @@ def test_wfi_mode():

@pytest.mark.parametrize("node_class", stnode.NODE_CLASSES)
@pytest.mark.filterwarnings("ignore:This function assumes shape is 2D")
@pytest.mark.filterwarnings("ignore:Input shape must be 4D")
@pytest.mark.filterwarnings("ignore:Input shape must be 5D")
def test_serialization(node_class, tmp_path):
file_path = tmp_path / "test.asdf"
Expand Down

0 comments on commit 77e5100

Please sign in to comment.