Add schema types description to docs (and misc cleanup)

README.md

@@ -5,7 +5,7 @@
 [![codecov](https://codecov.io/gh/spacetelescope/stdatamodels/branch/main/graph/badge.svg?token=TrmUKaTP2t)](https://codecov.io/gh/spacetelescope/stdatamodels)
 
 
-Provides `DataModel`, which is the base class for data models implemented in the JWST and Roman calibration software.
+Provides JWST data model classes and schemas.
 
 
 ## Unit Tests

changes/338.doc.rst

@@ -0,0 +1 @@
+Add to schema docs describing schema types.

docs/source/jwst/datamodels/attributes.rst

@@ -5,70 +5,6 @@ the underlying file format.  The same data model may be used for data
 created from scratch in memory, loaded from FITS or ASDF files, or from
 some other future format.
 
-Calling sequences of models
-===========================
-
-List of current models
-----------------------
-
-The current models are as follows:
-
-    'ABVegaOffsetModel',
-    'AmiLgModel',
-    'FgsImgApcorrModel', 'MirImgApcorrModel', 'NrcImgApcorrModel', 'NisImgApcorrModel',
-    'MirLrsApcorrModel', 'MirMrsApcorrModel', 'NrcWfssApcorrModel', 'NisWfssApcorrModel',
-    'NrsMosApcorrModel', 'NrsFsApcorrModel', 'NrsIfuApcorrModel',
-    'AsnModel',
-    'BarshadowModel', 'CameraModel', 'CollimatorModel',
-    'CombinedSpecModel', 'ContrastModel', 'CubeModel',
-    'DarkModel', 'DarkMIRIModel',
-    'DisperserModel', 'DistortionModel', 'DistortionMRSModel',
-    'Extract1dImageModel',
-    'Extract1dIFUModel',
-    'FilteroffsetModel',
-    'FlatModel', 'NirspecFlatModel', 'NirspecQuadFlatModel',
-    'FOREModel', 'FPAModel',
-    'FringeModel', 'GainModel', 'GLS_RampFitModel',
-    'GuiderRawModel', 'GuiderCalModel',
-    'IFUCubeModel',
-    'NirspecIFUCubeParsModel', 'MiriIFUCubeParsModel',
-    'IFUFOREModel', 'IFUImageModel', 'IFUPostModel', 'IFUSlicerModel',
-    'ImageModel', 'IPCModel', 'IRS2Model', 'LastFrameModel', 'Level1bModel',
-    'LinearityModel', 'MaskModel', 'MSAModel',
-    'MultiCombinedSpecModel', 'MultiExposureModel',
-    'MultiExtract1dImageModel', 'MultiSlitModel',
-    'MultiSpecModel',
-    'NIRCAMGrismModel', 'NIRISSGrismModel',
-    'OTEModel',
-    'OutlierParsModel',
-    'PathlossModel',
-    'PersistenceSatModel',
-    'PixelAreaModel', 'NirspecSlitAreaModel', 'NirspecMosAreaModel', 'NirspecIfuAreaModel',
-    'FgsImgPhotomModel',
-    'MirImgPhotomModel', 'MirLrsPhotomModel', 'MirMrsPhotomModel',
-    'NrcImgPhotomModel', 'NrcWfssPhotomModel',
-    'NisImgPhotomModel', 'NisSossPhotomModel', 'NisWfssPhotomModel',
-    'NrsFsPhotomModel', 'NrsMosPhotomModel',
-    'PsfMaskModel',
-    'QuadModel', 'RampModel',
-    'RampFitOutputModel', 'ReadnoiseModel',
-    'ReferenceFileModel', 'ReferenceCubeModel', 'ReferenceImageModel', 'ReferenceQuadModel',
-    'RegionsModel', 'ResetModel',
-    'ResolutionModel', 'MiriResolutionModel',
-    'RSCDModel', 'SaturationModel', 'SlitDataModel', 'SlitModel', 'SpecModel',
-    'SegmentationMapModel',
-    'SpecKernelModel',
-    'SpecProfileModel', 'SpecProfileSingleModel',
-    'SpecTraceModel', 'SpecTraceSingleModel',
-    'SpecwcsModel',
-    'StrayLightModel', 'SuperBiasModel',
-    'ThroughputModel',
-    'TrapDensityModel', 'TrapParsModel', 'TrapsFilledModel',
-    'TsoPhotModel',
-    'WavelengthrangeModel', 'WaveCorrModel',
-    'WaveMapModel', 'WaveMapSingleModel',
-    'WfssBkgModel'
-
 Commonly used attributes
 ------------------------
 Here are a few model attributes that are used by some of the pipeline
@@ -105,7 +41,7 @@ instruments have four columns or rows of reference pixels on each edge
 of the image.
 
 JwstDataModel Base Class
---------------------
+------------------------
 
 .. autoclass:: jwst.datamodels.JwstDataModel
    :members:

docs/source/jwst/datamodels/index.rst

@@ -12,5 +12,6 @@ Data Models
    metadata.rst
    new_model.rst
    structure.rst
+   schemas.rst
 
 .. automodapi:: jwst.datamodels

docs/source/jwst/datamodels/schemas.rst

@@ -0,0 +1,56 @@
+JWST schemas
+============
+
+This package contains schemas of the following types:
+
+- Data model schemas
+- Reference file schemas
+- Transform schemas (not covered here)
+
+
+Data model schemas
+------------------
+
+JWST datamodels are in part defined by an ASDF schema.
+For example ``RampModel`` uses the schema found in ``ramp.schema.yaml``.
+These data model schemas typically contain many references
+to other schemas to allow common structures to be shared across
+data models. Here is a (partial) example:
+
+.. code-block:: yaml
+
+  %YAML 1.1
+  ---
+  $schema: "http://stsci.edu/schemas/fits-schema/fits-schema"
+  id: "http://stsci.edu/schemas/jwst_datamodel/ramp.schema"
+  allOf:
+  - $ref: core.schema
+  - $ref: bunit.schema
+  - $ref: photometry.schema
+  - $ref: wcsinfo.schema
+  - type: object
+
+Each ``$ref`` above will pull in the common structure defined
+in the referenced schema. All data model schemas reference
+``core.schema``.
+
+
+Reference file schemas
+----------------------
+
+JWST reference file schemas are similar to the data model schemas
+but use a different set of shared schemas.
+
+.. code-block:: yaml
+
+  %YAML 1.1
+  ---
+  $schema: "http://stsci.edu/schemas/fits-schema/fits-schema"
+  id: "http://stsci.edu/schemas/jwst_datamodel/dark.schema"
+  title: Dark current data model
+  allOf:
+  - $ref: referencefile.schema
+  - $ref: keyword_exptype.schema
+  - $ref: keyword_readpatt.schema
+
+Note that reference file schemas ``$ref`` ``referencefile.schema``.

src/stdatamodels/jwst/datamodels/tests/test_integration.py

@@ -4,6 +4,9 @@
 import pytest
 import yaml
 
+from stdatamodels.jwst.datamodels import JwstDataModel
+import stdatamodels.schema
+
 
 METASCHEMAS = list(
     importlib.resources.files("stdatamodels.jwst.datamodels.metaschema").glob("*.yaml")
@@ -32,6 +35,17 @@
 RESOURCES = SCHEMAS + TRANSFORM_MANIFESTS
 
 
+def datamodel_associated_schemas():
+    """
+    Get all schemas directly associated with a datamodel
+    """
+    schema_urls = []
+    for subclass in JwstDataModel.__subclasses__():
+        if subclass.schema_url:
+            schema_urls.append(subclass.schema_url)
+    return schema_urls
+
+
 @pytest.mark.parametrize("resource", RESOURCES)
 def test_resource_id(resource):
     """
@@ -68,3 +82,39 @@ def test_manifest_tag_versions(manifest_filename):
         # although not generally required for stdatamodels transforms all
         # tag versions should match schema version
         assert tag_version == schema_version
+
+
+@pytest.mark.parametrize("datamodel_schema_file", datamodel_associated_schemas())
+def test_schema_refs_base(datamodel_schema_file):
+    """
+    Each datamodel schema should either reference:
+        - http://stsci.edu/schemas/jwst_datamodel/core.schema (for data models)
+        - http://stsci.edu/schemas/jwst_datamodel/referencefile.schema (for reference files)
+    But not both
+    """
+
+    # these schemas don't reference either core.schema or referencefile.schema
+    if datamodel_schema_file in [
+        "http://stsci.edu/schemas/jwst_datamodel/slitdata.schema",
+        "http://stsci.edu/schemas/jwst_datamodel/extract1dimage.schema",
+    ]:
+        return
+
+    schema = asdf.schema.load_schema(datamodel_schema_file, resolve_references=True)
+
+    def cb(subschema, path, combiner, ctx, recurse):
+        if not isinstance(subschema, dict):
+            return
+        if 'id' not in subschema:
+            return
+        ctx['ids'].add(subschema['id'])
+
+    seen_ids = set()
+    stdatamodels.schema.walk_schema(schema, cb, ctx={'ids': seen_ids})
+
+    if 'http://stsci.edu/schemas/jwst_datamodel/core.schema' in seen_ids:
+        assert 'http://stsci.edu/schemas/jwst_datamodel/referencefile.schema' not in seen_ids
+    elif 'http://stsci.edu/schemas/jwst_datamodel/referencefile.schema' in seen_ids:
+        assert 'http://stsci.edu/schemas/jwst_datamodel/core.schema' not in seen_ids
+    else:
+        assert False