RCAL-977 - Version datamodels

changes/528.doc.rst

@@ -0,0 +1 @@
+Add section describing versioning and old file support.

docs/index.rst

@@ -35,6 +35,7 @@ Developer Resources
   :maxdepth: 1
 
   creating.rst
+  versioning.rst
   contributing.rst
   changes.rst
 

docs/versioning.rst

@@ -0,0 +1,59 @@
+.. _versioning:
+
+Schema Versioning
+=================
+
+As RAD contains ASDF compatible schemas the versioning strategy will
+mostly follow that of other ASDF extensions where:
+
+- schema modifications trigger creation of a new schema version
+- the old schema is kept unmodified
+- a new tag version is created for the new schema version
+- the new tag version triggers a new manifest version
+
+This allows files created with the old tag to be validated (against
+the old schema) and opened.
+
+Manifest Versioning
+===================
+
+New manifests will often be created by:
+
+- copying the newest manifest version
+- incrementing the manifest version number, id and extension uri version
+- updating the tag definition for that tag/schema change that triggered
+  the manifest version
+
+One exception is if the newest manifest version has not yet been
+released. In this case the tag definition in the existing (unreleased)
+manifest can be modified and no manifest version increase is needed.
+
+
+Old version support
+===================
+
+RAD is not yet stable. Efforts will be made to retain support for
+opening old files. As noted above supporting old versions of files
+will require keeping several manifest versions and all old schemas.
+As development continues it may be advantageous to drop support
+for some old (pre-flight) versions.
+
+
+Dropping support for pre-flight versions
+========================================
+
+If it is decided that support for an old (pre-flight) version
+of a schema will be dropped the following steps will be taken:
+
+- removal of the unsupported schema versions
+- removal of the unsupported tag versions
+- removal of all manifest versions that contain the dropped schema or tag versions
+
+By following these steps, the unsupported old files can still
+be opened with ``asdf.open``. When an unsupported file is opened
+asdf will encounter one or more of the unsupported (and now unknown)
+tags and issue ``AsdfConversionWarning`` describing that the tagged objects
+are being returned as "raw data structures" (typically a
+dictionary-like ``TaggedDict``). This will allow users to continue
+to access the contents of the file and possibly migrate the old file
+contents to a new supported tag/structure.

tests/conftest.py

@@ -2,9 +2,20 @@
 import pytest
 import yaml
 
-MANIFEST = yaml.safe_load(asdf.get_config().resource_manager["asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0"])
+MANIFEST_URIS = [
+    "asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0",
+]
+MANIFESTS = [yaml.safe_load(asdf.get_config().resource_manager[manifest_uri]) for manifest_uri in MANIFEST_URIS]
+MANIFEST_ENTRIES = []
+for manifest in MANIFESTS:
+    MANIFEST_ENTRIES.extend(manifest["tags"])
 
 
-@pytest.fixture(scope="session")
-def manifest():
-    return MANIFEST
+@pytest.fixture(scope="session", params=MANIFEST_ENTRIES)
+def manifest_entry(request):
+    return request.param
+
+
+@pytest.fixture(scope="session", params=MANIFESTS)
+def manifest(request):
+    return request.param

tests/test_manifest.py

@@ -4,9 +4,6 @@
 """
 
 import asdf
-import pytest
-
-from .conftest import MANIFEST
 
 
 def test_manifest_valid(manifest):
@@ -18,20 +15,19 @@ def test_manifest_valid(manifest):
     assert "description" in manifest
 
 
-@pytest.mark.parametrize("entry", MANIFEST["tags"])
-def test_manifest_entries(entry):
+def test_manifest_entries(manifest_entry):
     # Check that the schema exists:
-    assert entry["schema_uri"] in asdf.get_config().resource_manager
+    assert manifest_entry["schema_uri"] in asdf.get_config().resource_manager
     # These are not required by the manifest schema but we're holding ourselves
     # to a higher standard:
-    assert "title" in entry
-    assert "description" in entry
+    assert "title" in manifest_entry
+    assert "description" in manifest_entry
 
     # Check the URIs
-    assert entry["tag_uri"].startswith("asdf://stsci.edu/datamodels/roman/tags/")
-    uri_suffix = entry["tag_uri"].split("asdf://stsci.edu/datamodels/roman/tags/")[-1]
+    assert manifest_entry["tag_uri"].startswith("asdf://stsci.edu/datamodels/roman/tags/")
+    uri_suffix = manifest_entry["tag_uri"].split("asdf://stsci.edu/datamodels/roman/tags/")[-1]
     # Remove tagged scalars from the uri string
-    schema_uri = entry["schema_uri"]
+    schema_uri = manifest_entry["schema_uri"]
     if "tagged_scalars" in schema_uri.split("/"):
         schema_uri = schema_uri.replace("tagged_scalars/", "")
     assert schema_uri.endswith(uri_suffix)

tests/test_schemas.py

@@ -11,18 +11,35 @@
 import yaml
 from crds.config import is_crds_name
 
-from .conftest import MANIFEST
+from .conftest import MANIFESTS
 
 SCHEMA_URI_PREFIX = "asdf://stsci.edu/datamodels/roman/schemas/"
 METASCHEMA_URI = "asdf://stsci.edu/datamodels/roman/schemas/rad_schema-1.0.0"
-SCHEMA_URIS = [u for u in asdf.get_config().resource_manager if u.startswith(SCHEMA_URI_PREFIX) and u != METASCHEMA_URI]
-REF_FILE_SCHEMA_URIS = [u["schema_uri"] for u in MANIFEST["tags"] if "/reference_files/" in u["schema_uri"]]
-WFI_OPTICAL_ELEMENTS = list(
+SCHEMA_URIS = tuple(u for u in asdf.get_config().resource_manager if u.startswith(SCHEMA_URI_PREFIX) and u != METASCHEMA_URI)
+TAG_DEFS = tuple(tag_def for manifest in MANIFESTS for tag_def in manifest["tags"])
+REF_FILE_TAG_DEFS = tuple(tag_def for tag_def in TAG_DEFS if "/reference_files" in tag_def["schema_uri"])
+WFI_OPTICAL_ELEMENTS = tuple(
     asdf.schema.load_schema("asdf://stsci.edu/datamodels/roman/schemas/wfi_optical_element-1.0.0")["enum"]
 )
-EXPOSURE_TYPE_ELEMENTS = list(asdf.schema.load_schema("asdf://stsci.edu/datamodels/roman/schemas/exposure_type-1.0.0")["enum"])
+EXPOSURE_TYPE_ELEMENTS = tuple(asdf.schema.load_schema("asdf://stsci.edu/datamodels/roman/schemas/exposure_type-1.0.0")["enum"])
 EXPECTED_COMMON_REFERENCE = {"$ref": "asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0"}
-METADATA_FORCING_REQUIRED = ["archive_catalog", "sdf"]
+METADATA_FORCING_REQUIRED = ("archive_catalog", "sdf")
+ALLOWED_SCHEMA_TAG_VALIDATORS = (
+    "tag:stsci.edu:asdf/time/time-1.*",
+    "tag:stsci.edu:asdf/core/ndarray-1.*",
+    "tag:stsci.edu:asdf/unit/quantity-1.*",
+    "tag:stsci.edu:asdf/unit/unit-1.*",
+    "tag:astropy.org:astropy/units/unit-1.*",
+    "tag:astropy.org:astropy/table/table-1.*",
+    "tag:stsci.edu:gwcs/wcs-*",
+)
+
+
+@pytest.fixture(scope="session")
+def valid_tag_uris():
+    uris = {t["tag_uri"] for manifest in MANIFESTS for t in manifest["tags"]}
+    uris.update(ALLOWED_SCHEMA_TAG_VALIDATORS)
+    return uris
 
 
 @pytest.fixture(scope="session", params=SCHEMA_URIS)
@@ -35,33 +52,16 @@ def schema(request):
     return yaml.safe_load(asdf.get_config().resource_manager[request.param])
 
 
-@pytest.fixture(scope="session", params=REF_FILE_SCHEMA_URIS)
+@pytest.fixture(scope="session", params=REF_FILE_TAG_DEFS)
 def ref_file_schema(request):
-    return yaml.safe_load(asdf.get_config().resource_manager[request.param])
+    return yaml.safe_load(asdf.get_config().resource_manager[request.param["schema_uri"]])
 
 
-@pytest.fixture(scope="session", params=[entry for entry in MANIFEST["tags"] if "/reference_files/" in entry["schema_uri"]])
+@pytest.fixture(scope="session", params=REF_FILE_TAG_DEFS)
 def ref_file_uris(request):
     return request.param["tag_uri"], request.param["schema_uri"]
 
 
-@pytest.fixture(scope="session")
-def valid_tag_uris(manifest):
-    uris = {t["tag_uri"] for t in manifest["tags"]}
-    uris.update(
-        [
-            "tag:stsci.edu:asdf/time/time-1.*",
-            "tag:stsci.edu:asdf/core/ndarray-1.*",
-            "tag:stsci.edu:asdf/unit/quantity-1.*",
-            "tag:stsci.edu:asdf/unit/unit-1.*",
-            "tag:astropy.org:astropy/units/unit-1.*",
-            "tag:astropy.org:astropy/table/table-1.*",
-            "tag:stsci.edu:gwcs/wcs-*",
-        ]
-    )
-    return uris
-
-
 def test_required_properties(schema):
     assert schema["$schema"] == METASCHEMA_URI
     assert "id" in schema
@@ -75,8 +75,8 @@ def test_schema_style(schema_content):
     assert not any(line != line.rstrip() for line in schema_content.split(b"\n"))
 
 
-def test_property_order(schema, manifest):
-    is_tag_schema = schema["id"] in {t["schema_uri"] for t in manifest["tags"]}
+def test_property_order(schema):
+    is_tag_schema = schema["id"] in {t["schema_uri"] for t in TAG_DEFS}
 
     if is_tag_schema: