Add validation task for supported filetype IDs

marda_registry/utils.py

@@ -1,14 +1,14 @@
 import glob
 import pathlib
-from typing import Type
+from typing import Any, Type
 
 import mongomock as pymongo
 import yaml
 
 
 def load_registry_collection(
     model: Type, database: pymongo.Database | None = None, validate: bool = True
-) -> int:
+) -> list[Any]:
     """Loads any entries of the specified model ty pes from the corresponding data directory,
     optionally validating and inserting them into the given database.
 
@@ -18,17 +18,22 @@ def load_registry_collection(
         validate: Whether to validate the entries before inserting them into the database.
 
     Returns:
-        The number of entries ingested for that type.
+        The entries ingested for that type.
 
     """
     name = model.__name__.lower() + "s"
-    entries = glob.glob(str(pathlib.Path(__file__).parent / "data" / name / "*.yml"))
-    for entry in entries:
+    filenames = glob.glob(str(pathlib.Path(__file__).parent / "data" / name / "*.yml"))
+    entries = []
+    for entry in filenames:
         with open(entry, "r") as f:
             data = yaml.safe_load(f)
+
         if validate:
-            model(**data)
+            entries.append(model(**data))
+        else:
+            entries.append(data)
+
         if database:
             database[name].insert_one(data)
 
-    return len(entries)
+    return entries

tasks.py

@@ -39,14 +39,35 @@ def validate_entries(_):
     from marda_registry.utils import load_registry_collection
 
     counts = {}
+    errors = []
     for type_ in (FileType, Extractor):
-        counts[type_] = load_registry_collection(
+        entries = load_registry_collection(
             type_,
             database=None,
             validate=True,
         )
+        counts[type_] = len(entries)
         print(f"Loaded {counts[type_]} {type_.__name__} entries")
 
+        if type_ is Extractor:
+            filetype_ids = set(
+                d.stem
+                for d in Path(__file__).parent.glob(
+                    "./marda_registry/data/filetypes/*.yml"
+                )
+            )
+
+            for extractor in entries:
+                for filetype in extractor.supported_filetypes:
+                    if filetype.id not in filetype_ids:
+                        breakpoint()
+                        errors.append(
+                            f"Extractor {extractor.name=} has invalid filetype {filetype.id=}. Should be one of {filetype_ids=}"
+                        )
+
+    if errors:
+        raise RuntimeError("\n".join(errors))
+
     print("Done!")