No inner sections

src/pynxtools/nomad/entrypoints.py

@@ -63,7 +63,7 @@ def load(self):
     name="pynxtools parser",
     description="A parser for nexus files.",
     mainfile_name_re=r".*\.nxs",
-    mainfile_mime_re="application/x-hdf5",
+    mainfile_mime_re="application/x-hdf*",
 )
 
 from nomad.config.models.ui import (
@@ -76,7 +76,7 @@ def load(self):
     SearchQuantities,
 )
 
-schema = "pynxtools.nomad.schema.NeXus"
+schema = "pynxtools.nomad.schema.Root"
 
 nexus_app = AppEntryPoint(
     name="NexusApp",
@@ -105,17 +105,17 @@ def load(self):
             Column(quantity=f"entry_type", selected=True),
             Column(
                 title="definition",
-                quantity=f"data.*.ENTRY[*].definition__field#{schema}",
+                quantity=f"data.ENTRY[*].definition__field#{schema}",
                 selected=True,
             ),
             Column(
                 title="start_time",
-                quantity=f"data.*.ENTRY[*].start_time__field#{schema}",
+                quantity=f"data.ENTRY[*].start_time__field#{schema}",
                 selected=True,
             ),
             Column(
                 title="title",
-                quantity=f"data.*.ENTRY[*].title__field#{schema}",
+                quantity=f"data.ENTRY[*].title__field#{schema}",
                 selected=True,
             ),
         ],
@@ -161,8 +161,8 @@ def load(self):
                     "autorange": True,
                     "nbins": 30,
                     "scale": "linear",
-                    "quantity": f"data.Root.datetime#{schema}",
-                    "title": "Procesing Time",
+                    "quantity": f"data.ENTRY.start_time__field#{schema}",
+                    "title": "Start Time",
                     "layout": {
                         "lg": {"minH": 3, "minW": 3, "h": 4, "w": 12, "y": 0, "x": 0}
                     },
@@ -177,6 +177,16 @@ def load(self):
                         "lg": {"minH": 3, "minW": 3, "h": 8, "w": 4, "y": 0, "x": 12}
                     },
                 },
+                {
+                    "type": "terms",
+                    "show_input": False,
+                    "scale": "linear",
+                    "quantity": f"data.ENTRY.definition__field#{schema}",
+                    "title": "Definition",
+                    "layout": {
+                        "lg": {"minH": 3, "minW": 3, "h": 8, "w": 4, "y": 0, "x": 16}
+                    },
+                },
                 {
                     "type": "periodic_table",
                     "scale": "linear",

src/pynxtools/nomad/parser.py

@@ -40,8 +40,9 @@
 
 import pynxtools.nomad.schema as nexus_schema
 from pynxtools.nexus.nexus import HandleNexus
-from pynxtools.nomad.utils import __REPLACEMENT_FOR_NX
+from pynxtools.nomad.utils import __FIELD_STATISTICS, __REPLACEMENT_FOR_NX
 from pynxtools.nomad.utils import __rename_nx_for_nomad as rename_nx_for_nomad
+from pynxtools.nomad.utils import get_quantity_base_name
 
 
 def _to_group_name(nx_node: ET.Element):
@@ -60,6 +61,7 @@ def _to_section(
     nx_def: str,
     nx_node: Optional[ET.Element],
     current: MSection,
+    nx_root,
 ) -> MSection:
     """
     Args:
@@ -95,17 +97,25 @@ def _to_section(
 
     nomad_def_name = rename_nx_for_nomad(nomad_def_name, is_group=True)
 
-    # for groups, get the definition from the package
-    new_def = current.m_def.all_sub_sections[nomad_def_name]
-
-    new_section: MSection = None  # type:ignore
-
-    for section in current.m_get_sub_sections(new_def):
-        if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
-            new_section = section
-            break
-
-    if new_section is None:
+    if current == nx_root:
+        # for groups, get the definition from the package
+        new_def = current.m_def.all_sub_sections["ENTRY"]
+        for section in current.m_get_sub_sections(new_def):
+            if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
+                return section
+        cls = getattr(nexus_schema, nx_def, None)
+        sec = cls()
+        new_def_spec = sec.m_def.all_sub_sections[nomad_def_name]
+        sec.m_create(new_def_spec.section_def.section_cls)
+        new_section = sec.m_get_sub_section(new_def_spec, -1)
+        current.ENTRY.append(new_section)
+        new_section.__dict__["nx_name"] = hdf_name
+    else:
+        # for groups, get the definition from the package
+        new_def = current.m_def.all_sub_sections[nomad_def_name]
+        for section in current.m_get_sub_sections(new_def):
+            if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
+                return section
         current.m_create(new_def.section_def.section_cls)
         new_section = current.m_get_sub_section(new_def, -1)
         new_section.__dict__["nx_name"] = hdf_name
@@ -120,11 +130,13 @@ def _get_value(hdf_node):
 
     hdf_value = hdf_node[...]
     if str(hdf_value.dtype) == "bool":
+        if len(hdf_value.shape) > 0:
+            return bool(hdf_value.tolist()[0])
         return bool(hdf_value)
     if hdf_value.dtype.kind in "iufc":
         return hdf_value
     if len(hdf_value.shape) > 0:
-        return hdf_value.astype(str)
+        return str([i for i in hdf_value.astype(str)])
     return hdf_node[()].decode()
 
 
@@ -188,17 +200,28 @@ def _populate_data(
                 attr_value = hdf_node.attrs[attr_name]
                 if not isinstance(attr_value, str):
                     if isinstance(attr_value, np.ndarray):
-                        attr_value = attr_value.tolist()
-                        if len(attr_value) == 1:
-                            attr_value = attr_value[0]
-                        # so values of non-scalar attribute will not end up in metainfo!
+                        attr_list = attr_value.tolist()
+                        if len(attr_list) == 1 or attr_value.dtype.kind in "iufc":
+                            attr_value = attr_list[0]
+                        else:
+                            attr_value = str(attr_list)
 
-                attr_name = attr_name + "__attribute"
-                current = _to_section(attr_name, nx_def, nx_attr, current)
+                current = _to_section(attr_name, nx_def, nx_attr, current, self.nx_root)
 
+                attribute = attr_value
+                # TODO: get unit from attribute <xxx>_units
                 try:
                     if nx_root or nx_parent.tag.endswith("group"):
-                        current.m_set_section_attribute(attr_name, attr_value)
+                        attribute_name = "___" + attr_name
+                        metainfo_def = resolve_variadic_name(
+                            current.m_def.all_properties, attribute_name
+                        )
+                        if metainfo_def.use_full_storage:
+                            attribute = MQuantity.wrap(attribute, attribute_name)
+                        current.m_set(metainfo_def, attribute)
+                        # if attributes are set before setting the quantity, a bug can cause them being set under a wrong variadic name
+                        attribute.m_set_attribute("m_nx_data_path", hdf_node.name)
+                        attribute.m_set_attribute("m_nx_data_file", self.nxs_fname)
                     else:
                         parent_html_name = nx_path[-2].get("name")
 
@@ -207,25 +230,26 @@ def _populate_data(
 
                         metainfo_def = None
                         try:
+                            attribute_name = parent_html_name + "___" + attr_name
                             metainfo_def = resolve_variadic_name(
-                                current.m_def.all_properties, parent_field_name
+                                current.m_def.all_properties, attribute_name
                             )
+                            data_instance_name = (
+                                hdf_node.name.split("/")[-1] + "___" + attr_name
+                            )
+                            if metainfo_def.use_full_storage:
+                                attribute = MQuantity.wrap(
+                                    attribute, data_instance_name
+                                )
                         except ValueError as exc:
                             self._logger.warning(
-                                f"{current.m_def} has no suitable property for {parent_field_name}",
+                                f"{current.m_def} has no suitable property for {parent_field_name} and {attr_name} as {attribute_name}",
                                 target_name=attr_name,
                                 exc_info=exc,
                             )
-                        if parent_field_name in current.__dict__:
-                            quantity = current.__dict__[parent_field_name]
-                            if isinstance(quantity, dict):
-                                quantity = quantity[parent_instance_name]
-                        else:
-                            quantity = None
-                            raise Warning(
-                                "setting attribute attempt before creating quantity"
-                            )
-                        quantity.m_set_attribute(attr_name, attr_value)
+                        current.m_set(metainfo_def, attribute)
+                        attribute.m_set_attribute("m_nx_data_path", hdf_node.name)
+                        attribute.m_set_attribute("m_nx_data_file", self.nxs_fname)
                 except Exception as e:
                     self._logger.warning(
                         "error while setting attribute",
@@ -242,21 +266,21 @@ def _populate_data(
             metainfo_def = resolve_variadic_name(
                 current.m_def.all_properties, field_name
             )
+            isvariadic = any(char.isupper() for char in metainfo_def.more["nx_name"])
 
             # for data arrays only statistics if not all values NINF, Inf, or NaN
             field_stats = None
             if hdf_node[...].dtype.kind in "iufc":
                 if isinstance(field, np.ndarray) and field.size > 1:
                     mask = np.isfinite(field)
                     if np.any(mask):
-                        field_stats = np.array(
-                            [
-                                np.mean(field[mask]),
-                                np.var(field[mask]),
-                                np.min(field[mask]),
-                                np.max(field[mask]),
-                            ]
-                        )
+                        field_stats = [
+                            func(field[mask] if ismask else field)
+                            for func, ismask in zip(
+                                __FIELD_STATISTICS["function"],
+                                __FIELD_STATISTICS["mask"],
+                            )
+                        ]
                         field = field_stats[0]
                         if not np.isfinite(field):
                             self._logger.warning(
@@ -285,6 +309,9 @@ def _populate_data(
                     else:
                         pint_unit = ureg.parse_units("1")
                     field = ureg.Quantity(field, pint_unit)
+                    if field_stats is not None:
+                        for i in range(4):
+                            field_stats[i] = ureg.Quantity(field_stats[i], pint_unit)
 
                 except (ValueError, UndefinedUnitError):
                     pass
@@ -299,14 +326,30 @@ def _populate_data(
                 current.m_set(metainfo_def, field)
                 field.m_set_attribute("m_nx_data_path", hdf_node.name)
                 field.m_set_attribute("m_nx_data_file", self.nxs_fname)
+                if isvariadic:
+                    concept_basename = get_quantity_base_name(field.name)
+                    instancename = get_quantity_base_name(data_instance_name)
+                    name_metainfo_def = resolve_variadic_name(
+                        current.m_def.all_properties, concept_basename + "__name"
+                    )
+                    name_value = MQuantity.wrap(instancename, instancename + "__name")
+                    current.m_set(name_metainfo_def, name_value)
+                    name_value.m_set_attribute("m_nx_data_path", hdf_node.name)
+                    name_value.m_set_attribute("m_nx_data_file", self.nxs_fname)
                 if field_stats is not None:
-                    # TODO _add_additional_attributes function has created these nx_data_*
-                    # attributes speculatively already so if the field_stats is None
-                    # this will cause unpopulated attributes in the GUI
-                    field.m_set_attribute("nx_data_mean", field_stats[0])
-                    field.m_set_attribute("nx_data_var", field_stats[1])
-                    field.m_set_attribute("nx_data_min", field_stats[2])
-                    field.m_set_attribute("nx_data_max", field_stats[3])
+                    concept_basename = get_quantity_base_name(field.name)
+                    instancename = get_quantity_base_name(data_instance_name)
+                    for suffix, stat in zip(
+                        __FIELD_STATISTICS["suffix"][1:],
+                        field_stats[1:],
+                    ):
+                        stat_metainfo_def = resolve_variadic_name(
+                            current.m_def.all_properties, concept_basename + suffix
+                        )
+                        stat = MQuantity.wrap(stat, instancename + suffix)
+                        current.m_set(stat_metainfo_def, stat)
+                        stat.m_set_attribute("m_nx_data_path", hdf_node.name)
+                        stat.m_set_attribute("m_nx_data_file", self.nxs_fname)
             except Exception as e:
                 self._logger.warning(
                     "error while setting field",
@@ -332,12 +375,13 @@ def __nexus_populate(self, params: dict, attr=None):  # pylint: disable=W0613
         if nx_path is None or nx_path == "/":
             return
 
-        current: MSection = _to_section(None, nx_def, None, self.nx_root)
+        # current: MSection = _to_section(None, nx_def, None, self.nx_root)
+        current = self.nx_root
         depth: int = 1
         current_hdf_path = ""
         for name in hdf_path.split("/")[1:]:
             nx_node = nx_path[depth] if depth < len(nx_path) else name
-            current = _to_section(name, nx_def, nx_node, current)
+            current = _to_section(name, nx_def, nx_node, current, self.nx_root)
             self._collect_class(current)
             depth += 1
             if depth < len(nx_path):
@@ -468,13 +512,15 @@ def parse(
         child_archives: Dict[str, EntryArchive] = None,
     ) -> None:
         self.archive = archive
-        self.nx_root = nexus_schema.NeXus()  # type: ignore # pylint: disable=no-member
+        self.nx_root = nexus_schema.Root()  # type: ignore # pylint: disable=no-member
 
         self.archive.data = self.nx_root
         self._logger = logger if logger else get_logger(__name__)
         self._clear_class_refs()
 
-        *_, self.nxs_fname = mainfile.rsplit("/", 1)
+        # if filename does not follow the pattern
+        # .volumes/fs/<upload type>/<upload 2char>/<upoad>/<raw/arch>/[subdirs?]/<filename>
+        self.nxs_fname = "/".join(mainfile.split("/")[6:]) or mainfile
         nexus_helper = HandleNexus(logger, mainfile)
         nexus_helper.process_nexus_master_file(self.__nexus_populate)
 
@@ -483,25 +529,18 @@ def parse(
             archive.metadata = EntryMetadata()
 
         # Normalise experiment type
-        app_defs = str(self.nx_root).split("(")[1].split(")")[0].split(",")
-        app_def_list = []
-        for app_elem in app_defs:
-            app = app_elem.lstrip()
-            try:
-                app_sec = getattr(self.nx_root, app)
+        # app_defs = str(self.nx_root).split("(")[1].split(")")[0].split(",")
+        app_def_list = set()
+        try:
+            app_entries = getattr(self.nx_root, "ENTRY")
+            for entry in app_entries:
                 try:
-                    app_entry = getattr(app_sec, "ENTRY")
-                    if len(app_entry) < 1:
-                        raise AttributeError()
+                    app = entry.definition__field
+                    app_def_list.add(rename_nx_for_nomad(app) if app else "Generic")
                 except (AttributeError, TypeError):
-                    app_entry = getattr(app_sec, "entry")
-                    if len(app_entry) < 1:
-                        raise AttributeError()
-                app_def_list.append(
-                    app if app != rename_nx_for_nomad("NXroot") else "Generic"
-                )
-            except (AttributeError, TypeError):
-                pass
+                    pass
+        except (AttributeError, TypeError):
+            pass
         if len(app_def_list) == 0:
             app_def = "Experiment"
         else: