No inner sections

src/pynxtools/nomad/entrypoints.py

@@ -63,7 +63,7 @@ def load(self):
     name="pynxtools parser",
     description="A parser for nexus files.",
     mainfile_name_re=r".*\.nxs",
-    mainfile_mime_re="application/x-hdf5",
+    mainfile_mime_re="application/x-hdf*",
 )
 
 from nomad.config.models.ui import (
@@ -76,7 +76,7 @@ def load(self):
     SearchQuantities,
 )
 
-schema = "pynxtools.nomad.schema.NeXus"
+schema = "pynxtools.nomad.schema.Root"
 
 nexus_app = AppEntryPoint(
     name="NexusApp",
@@ -105,17 +105,17 @@ def load(self):
             Column(quantity=f"entry_type", selected=True),
             Column(
                 title="definition",
-                quantity=f"data.*.ENTRY[*].definition__field#{schema}",
+                quantity=f"data.ENTRY[*].definition__field#{schema}",
                 selected=True,
             ),
             Column(
                 title="start_time",
-                quantity=f"data.*.ENTRY[*].start_time__field#{schema}",
+                quantity=f"data.ENTRY[*].start_time__field#{schema}",
                 selected=True,
             ),
             Column(
                 title="title",
-                quantity=f"data.*.ENTRY[*].title__field#{schema}",
+                quantity=f"data.ENTRY[*].title__field#{schema}",
                 selected=True,
             ),
         ],
@@ -161,8 +161,8 @@ def load(self):
                     "autorange": True,
                     "nbins": 30,
                     "scale": "linear",
-                    "quantity": f"data.Root.datetime#{schema}",
-                    "title": "Procesing Time",
+                    "quantity": f"data.ENTRY.start_time__field#{schema}",
+                    "title": "Start Time",
                     "layout": {
                         "lg": {"minH": 3, "minW": 3, "h": 4, "w": 12, "y": 0, "x": 0}
                     },
@@ -177,6 +177,16 @@ def load(self):
                         "lg": {"minH": 3, "minW": 3, "h": 8, "w": 4, "y": 0, "x": 12}
                     },
                 },
+                {
+                    "type": "terms",
+                    "show_input": False,
+                    "scale": "linear",
+                    "quantity": f"data.ENTRY.definition__field#{schema}",
+                    "title": "Definition",
+                    "layout": {
+                        "lg": {"minH": 3, "minW": 3, "h": 8, "w": 4, "y": 0, "x": 16}
+                    },
+                },
                 {
                     "type": "periodic_table",
                     "scale": "linear",

src/pynxtools/nomad/parser.py

@@ -60,6 +60,7 @@ def _to_section(
     nx_def: str,
     nx_node: Optional[ET.Element],
     current: MSection,
+    nx_root,
 ) -> MSection:
     """
     Args:
@@ -95,17 +96,25 @@ def _to_section(
 
     nomad_def_name = rename_nx_for_nomad(nomad_def_name, is_group=True)
 
-    # for groups, get the definition from the package
-    new_def = current.m_def.all_sub_sections[nomad_def_name]
-
-    new_section: MSection = None  # type:ignore
-
-    for section in current.m_get_sub_sections(new_def):
-        if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
-            new_section = section
-            break
-
-    if new_section is None:
+    if current == nx_root:
+        # for groups, get the definition from the package
+        new_def = current.m_def.all_sub_sections["ENTRY"]
+        for section in current.m_get_sub_sections(new_def):
+            if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
+                return section
+        cls = getattr(nexus_schema, nx_def, None)
+        sec = cls()
+        new_def_spec = sec.m_def.all_sub_sections[nomad_def_name]
+        sec.m_create(new_def_spec.section_def.section_cls)
+        new_section = sec.m_get_sub_section(new_def_spec, -1)
+        current.ENTRY.append(new_section)
+        new_section.__dict__["nx_name"] = hdf_name
+    else:
+        # for groups, get the definition from the package
+        new_def = current.m_def.all_sub_sections[nomad_def_name]
+        for section in current.m_get_sub_sections(new_def):
+            if hdf_name is None or getattr(section, "nx_name", None) == hdf_name:
+                return section
         current.m_create(new_def.section_def.section_cls)
         new_section = current.m_get_sub_section(new_def, -1)
         new_section.__dict__["nx_name"] = hdf_name
@@ -193,12 +202,22 @@ def _populate_data(
                             attr_value = attr_value[0]
                         # so values of non-scalar attribute will not end up in metainfo!
 
-                attr_name = attr_name + "__attribute"
-                current = _to_section(attr_name, nx_def, nx_attr, current)
+                current = _to_section(attr_name, nx_def, nx_attr, current, self.nx_root)
 
+                attribute = attr_value
+                # TODO: get unit from attribute <xxx>_units
                 try:
                     if nx_root or nx_parent.tag.endswith("group"):
-                        current.m_set_section_attribute(attr_name, attr_value)
+                        attribute_name = "___" + attr_name
+                        metainfo_def = resolve_variadic_name(
+                            current.m_def.all_properties, attribute_name
+                        )
+                        if metainfo_def.use_full_storage:
+                            attribute = MQuantity.wrap(attribute, attribute_name)
+                        current.m_set(metainfo_def, attribute)
+                        # if attributes are set before setting the quantity, a bug can cause them being set under a wrong variadic name
+                        attribute.m_set_attribute("m_nx_data_path", hdf_node.name)
+                        attribute.m_set_attribute("m_nx_data_file", self.nxs_fname)
                     else:
                         parent_html_name = nx_path[-2].get("name")
 
@@ -207,25 +226,26 @@ def _populate_data(
 
                         metainfo_def = None
                         try:
+                            attribute_name = parent_html_name + "___" + attr_name
                             metainfo_def = resolve_variadic_name(
-                                current.m_def.all_properties, parent_field_name
+                                current.m_def.all_properties, attribute_name
+                            )
+                            data_instance_name = (
+                                hdf_node.name.split("/")[-1] + "___" + attr_name
                             )
+                            if metainfo_def.use_full_storage:
+                                attribute = MQuantity.wrap(
+                                    attribute, data_instance_name
+                                )
                         except ValueError as exc:
                             self._logger.warning(
-                                f"{current.m_def} has no suitable property for {parent_field_name}",
+                                f"{current.m_def} has no suitable property for {parent_field_name} and {attr_name} as {attribute_name}",
                                 target_name=attr_name,
                                 exc_info=exc,
                             )
-                        if parent_field_name in current.__dict__:
-                            quantity = current.__dict__[parent_field_name]
-                            if isinstance(quantity, dict):
-                                quantity = quantity[parent_instance_name]
-                        else:
-                            quantity = None
-                            raise Warning(
-                                "setting attribute attempt before creating quantity"
-                            )
-                        quantity.m_set_attribute(attr_name, attr_value)
+                        current.m_set(metainfo_def, attribute)
+                        attribute.m_set_attribute("m_nx_data_path", hdf_node.name)
+                        attribute.m_set_attribute("m_nx_data_file", self.nxs_fname)
                 except Exception as e:
                     self._logger.warning(
                         "error while setting attribute",
@@ -332,12 +352,13 @@ def __nexus_populate(self, params: dict, attr=None):  # pylint: disable=W0613
         if nx_path is None or nx_path == "/":
             return
 
-        current: MSection = _to_section(None, nx_def, None, self.nx_root)
+        # current: MSection = _to_section(None, nx_def, None, self.nx_root)
+        current = self.nx_root
         depth: int = 1
         current_hdf_path = ""
         for name in hdf_path.split("/")[1:]:
             nx_node = nx_path[depth] if depth < len(nx_path) else name
-            current = _to_section(name, nx_def, nx_node, current)
+            current = _to_section(name, nx_def, nx_node, current, self.nx_root)
             self._collect_class(current)
             depth += 1
             if depth < len(nx_path):
@@ -468,13 +489,19 @@ def parse(
         child_archives: Dict[str, EntryArchive] = None,
     ) -> None:
         self.archive = archive
-        self.nx_root = nexus_schema.NeXus()  # type: ignore # pylint: disable=no-member
+        self.nx_root = nexus_schema.Root()  # type: ignore # pylint: disable=no-member
 
         self.archive.data = self.nx_root
         self._logger = logger if logger else get_logger(__name__)
         self._clear_class_refs()
 
-        *_, self.nxs_fname = mainfile.rsplit("/", 1)
+        mf = mainfile.split("/")
+        # if filename does not follow the pattern
+        # .volumes/fs/<upload type>/<upload 2char>/<upoad>/<raw/arch>/[subdirs?]/<filename>
+        if len(mf) < 7:
+            self.nxs_fname = mainfile
+        else:
+            self.nxs_fname = "/".join(mf[6:])
         nexus_helper = HandleNexus(logger, mainfile)
         nexus_helper.process_nexus_master_file(self.__nexus_populate)
 
@@ -483,25 +510,18 @@ def parse(
             archive.metadata = EntryMetadata()
 
         # Normalise experiment type
-        app_defs = str(self.nx_root).split("(")[1].split(")")[0].split(",")
-        app_def_list = []
-        for app_elem in app_defs:
-            app = app_elem.lstrip()
-            try:
-                app_sec = getattr(self.nx_root, app)
+        # app_defs = str(self.nx_root).split("(")[1].split(")")[0].split(",")
+        app_def_list = set()
+        try:
+            app_entries = getattr(self.nx_root, "ENTRY")
+            for entry in app_entries:
                 try:
-                    app_entry = getattr(app_sec, "ENTRY")
-                    if len(app_entry) < 1:
-                        raise AttributeError()
+                    app = entry.definition__field
+                    app_def_list.add(rename_nx_for_nomad(app) if app else "Generic")
                 except (AttributeError, TypeError):
-                    app_entry = getattr(app_sec, "entry")
-                    if len(app_entry) < 1:
-                        raise AttributeError()
-                app_def_list.append(
-                    app if app != rename_nx_for_nomad("NXroot") else "Generic"
-                )
-            except (AttributeError, TypeError):
-                pass
+                    pass
+        except (AttributeError, TypeError):
+            pass
         if len(app_def_list) == 0:
             app_def = "Experiment"
         else: