Uniform Conversion with Backend Configuration

src/electron/frontend/core/components/pages/guided-mode/data/GuidedBackendConfiguration.js

@@ -19,8 +19,6 @@ import { getResourceUsageBytes } from "../../../../validation/backend-configurat
 import { resolveBackendResults, updateSchema } from "../../../../../../../schemas/backend-configuration.schema";
 import { getInfoFromId } from "./utils.js";
 
-const getBackendConfigurations = (info, options = {}) => run(`neuroconv/configuration`, info, options);
-
 const itemIgnore = {
     full_shape: true,
     buffer_shape: true,
@@ -39,6 +37,12 @@ export class GuidedBackendConfigurationPage extends ManagedPage {
         this.style.height = "100%"; // Fix main section
     }
 
+    getBackendConfigurations = (info, options = {}) =>
+        run(`neuroconv/configuration`, info, options).catch((e) => {
+            this.notify(e.message, "error");
+            throw e;
+        });
+
     beforeSave = () => {
         merge(this.localState, this.info.globalState);
     };
@@ -227,13 +231,18 @@ export class GuidedBackendConfigurationPage extends ManagedPage {
             {
                 title: "Getting backend options",
             },
-            getBackendConfigurations
+            this.getBackendConfigurations
         );
     };
 
     validate = (toRun) => {
         if (!toRun)
-            return this.runConversions({}, true, { title: "Validating backend options" }, getBackendConfigurations);
+            return this.runConversions(
+                {},
+                true,
+                { title: "Validating backend options" },
+                this.getBackendConfigurations
+            );
 
         const { subject, session } = toRun;
         return this.runConversions(
@@ -243,7 +252,7 @@ export class GuidedBackendConfigurationPage extends ManagedPage {
                 title: "Validating backend options",
                 showCancelButton: false,
             },
-            getBackendConfigurations
+            this.getBackendConfigurations
         );
     };
 

src/pyflask/app.py

@@ -115,6 +115,7 @@ def post(self):
             message += f"\n{traceback}\n"
 
         selected_logger = getattr(api.logger, type)
+        api.logger.info(f"Logging {type} message: {header}")
         selected_logger(message)
 
 

src/pyflask/manageNeuroconv/manage_neuroconv.py

@@ -827,58 +827,27 @@ def get_interface_alignment(info: dict) -> dict:
     )
 
 
-def configure_dataset_backends(nwbfile, backend_configuration, configuration=None):
-    from neuroconv.tools.nwb_helpers import (
-        configure_backend,
-        get_default_backend_configuration,
-    )
-
-    PROPS_TO_AVOID = ["full_shape"]
-
-    # Default to HDF5 backend configuration
-    if configuration is None:
-        configuration = get_default_backend_configuration(nwbfile=nwbfile, backend="hdf5")
-
-    # Ensure the configuration is a dictionary
-    elif isinstance(configuration, str):
-        configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=configuration)
-
-    for name, item in backend_configuration.items():
-        for key, value in item.items():
-
-            # Avoid setting compression options if unspecified
-            if key == "compression_options" and (value is None or len(value) == 0):
-                setattr(configuration.dataset_configurations[name], key, None)
-
-            # Avoid certain properties passed to the GUIDE
-            elif key not in PROPS_TO_AVOID:
-                setattr(configuration.dataset_configurations[name], key, value)
-
-    configure_backend(nwbfile=nwbfile, backend_configuration=configuration)
-
-
 def create_file(
     info: dict,
     log_url: Optional[str] = None,
 ) -> dict:
 
     import requests
-    from neuroconv.tools.nwb_helpers import (
-        get_default_backend_configuration,
-        make_or_load_nwbfile,
-    )
     from tqdm_publisher import TQDMProgressSubscriber
 
     project_name = info.get("project_name")
 
     run_stub_test = info.get("stub_test", False)
-    backend_configuration = info.get("configuration")
+
     overwrite = info.get("overwrite", False)
 
+    # Progress update info
     url = info.get("url")
     request_id = info.get("request_id")
 
-    will_configure_backend = backend_configuration is not None and run_stub_test is False
+    # Backend configuration info
+    backend_configuration = info.get("configuration")
+    backend = backend_configuration.get("backend", "hdf5")
 
     converter, metadata, path_info = get_conversion_info(info)
 
@@ -894,60 +863,50 @@ def create_file(
                 else:
                     nwbfile_path.unlink()
 
-        if will_configure_backend:
-
-            backend = backend_configuration.get("backend", "hdf5")
-            configuration_values = backend_configuration.get("results", {}).get(backend, {})
+        def update_conversion_progress(message):
+            update_dict = dict(request_id=request_id, **message)
+            if url or not run_stub_test:
+                requests.post(url=url, json=update_dict)
+            else:
+                progress_handler.announce(update_dict)
 
-            # Create NWB file with appropriate backend configuration
-            with make_or_load_nwbfile(
-                nwbfile_path=nwbfile_path, metadata=metadata, overwrite=overwrite, backend=backend
-            ) as nwbfile:
-                converter.add_to_nwbfile(nwbfile, metadata=metadata)
-                configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
-                configure_dataset_backends(nwbfile, configuration_values, configuration)
+        progress_bar_options = dict(
+            mininterval=0,
+            on_progress_update=update_conversion_progress,
+        )
 
-        else:
+        # Assume all interfaces have the same conversion options for now
+        available_options = converter.get_conversion_options_schema()
+        options = {interface: {} for interface in info["source_data"]}
 
-            def update_conversion_progress(message):
-                update_dict = dict(request_id=request_id, **message)
-                if url or not run_stub_test:
-                    requests.post(url=url, json=update_dict)
-                else:
-                    progress_handler.announce(update_dict)
+        for interface in options:
+            available_opts = available_options.get("properties").get(interface).get("properties", {})
 
-            progress_bar_options = dict(
-                mininterval=0,
-                on_progress_update=update_conversion_progress,
-            )
+            # Specify if stub test
+            if run_stub_test:
+                if available_opts.get("stub_test"):
+                    options[interface]["stub_test"] = True
 
-            # Assume all interfaces have the same conversion options for now
-            available_options = converter.get_conversion_options_schema()
-            options = {interface: {} for interface in info["source_data"]}
-
-            for interface in options:
-                available_opts = available_options.get("properties").get(interface).get("properties", {})
+            # Specify if iterator options are available
+            elif available_opts.get("iterator_opts"):
+                options[interface]["iterator_opts"] = dict(
+                    display_progress=True,
+                    progress_bar_class=TQDMProgressSubscriber,
+                    progress_bar_options=progress_bar_options,
+                )
 
-                # Specify if stub test
-                if run_stub_test:
-                    if available_opts.get("stub_test"):
-                        options[interface]["stub_test"] = True
+        run_conversion_kwargs = dict(
+            metadata=metadata,
+            nwbfile_path=nwbfile_path,
+            overwrite=overwrite,
+            conversion_options=options,
+            backend=backend,
+        )
 
-                # Specify if iterator options are available
-                elif available_opts.get("iterator_opts"):
-                    options[interface]["iterator_opts"] = dict(
-                        display_progress=True,
-                        progress_bar_class=TQDMProgressSubscriber,
-                        progress_bar_options=progress_bar_options,
-                    )
+        if not run_stub_test:
+            run_conversion_kwargs.update(dict(backend_configuration=update_backend_configuration(info)))
 
-            # Actually run the conversion
-            converter.run_conversion(
-                metadata=metadata,
-                nwbfile_path=nwbfile_path,
-                overwrite=overwrite,
-                conversion_options=options,
-            )
+        converter.run_conversion(**run_conversion_kwargs)
 
     except Exception as e:
         if log_url:
@@ -964,42 +923,51 @@ def update_conversion_progress(message):
         raise e
 
 
-def get_backend_configuration(info: dict) -> dict:
-
-    import numpy as np
-
-    PROPS_TO_REMOVE = [
-        # Immutable
-        "object_id",
-        "dataset_name",
-        "location_in_file",
-        "dtype",
-    ]
-
-    PROPS_TO_IGNORE = ["full_shape"]
-
-    info["overwrite"] = True  # Always overwrite the file
+def update_backend_configuration(info: dict) -> dict:
 
     from neuroconv.tools.nwb_helpers import (
         get_default_backend_configuration,
         make_nwbfile_from_metadata,
     )
 
-    backend_configuration = info.get("configuration", {})
-    backend = backend_configuration.get("backend", "hdf5")
-    results = backend_configuration.get("results", {}).get(backend, {})
+    PROPS_TO_IGNORE = ["full_shape"]
 
-    # raise ValueError(f"This function is not currently supported. {results}")
+    info_from_frontend = info.get("configuration", {})
+    backend = info_from_frontend.get("backend", "hdf5")
+    backend_configuration_from_frontend = info_from_frontend.get("results", {}).get(backend, {})
 
     converter, metadata, __ = get_conversion_info(info)
 
     nwbfile = make_nwbfile_from_metadata(metadata=metadata)
     converter.add_to_nwbfile(nwbfile, metadata=metadata)
-    configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
-    for dataset_name, dataset_configuration in results.items():
+
+    backend_configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend)
+
+    for location_in_file, dataset_configuration in backend_configuration_from_frontend.items():
         for key, value in dataset_configuration.items():
             if key not in PROPS_TO_IGNORE:
-                setattr(configuration.dataset_configurations[dataset_name], key, value)
+                # Pydantic models only allow setting of attributes
+                setattr(backend_configuration.dataset_configurations[location_in_file], key, value)
+
+    return backend_configuration
+
+
+def get_backend_configuration(info: dict) -> dict:
+
+    import numpy as np
+
+    PROPS_TO_REMOVE = [
+        # Immutable
+        "object_id",
+        "dataset_name",
+        "location_in_file",
+        "dtype",
+    ]
+
+    info["overwrite"] = True  # Always overwrite the file
+
+    backend = info.get("backend", "hdf5")
+    configuration = update_backend_configuration(info)
 
     def custom_encoder(obj):
         if isinstance(obj, np.ndarray):