diff --git a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedBackendConfiguration.js b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedBackendConfiguration.js index 0abb616f8..035e576f3 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedBackendConfiguration.js +++ b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedBackendConfiguration.js @@ -19,8 +19,6 @@ import { getResourceUsageBytes } from "../../../../validation/backend-configurat import { resolveBackendResults, updateSchema } from "../../../../../../../schemas/backend-configuration.schema"; import { getInfoFromId } from "./utils.js"; -const getBackendConfigurations = (info, options = {}) => run(`neuroconv/configuration`, info, options); - const itemIgnore = { full_shape: true, buffer_shape: true, @@ -39,6 +37,12 @@ export class GuidedBackendConfigurationPage extends ManagedPage { this.style.height = "100%"; // Fix main section } + getBackendConfigurations = (info, options = {}) => + run(`neuroconv/configuration`, info, options).catch((e) => { + this.notify(e.message, "error"); + throw e; + }); + beforeSave = () => { merge(this.localState, this.info.globalState); }; @@ -227,13 +231,18 @@ export class GuidedBackendConfigurationPage extends ManagedPage { { title: "Getting backend options", }, - getBackendConfigurations + this.getBackendConfigurations ); }; validate = (toRun) => { if (!toRun) - return this.runConversions({}, true, { title: "Validating backend options" }, getBackendConfigurations); + return this.runConversions( + {}, + true, + { title: "Validating backend options" }, + this.getBackendConfigurations + ); const { subject, session } = toRun; return this.runConversions( @@ -243,7 +252,7 @@ export class GuidedBackendConfigurationPage extends ManagedPage { title: "Validating backend options", showCancelButton: false, }, - getBackendConfigurations + this.getBackendConfigurations ); }; diff --git a/src/pyflask/app.py b/src/pyflask/app.py index 731c196bd..3301c21a8 100644 --- a/src/pyflask/app.py +++ b/src/pyflask/app.py @@ -115,6 +115,7 @@ def post(self): message += f"\n{traceback}\n" selected_logger = getattr(api.logger, type) + api.logger.info(f"Logging {type} message: {header}") selected_logger(message) diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index 475d69e3c..0c97ad3ed 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -827,58 +827,27 @@ def get_interface_alignment(info: dict) -> dict: ) -def configure_dataset_backends(nwbfile, backend_configuration, configuration=None): - from neuroconv.tools.nwb_helpers import ( - configure_backend, - get_default_backend_configuration, - ) - - PROPS_TO_AVOID = ["full_shape"] - - # Default to HDF5 backend configuration - if configuration is None: - configuration = get_default_backend_configuration(nwbfile=nwbfile, backend="hdf5") - - # Ensure the configuration is a dictionary - elif isinstance(configuration, str): - configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=configuration) - - for name, item in backend_configuration.items(): - for key, value in item.items(): - - # Avoid setting compression options if unspecified - if key == "compression_options" and (value is None or len(value) == 0): - setattr(configuration.dataset_configurations[name], key, None) - - # Avoid certain properties passed to the GUIDE - elif key not in PROPS_TO_AVOID: - setattr(configuration.dataset_configurations[name], key, value) - - configure_backend(nwbfile=nwbfile, backend_configuration=configuration) - - def create_file( info: dict, log_url: Optional[str] = None, ) -> dict: import requests - from neuroconv.tools.nwb_helpers import ( - get_default_backend_configuration, - make_or_load_nwbfile, - ) from tqdm_publisher import TQDMProgressSubscriber project_name = info.get("project_name") run_stub_test = info.get("stub_test", False) - backend_configuration = info.get("configuration") + overwrite = info.get("overwrite", False) + # Progress update info url = info.get("url") request_id = info.get("request_id") - will_configure_backend = backend_configuration is not None and run_stub_test is False + # Backend configuration info + backend_configuration = info.get("configuration") + backend = backend_configuration.get("backend", "hdf5") converter, metadata, path_info = get_conversion_info(info) @@ -894,60 +863,50 @@ def create_file( else: nwbfile_path.unlink() - if will_configure_backend: - - backend = backend_configuration.get("backend", "hdf5") - configuration_values = backend_configuration.get("results", {}).get(backend, {}) + def update_conversion_progress(message): + update_dict = dict(request_id=request_id, **message) + if url or not run_stub_test: + requests.post(url=url, json=update_dict) + else: + progress_handler.announce(update_dict) - # Create NWB file with appropriate backend configuration - with make_or_load_nwbfile( - nwbfile_path=nwbfile_path, metadata=metadata, overwrite=overwrite, backend=backend - ) as nwbfile: - converter.add_to_nwbfile(nwbfile, metadata=metadata) - configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend) - configure_dataset_backends(nwbfile, configuration_values, configuration) + progress_bar_options = dict( + mininterval=0, + on_progress_update=update_conversion_progress, + ) - else: + # Assume all interfaces have the same conversion options for now + available_options = converter.get_conversion_options_schema() + options = {interface: {} for interface in info["source_data"]} - def update_conversion_progress(message): - update_dict = dict(request_id=request_id, **message) - if url or not run_stub_test: - requests.post(url=url, json=update_dict) - else: - progress_handler.announce(update_dict) + for interface in options: + available_opts = available_options.get("properties").get(interface).get("properties", {}) - progress_bar_options = dict( - mininterval=0, - on_progress_update=update_conversion_progress, - ) + # Specify if stub test + if run_stub_test: + if available_opts.get("stub_test"): + options[interface]["stub_test"] = True - # Assume all interfaces have the same conversion options for now - available_options = converter.get_conversion_options_schema() - options = {interface: {} for interface in info["source_data"]} - - for interface in options: - available_opts = available_options.get("properties").get(interface).get("properties", {}) + # Specify if iterator options are available + elif available_opts.get("iterator_opts"): + options[interface]["iterator_opts"] = dict( + display_progress=True, + progress_bar_class=TQDMProgressSubscriber, + progress_bar_options=progress_bar_options, + ) - # Specify if stub test - if run_stub_test: - if available_opts.get("stub_test"): - options[interface]["stub_test"] = True + run_conversion_kwargs = dict( + metadata=metadata, + nwbfile_path=nwbfile_path, + overwrite=overwrite, + conversion_options=options, + backend=backend, + ) - # Specify if iterator options are available - elif available_opts.get("iterator_opts"): - options[interface]["iterator_opts"] = dict( - display_progress=True, - progress_bar_class=TQDMProgressSubscriber, - progress_bar_options=progress_bar_options, - ) + if not run_stub_test: + run_conversion_kwargs.update(dict(backend_configuration=update_backend_configuration(info))) - # Actually run the conversion - converter.run_conversion( - metadata=metadata, - nwbfile_path=nwbfile_path, - overwrite=overwrite, - conversion_options=options, - ) + converter.run_conversion(**run_conversion_kwargs) except Exception as e: if log_url: @@ -964,42 +923,51 @@ def update_conversion_progress(message): raise e -def get_backend_configuration(info: dict) -> dict: - - import numpy as np - - PROPS_TO_REMOVE = [ - # Immutable - "object_id", - "dataset_name", - "location_in_file", - "dtype", - ] - - PROPS_TO_IGNORE = ["full_shape"] - - info["overwrite"] = True # Always overwrite the file +def update_backend_configuration(info: dict) -> dict: from neuroconv.tools.nwb_helpers import ( get_default_backend_configuration, make_nwbfile_from_metadata, ) - backend_configuration = info.get("configuration", {}) - backend = backend_configuration.get("backend", "hdf5") - results = backend_configuration.get("results", {}).get(backend, {}) + PROPS_TO_IGNORE = ["full_shape"] - # raise ValueError(f"This function is not currently supported. {results}") + info_from_frontend = info.get("configuration", {}) + backend = info_from_frontend.get("backend", "hdf5") + backend_configuration_from_frontend = info_from_frontend.get("results", {}).get(backend, {}) converter, metadata, __ = get_conversion_info(info) nwbfile = make_nwbfile_from_metadata(metadata=metadata) converter.add_to_nwbfile(nwbfile, metadata=metadata) - configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend) - for dataset_name, dataset_configuration in results.items(): + + backend_configuration = get_default_backend_configuration(nwbfile=nwbfile, backend=backend) + + for location_in_file, dataset_configuration in backend_configuration_from_frontend.items(): for key, value in dataset_configuration.items(): if key not in PROPS_TO_IGNORE: - setattr(configuration.dataset_configurations[dataset_name], key, value) + # Pydantic models only allow setting of attributes + setattr(backend_configuration.dataset_configurations[location_in_file], key, value) + + return backend_configuration + + +def get_backend_configuration(info: dict) -> dict: + + import numpy as np + + PROPS_TO_REMOVE = [ + # Immutable + "object_id", + "dataset_name", + "location_in_file", + "dtype", + ] + + info["overwrite"] = True # Always overwrite the file + + backend = info.get("backend", "hdf5") + configuration = update_backend_configuration(info) def custom_encoder(obj): if isinstance(obj, np.ndarray):