From d128c6a21243c82d370c647e9e93d8460fd91a6d Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 13 May 2024 14:38:33 -0700 Subject: [PATCH 01/23] Attempt parallel conversion --- pyflask/apis/neuroconv.py | 8 +- pyflask/manageNeuroconv/__init__.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 80 +++++++++++++- src/renderer/src/stories/pages/Page.js | 110 ++++++++++++++------ 4 files changed, 166 insertions(+), 33 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 1877e763e1..58f02699a0 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -15,6 +15,7 @@ get_source_schema, get_metadata_schema, convert_to_nwb, + convert_all_to_nwb, validate_metadata, listen_to_neuroconv_events, inspect_nwb_file, @@ -110,7 +111,12 @@ class Convert(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - return convert_to_nwb(neuroconv_api.payload) + has_files = "files" in neuroconv_api.payload + if has_files: + url = f"{request.url_root}neuroconv/announce" + return convert_all_to_nwb(url, **neuroconv_api.payload) + else: + return convert_to_nwb(neuroconv_api.payload) except Exception as exception: if notBadRequestException(exception): diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index ddcc2b8b61..fac05f4779 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -6,6 +6,7 @@ get_source_schema, get_metadata_schema, convert_to_nwb, + convert_all_to_nwb, validate_metadata, upload_project_to_dandi, upload_folder_to_dandi, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 54e189b628..9d9576da17 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -124,6 +124,7 @@ def resolve_references(schema, root_schema=None): dict: The resolved JSON schema. """ from jsonschema import RefResolver + if root_schema is None: root_schema = schema @@ -663,10 +664,15 @@ def get_interface_alignment(info: dict) -> dict: return timestamps - def convert_to_nwb(info: dict) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" + from tqdm_publisher import TQDMProgressSubscriber + import requests + + url = info.get("url", None) + request_id = info.get("request_id", None) + nwbfile_path = Path(info["nwbfile_path"]) custom_output_directory = info.get("output_folder") project_name = info.get("project_name") @@ -695,14 +701,27 @@ def convert_to_nwb(info: dict) -> str: converter = instantiate_custom_converter(resolved_source_data, info["interfaces"]) def update_conversion_progress(**kwargs): - announcer.announce(dict(**kwargs, nwbfile_path=nwbfile_path), "conversion_progress") + update_dict = dict(request_id=request_id, **kwargs) + if (url) or not run_stub_test: + requests.post(url=url, json=update_dict) + else: + announcer.announce(update_dict) + + progress_bar_options = dict( + mininterval=0, + on_progress_update=update_conversion_progress, + ) + # Assume all interfaces have the same conversion options for now available_options = converter.get_conversion_options_schema() options = ( { interface: ( - {"stub_test": info["stub_test"]} # , "iter_opts": {"report_hook": update_conversion_progress}} + { + "stub_test": info["stub_test"], + # "iterator_opts": dict( display_progress=True, progress_bar_class=TQDMProgressSubscriber, progress_bar_options=progress_bar_options ) + } if available_options.get("properties").get(interface).get("properties", {}).get("stub_test") else {} ) @@ -788,6 +807,61 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) +def _convert_to_nwb(label, info: dict) -> dict: + return label, convert_to_nwb(info) + +def convert_all_to_nwb( + url: str, + files: List[dict], + request_id: Optional[str], + max_workers: int = 1, +) -> List[str]: + + from tqdm_publisher import TQDMProgressSubscriber + from concurrent.futures import ProcessPoolExecutor, as_completed + + def on_progress_update(message): + message["progress_bar_id"] = request_id # Ensure request_id matches + announcer.announce( + dict( + request_id=request_id, + **message, + ) + ) + + results = {} + futures = [] + with ProcessPoolExecutor(max_workers=max_workers) as executor: + for file_info in files: + + futures.append( + executor.submit( + _convert_to_nwb, + file_info.get('nwbfile_path'), + dict( + url=url, + request_id=request_id, + **file_info, + ), + ) + ) + + inspection_iterable = TQDMProgressSubscriber( + iterable=as_completed(futures), + desc="Total files converted", + total=len(futures), + mininterval=0, + on_progress_update=on_progress_update, + ) + + + for _ in inspection_iterable: + label, result = _.result() + results[label] = result + + return result + + def upload_multiple_filesystem_objects_to_dandi(**kwargs): tmp_folder_path = _aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} diff --git a/src/renderer/src/stories/pages/Page.js b/src/renderer/src/stories/pages/Page.js index 7a9bfaff7a..2878869489 100644 --- a/src/renderer/src/stories/pages/Page.js +++ b/src/renderer/src/stories/pages/Page.js @@ -1,5 +1,5 @@ import { LitElement, html } from "lit"; -import { runConversion } from "./guided-mode/options/utils.js"; +import { run, runConversion } from "./guided-mode/options/utils.js"; import { get, save } from "../../progress/index.js"; import { dismissNotification, isStorybook, notify } from "../../dependencies/globals.js"; import { randomizeElements, mapSessions, merge } from "./utils.js"; @@ -167,6 +167,8 @@ export class Page extends LitElement { let completed = 0; elements.progress.format = { n: completed, total: toRun.length }; + const fileConfiguration = [] + for (let info of toRun) { const { subject, session, globalState = this.info.globalState } = info; const file = `sub-${subject}/sub-${subject}_ses-${session}.nwb`; @@ -184,41 +186,91 @@ export class Page extends LitElement { source_data: merge(SourceData, sourceDataCopy), }; - const result = await runConversion( - { - output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, - project_name: name, - nwbfile_path: file, - overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) - ...sessionInfo, // source_data and metadata are passed in here - ...conversionOptions, // Any additional conversion options override the defaults - - interfaces: globalState.interfaces, - }, - swalOpts - ).catch((error) => { - let message = error.message; - - if (message.includes("The user aborted a request.")) { - this.notify("Conversion was cancelled.", "warning"); - throw error; - } + const payload = { + output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, + project_name: name, + nwbfile_path: file, + overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) + ...sessionInfo, // source_data and metadata are passed in here + ...conversionOptions, // Any additional conversion options override the defaults - this.notify(message, "error"); - closeProgressPopup(); - throw error; - }); + interfaces: globalState.interfaces, + } + + fileConfiguration.push(payload) + + // const result = await runConversion( + // { + // output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, + // project_name: name, + // nwbfile_path: file, + // overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) + // ...sessionInfo, // source_data and metadata are passed in here + // ...conversionOptions, // Any additional conversion options override the defaults + + // interfaces: globalState.interfaces, + // }, + // swalOpts + // ).catch((error) => { + // let message = error.message; + + // if (message.includes("The user aborted a request.")) { + // this.notify("Conversion was cancelled.", "warning"); + // throw error; + // } + + // this.notify(message, "error"); + // closeProgressPopup(); + // throw error; + // }); + + // completed++; + // if (isMultiple) { + // const progressInfo = { n: completed, total: toRun.length }; + // elements.progress.format = progressInfo; + // } + } - completed++; - if (isMultiple) { - const progressInfo = { n: completed, total: toRun.length }; - elements.progress.format = progressInfo; + + const request_id = Math.random().toString(36).substring(7); + + const conversionResults = run(`convert`, { + files: fileConfiguration, + max_workers: 4, + request_id + }, { + title: "Running the conversion", + onError: (results) => { + if (results.message.includes("already exists")) { + return "File already exists. Please specify another location to store the conversion results"; + } else { + return "Conversion failed with current metadata. Please try again."; + } + }, + ...swalOpts, + }) + + .catch((error) => { + let message = error.message; + + if (message.includes("The user aborted a request.")) { + this.notify("Conversion was cancelled.", "warning"); + throw error; } + this.notify(message, "error"); + closeProgressPopup(); + throw error; + }); + + + for (let file in conversionResults) { + const [ subject, session ] = file.match(/sub-(\d+)\/sub-\d+_ses-(\d+)\.nwb/).slice(1); const subRef = results[subject] ?? (results[subject] = {}); - subRef[session] = result; + subRef[session] = conversionResults[file]; } + closeProgressPopup(); elements.container.style.textAlign = ""; // Clear style update From 4c452f9fd4020f613b931de7b2432951e4de461e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 13 May 2024 16:17:36 -0700 Subject: [PATCH 02/23] Add global conversion progress bar --- pyflask/manageNeuroconv/manage_neuroconv.py | 19 +++-- src/renderer/src/stories/pages/Page.js | 79 ++++--------------- .../options/GuidedInspectorPage.js | 6 +- .../src/stories/pages/inspect/InspectPage.js | 6 +- src/renderer/src/stories/pages/utils.js | 3 +- src/renderer/src/stories/utils/progress.js | 12 +-- 6 files changed, 39 insertions(+), 86 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 9d9576da17..0fa8ab8e67 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -807,9 +807,6 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) -def _convert_to_nwb(label, info: dict) -> dict: - return label, convert_to_nwb(info) - def convert_all_to_nwb( url: str, files: List[dict], @@ -829,15 +826,17 @@ def on_progress_update(message): ) ) - results = {} + futures = [] + file_paths = [] + with ProcessPoolExecutor(max_workers=max_workers) as executor: + for file_info in files: futures.append( executor.submit( - _convert_to_nwb, - file_info.get('nwbfile_path'), + convert_to_nwb, dict( url=url, request_id=request_id, @@ -855,11 +854,11 @@ def on_progress_update(message): ) - for _ in inspection_iterable: - label, result = _.result() - results[label] = result + for future in inspection_iterable: + output_filepath = future.result() + file_paths.append(output_filepath) - return result + return file_paths def upload_multiple_filesystem_objects_to_dandi(**kwargs): diff --git a/src/renderer/src/stories/pages/Page.js b/src/renderer/src/stories/pages/Page.js index 2878869489..9d3407f7d0 100644 --- a/src/renderer/src/stories/pages/Page.js +++ b/src/renderer/src/stories/pages/Page.js @@ -1,5 +1,5 @@ import { LitElement, html } from "lit"; -import { run, runConversion } from "./guided-mode/options/utils.js"; +import { run } from "./guided-mode/options/utils.js"; import { get, save } from "../../progress/index.js"; import { dismissNotification, isStorybook, notify } from "../../dependencies/globals.js"; import { randomizeElements, mapSessions, merge } from "./utils.js"; @@ -154,18 +154,8 @@ export class Page extends LitElement { const results = {}; - const isMultiple = toRun.length > 1; - const swalOpts = await createProgressPopup({ title: `Running conversion`, ...options }); - const { close: closeProgressPopup, elements } = swalOpts; - - elements.container.insertAdjacentHTML( - "beforeend", - `Note: This may take a while to complete...
` - ); - - let completed = 0; - elements.progress.format = { n: completed, total: toRun.length }; + const { close: closeProgressPopup } = swalOpts; const fileConfiguration = [] @@ -198,59 +188,19 @@ export class Page extends LitElement { } fileConfiguration.push(payload) - - // const result = await runConversion( - // { - // output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, - // project_name: name, - // nwbfile_path: file, - // overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) - // ...sessionInfo, // source_data and metadata are passed in here - // ...conversionOptions, // Any additional conversion options override the defaults - - // interfaces: globalState.interfaces, - // }, - // swalOpts - // ).catch((error) => { - // let message = error.message; - - // if (message.includes("The user aborted a request.")) { - // this.notify("Conversion was cancelled.", "warning"); - // throw error; - // } - - // this.notify(message, "error"); - // closeProgressPopup(); - // throw error; - // }); - - // completed++; - // if (isMultiple) { - // const progressInfo = { n: completed, total: toRun.length }; - // elements.progress.format = progressInfo; - // } } - - const request_id = Math.random().toString(36).substring(7); - - const conversionResults = run(`convert`, { + const conversionResults = await run(`convert`, { files: fileConfiguration, - max_workers: 4, - request_id + max_workers: 2, // TODO: Make this configurable and confirm default value + request_id: swalOpts.id }, { title: "Running the conversion", - onError: (results) => { - if (results.message.includes("already exists")) { - return "File already exists. Please specify another location to store the conversion results"; - } else { - return "Conversion failed with current metadata. Please try again."; - } - }, + onError: () => "Conversion failed with current metadata. Please try again.", ...swalOpts, }) - .catch((error) => { + .catch(async (error) => { let message = error.message; if (message.includes("The user aborted a request.")) { @@ -259,20 +209,21 @@ export class Page extends LitElement { } this.notify(message, "error"); - closeProgressPopup(); + await closeProgressPopup(); throw error; }); - for (let file in conversionResults) { - const [ subject, session ] = file.match(/sub-(\d+)\/sub-\d+_ses-(\d+)\.nwb/).slice(1); + conversionResults.forEach((info) => { + const { file } = info + const fileName = file.split("/").pop(); + const [ subject, session ] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1); const subRef = results[subject] ?? (results[subject] = {}); - subRef[session] = conversionResults[file]; - } + subRef[session] = info; + }) - closeProgressPopup(); - elements.container.style.textAlign = ""; // Clear style update + await closeProgressPopup(); return results; } diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js index 82fa2fa77b..7b23aa5933 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js @@ -162,15 +162,15 @@ export class GuidedInspectorPage extends Page { "inspect_folder", { path, ...options, request_id: swalOpts.id }, swalOpts - ).catch((error) => { + ).catch(async (error) => { this.notify(error.message, "error"); - closeProgressPopup(); + await closeProgressPopup(); return null; }); if (!result) return "Failed to generate inspector report."; - closeProgressPopup(); + await closeProgressPopup(); this.report = globalState.preview.inspector = { ...result, diff --git a/src/renderer/src/stories/pages/inspect/InspectPage.js b/src/renderer/src/stories/pages/inspect/InspectPage.js index 577d7fbdee..c7c0eca348 100644 --- a/src/renderer/src/stories/pages/inspect/InspectPage.js +++ b/src/renderer/src/stories/pages/inspect/InspectPage.js @@ -29,13 +29,13 @@ export class InspectPage extends Page { const { close: closeProgressPopup } = swalOpts; - const result = await run("inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch((error) => { + const result = await run("inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch(async (error) => { this.notify(error.message, "error"); - closeProgressPopup(); + await closeProgressPopup(); throw error; }); - closeProgressPopup(); + await closeProgressPopup(); if (typeof result === "string") return result; diff --git a/src/renderer/src/stories/pages/utils.js b/src/renderer/src/stories/pages/utils.js index 9df6ae61dd..052e34007c 100644 --- a/src/renderer/src/stories/pages/utils.js +++ b/src/renderer/src/stories/pages/utils.js @@ -35,7 +35,8 @@ export const sanitize = (item, condition = isPrivate) => { if (condition(k, value)) delete item[k]; else sanitize(value, condition); } - } + } else if (Array.isArray(item)) item.forEach((value) => sanitize(value, condition)); + return item; }; diff --git a/src/renderer/src/stories/utils/progress.js b/src/renderer/src/stories/utils/progress.js index 057a41ae25..e40b91040d 100644 --- a/src/renderer/src/stories/utils/progress.js +++ b/src/renderer/src/stories/utils/progress.js @@ -60,7 +60,6 @@ export const createProgressPopup = async (options, tqdmCallback) => { const onProgressMessage = ({ data }) => { const parsed = JSON.parse(data); const { request_id, ...update } = parsed; - console.warn("parsed", parsed); if (request_id && request_id !== id) return; lastUpdate = Date.now(); @@ -74,13 +73,16 @@ export const createProgressPopup = async (options, tqdmCallback) => { progressHandler.addEventListener("message", onProgressMessage); - const close = () => { + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + const close = async () => { if (lastUpdate) { // const timeSinceLastUpdate = now - lastUpdate; const animationLeft = 1000; // ProgressBar.animationDuration - timeSinceLastUpdate; // Add 100ms to ensure the animation has time to complete - if (animationLeft) setTimeout(() => popup.close(), animationLeft); - else popup.close(); - } else popup.close(); + if (animationLeft) await sleep(animationLeft); + } + + popup.close(); progressHandler.removeEventListener("message", onProgressMessage); }; From 0d3935d59ab617f33f5e815574aef3587a29dd1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 23:24:01 +0000 Subject: [PATCH 03/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 11 ++---- src/renderer/src/stories/pages/Page.js | 38 +++++++++---------- .../src/stories/pages/inspect/InspectPage.js | 12 +++--- src/renderer/src/stories/pages/utils.js | 1 - src/renderer/src/stories/utils/progress.js | 4 +- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 0fa8ab8e67..4f54bd4f0c 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -124,7 +124,6 @@ def resolve_references(schema, root_schema=None): dict: The resolved JSON schema. """ from jsonschema import RefResolver - if root_schema is None: root_schema = schema @@ -664,6 +663,7 @@ def get_interface_alignment(info: dict) -> dict: return timestamps + def convert_to_nwb(info: dict) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" @@ -703,7 +703,7 @@ def convert_to_nwb(info: dict) -> str: def update_conversion_progress(**kwargs): update_dict = dict(request_id=request_id, **kwargs) if (url) or not run_stub_test: - requests.post(url=url, json=update_dict) + requests.post(url=url, json=update_dict) else: announcer.announce(update_dict) @@ -712,14 +712,13 @@ def update_conversion_progress(**kwargs): on_progress_update=update_conversion_progress, ) - # Assume all interfaces have the same conversion options for now available_options = converter.get_conversion_options_schema() options = ( { interface: ( { - "stub_test": info["stub_test"], + "stub_test": info["stub_test"], # "iterator_opts": dict( display_progress=True, progress_bar_class=TQDMProgressSubscriber, progress_bar_options=progress_bar_options ) } if available_options.get("properties").get(interface).get("properties", {}).get("stub_test") @@ -813,7 +812,7 @@ def convert_all_to_nwb( request_id: Optional[str], max_workers: int = 1, ) -> List[str]: - + from tqdm_publisher import TQDMProgressSubscriber from concurrent.futures import ProcessPoolExecutor, as_completed @@ -826,7 +825,6 @@ def on_progress_update(message): ) ) - futures = [] file_paths = [] @@ -853,7 +851,6 @@ def on_progress_update(message): on_progress_update=on_progress_update, ) - for future in inspection_iterable: output_filepath = future.result() file_paths.append(output_filepath) diff --git a/src/renderer/src/stories/pages/Page.js b/src/renderer/src/stories/pages/Page.js index 9d3407f7d0..cff7f18d04 100644 --- a/src/renderer/src/stories/pages/Page.js +++ b/src/renderer/src/stories/pages/Page.js @@ -157,7 +157,7 @@ export class Page extends LitElement { const swalOpts = await createProgressPopup({ title: `Running conversion`, ...options }); const { close: closeProgressPopup } = swalOpts; - const fileConfiguration = [] + const fileConfiguration = []; for (let info of toRun) { const { subject, session, globalState = this.info.globalState } = info; @@ -185,22 +185,24 @@ export class Page extends LitElement { ...conversionOptions, // Any additional conversion options override the defaults interfaces: globalState.interfaces, - } + }; - fileConfiguration.push(payload) + fileConfiguration.push(payload); } - const conversionResults = await run(`convert`, { - files: fileConfiguration, - max_workers: 2, // TODO: Make this configurable and confirm default value - request_id: swalOpts.id - }, { - title: "Running the conversion", - onError: () => "Conversion failed with current metadata. Please try again.", - ...swalOpts, - }) - - .catch(async (error) => { + const conversionResults = await run( + `convert`, + { + files: fileConfiguration, + max_workers: 2, // TODO: Make this configurable and confirm default value + request_id: swalOpts.id, + }, + { + title: "Running the conversion", + onError: () => "Conversion failed with current metadata. Please try again.", + ...swalOpts, + } + ).catch(async (error) => { let message = error.message; if (message.includes("The user aborted a request.")) { @@ -213,15 +215,13 @@ export class Page extends LitElement { throw error; }); - conversionResults.forEach((info) => { - const { file } = info + const { file } = info; const fileName = file.split("/").pop(); - const [ subject, session ] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1); + const [subject, session] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1); const subRef = results[subject] ?? (results[subject] = {}); subRef[session] = info; - }) - + }); await closeProgressPopup(); diff --git a/src/renderer/src/stories/pages/inspect/InspectPage.js b/src/renderer/src/stories/pages/inspect/InspectPage.js index c7c0eca348..ed648c127a 100644 --- a/src/renderer/src/stories/pages/inspect/InspectPage.js +++ b/src/renderer/src/stories/pages/inspect/InspectPage.js @@ -29,11 +29,13 @@ export class InspectPage extends Page { const { close: closeProgressPopup } = swalOpts; - const result = await run("inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch(async (error) => { - this.notify(error.message, "error"); - await closeProgressPopup(); - throw error; - }); + const result = await run("inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch( + async (error) => { + this.notify(error.message, "error"); + await closeProgressPopup(); + throw error; + } + ); await closeProgressPopup(); diff --git a/src/renderer/src/stories/pages/utils.js b/src/renderer/src/stories/pages/utils.js index 052e34007c..e6256bdbf8 100644 --- a/src/renderer/src/stories/pages/utils.js +++ b/src/renderer/src/stories/pages/utils.js @@ -36,7 +36,6 @@ export const sanitize = (item, condition = isPrivate) => { else sanitize(value, condition); } } else if (Array.isArray(item)) item.forEach((value) => sanitize(value, condition)); - return item; }; diff --git a/src/renderer/src/stories/utils/progress.js b/src/renderer/src/stories/utils/progress.js index e40b91040d..7c35f7d31c 100644 --- a/src/renderer/src/stories/utils/progress.js +++ b/src/renderer/src/stories/utils/progress.js @@ -80,8 +80,8 @@ export const createProgressPopup = async (options, tqdmCallback) => { // const timeSinceLastUpdate = now - lastUpdate; const animationLeft = 1000; // ProgressBar.animationDuration - timeSinceLastUpdate; // Add 100ms to ensure the animation has time to complete if (animationLeft) await sleep(animationLeft); - } - + } + popup.close(); progressHandler.removeEventListener("message", onProgressMessage); From 835126bcedb25efe8ef2819c12ca171dc2663b6e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 11:17:40 -0700 Subject: [PATCH 04/23] Update manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 8e1725e4ba..b48db29352 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -722,21 +722,23 @@ def update_conversion_progress(**kwargs): # Assume all interfaces have the same conversion options for now available_options = converter.get_conversion_options_schema() - options = ( - { - interface: ( - { - "stub_test": info["stub_test"], - # "iterator_opts": dict( display_progress=True, progress_bar_class=TQDMProgressSubscriber, progress_bar_options=progress_bar_options ) - } - if available_options.get("properties").get(interface).get("properties", {}).get("stub_test") - else {} + options = {interface: {} for interface in info["source_data"]} + + for interface in options: + available_opts = available_options.get("properties").get(interface).get("properties", {}) + + # Specify if stub test + if run_stub_test: + if available_opts.get("stub_test"): + options[interface]["stub_test"] = True + + # Specify if iterator options are available + elif available_opts.get("iterator_opts"): + options[interface]["iterator_opts"] = dict( + display_progress=True, + progress_bar_class=TQDMProgressSubscriber, + progress_bar_options=progress_bar_options ) - for interface in info["source_data"] - } - if run_stub_test - else None - ) # Ensure Ophys NaN values are resolved resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) From 511ca604f496a694af52fc33e0f517c627d92c76 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 18:17:59 +0000 Subject: [PATCH 05/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b48db29352..31346cb6e8 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -735,9 +735,9 @@ def update_conversion_progress(**kwargs): # Specify if iterator options are available elif available_opts.get("iterator_opts"): options[interface]["iterator_opts"] = dict( - display_progress=True, - progress_bar_class=TQDMProgressSubscriber, - progress_bar_options=progress_bar_options + display_progress=True, + progress_bar_class=TQDMProgressSubscriber, + progress_bar_options=progress_bar_options, ) # Ensure Ophys NaN values are resolved From fd741e98e36010324e6567fac28ffce8f406197e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 11:52:09 -0700 Subject: [PATCH 06/23] Working conversion updates from HDMF using new PRs across different dependencies --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 ++-- src/renderer/src/stories/ProgressBar.ts | 8 +++++--- src/renderer/src/stories/utils/progress.js | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b48db29352..71dac99381 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -708,8 +708,8 @@ def convert_to_nwb(info: dict) -> str: converter = instantiate_custom_converter(resolved_source_data, info["interfaces"]) - def update_conversion_progress(**kwargs): - update_dict = dict(request_id=request_id, **kwargs) + def update_conversion_progress(message): + update_dict = dict(request_id=request_id, **message) if (url) or not run_stub_test: requests.post(url=url, json=update_dict) else: diff --git a/src/renderer/src/stories/ProgressBar.ts b/src/renderer/src/stories/ProgressBar.ts index 06da911777..0510769004 100644 --- a/src/renderer/src/stories/ProgressBar.ts +++ b/src/renderer/src/stories/ProgressBar.ts @@ -100,8 +100,10 @@ export class ProgressBar extends LitElement { render() { - const percent = this.format.total ? 100 * (this.format.n / this.format.total) : 0; - const remaining = this.format.rate && this.format.total ? (this.format.total - this.format.n) / this.format.rate : 0; // Seconds + const n = this.format.n > this.format.total ? this.format.total : this.format.n + const ratio = n/ this.format.total; + const percent = this.format.total ? 100 * ratio : 0; + const remaining = this.format.rate && this.format.total ? (this.format.total - n) / this.format.rate : 0; // Seconds return html`
@@ -112,7 +114,7 @@ export class ProgressBar extends LitElement {
- ${this.format.n} / ${this.format.total} (${percent.toFixed(1)}%) + ${n} / ${this.format.total} (${percent.toFixed(1)}%)
${'elapsed' in this.format && 'rate' in this.format ? html`${this.format.elapsed?.toFixed(1)}s elapsed, ${remaining.toFixed(1)}s remaining` : ''} diff --git a/src/renderer/src/stories/utils/progress.js b/src/renderer/src/stories/utils/progress.js index 7c35f7d31c..3af8afe379 100644 --- a/src/renderer/src/stories/utils/progress.js +++ b/src/renderer/src/stories/utils/progress.js @@ -29,6 +29,8 @@ export const createProgressPopup = async (options, tqdmCallback) => { textAlign: "left", display: "flex", flexDirection: "column", + overflow: "hidden", + width: '100%', gap: "5px", }); element.append(container); From 448f47d7693cd6d13f4a3d95df13c24272d934df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 18:52:28 +0000 Subject: [PATCH 07/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/utils/progress.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/stories/utils/progress.js b/src/renderer/src/stories/utils/progress.js index 3af8afe379..eda3671715 100644 --- a/src/renderer/src/stories/utils/progress.js +++ b/src/renderer/src/stories/utils/progress.js @@ -30,7 +30,7 @@ export const createProgressPopup = async (options, tqdmCallback) => { display: "flex", flexDirection: "column", overflow: "hidden", - width: '100%', + width: "100%", gap: "5px", }); element.append(container); From 46b1626d414ea5202546992945b95a7d665e7712 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 11:58:08 -0700 Subject: [PATCH 08/23] Show incorrect progress but do not render --- src/renderer/src/stories/ProgressBar.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/stories/ProgressBar.ts b/src/renderer/src/stories/ProgressBar.ts index 0510769004..ea35cb6e55 100644 --- a/src/renderer/src/stories/ProgressBar.ts +++ b/src/renderer/src/stories/ProgressBar.ts @@ -114,7 +114,7 @@ export class ProgressBar extends LitElement {
- ${n} / ${this.format.total} (${percent.toFixed(1)}%) + ${this.format.n} / ${this.format.total} (${percent.toFixed(1)}%)
${'elapsed' in this.format && 'rate' in this.format ? html`${this.format.elapsed?.toFixed(1)}s elapsed, ${remaining.toFixed(1)}s remaining` : ''} From 499c2c05e469963da759dabfb0186601a3b8dc2c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 16:30:14 -0700 Subject: [PATCH 09/23] Pass errors from individual processes to a run-specific log file in the NWB GUIDE root folder --- pyflask/apis/neuroconv.py | 16 +- pyflask/app.py | 60 ++++-- pyflask/manageNeuroconv/info/__init__.py | 2 +- pyflask/manageNeuroconv/manage_neuroconv.py | 221 +++++++++++--------- 4 files changed, 176 insertions(+), 123 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 58f02699a0..3fda7dabec 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -110,13 +110,17 @@ def post(self): class Convert(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): + log_url = f"{request.url_root}log" + try: - has_files = "files" in neuroconv_api.payload - if has_files: - url = f"{request.url_root}neuroconv/announce" - return convert_all_to_nwb(url, **neuroconv_api.payload) - else: - return convert_to_nwb(neuroconv_api.payload) + + url = f"{request.url_root}neuroconv/announce" + + return convert_all_to_nwb( + url, + **neuroconv_api.payload, + log_url=log_url, + ) except Exception as exception: if notBadRequestException(exception): diff --git a/pyflask/app.py b/pyflask/app.py index 924e9454ac..153b413278 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -12,6 +12,9 @@ from pathlib import Path from urllib.parse import unquote +from errorHandlers import notBadRequestException +from datetime import datetime + # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() @@ -22,7 +25,8 @@ from flask_restx import Api, Resource from apis import startup_api, neuroconv_api, data_api -from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH +from manageNeuroconv.info import resource_path, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH + app = Flask(__name__) @@ -30,20 +34,11 @@ CORS(app) app.config["CORS_HEADERS"] = "Content-Type" -# Configure logger -LOG_FILE_PATH = Path.home() / "NWB_GUIDE" / "logs" / "api.log" -LOG_FILE_PATH.parent.mkdir(parents=True, exist_ok=True) - -log_handler = RotatingFileHandler(LOG_FILE_PATH, maxBytes=5 * 1024 * 1024, backupCount=3) -log_formatter = Formatter( - fmt="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", -) -log_handler.setFormatter(log_formatter) - -app.logger.addHandler(log_handler) -app.logger.setLevel(DEBUG) - +# Create logger configuration +LOG_FOLDER = Path(GUIDE_ROOT_FOLDER, "logs") +LOG_FOLDER.mkdir(exist_ok=True, parents=True) +timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") +LOG_FILE_PATH = Path(LOG_FOLDER, f"{timestamp}.log") # Initialize API package_json_file_path = resource_path("package.json") @@ -113,6 +108,26 @@ def get_species(): return species_map +@api.route("/log") +class Log(Resource): + @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self): + try: + + + payload = api.payload + type = payload["type"] + header = payload["header"] + inputs = payload["inputs"] + traceback = payload["traceback"] + + message = f"{header}\n{'-'*len(header)}\n\n{json.dumps(inputs, indent=2)}\n\n{traceback}\n" + selected_logger = getattr(api.logger, type) + selected_logger(message) + + except Exception as exception: + if notBadRequestException(exception): + api.abort(500, str(exception)) @api.route("/server_shutdown", endpoint="shutdown") class Shutdown(Resource): @@ -130,6 +145,21 @@ def get(self): if __name__ == "__main__": port = sys.argv[len(sys.argv) - 1] if port.isdigit(): + + # Configure logger (avoid reinstantiation for processes) + log_handler = RotatingFileHandler(LOG_FILE_PATH, maxBytes=5 * 1024 * 1024, backupCount=3) + log_formatter = Formatter( + fmt="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + log_handler.setFormatter(log_formatter) + + app.logger.addHandler(log_handler) + app.logger.setLevel(DEBUG) + + app.logger.info(f"Logging to {LOG_FILE_PATH}") + + # Run the server api.logger.info(f"Starting server on port {port}") app.run(host="127.0.0.1", port=port) else: diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index 915d74aeed..1229ca6892 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -2,7 +2,7 @@ resource_path, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - CONVERSION_SAVE_FOLDER_PATH, + CONVERSION_SAVE_FOLDER_PATH ) from .sse import announcer, format_sse diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ff3f672149..fd764a0452 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -12,6 +12,7 @@ from shutil import rmtree, copytree from pathlib import Path from typing import Any, Dict, List, Optional +import traceback from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, announcer @@ -672,7 +673,10 @@ def get_interface_alignment(info: dict) -> dict: return timestamps -def convert_to_nwb(info: dict) -> str: +def convert_to_nwb( + info: dict, + log_url=None, + ) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" from tqdm_publisher import TQDMProgressSubscriber @@ -680,144 +684,157 @@ def convert_to_nwb(info: dict) -> str: url = info.get("url", None) request_id = info.get("request_id", None) - nwbfile_path = Path(info["nwbfile_path"]) custom_output_directory = info.get("output_folder") project_name = info.get("project_name") run_stub_test = info.get("stub_test", False) - default_output_base = STUB_SAVE_FOLDER_PATH if run_stub_test else CONVERSION_SAVE_FOLDER_PATH default_output_directory = default_output_base / project_name - run_stub_test = info.get("stub_test", False) - # add a subdirectory to a filepath if stub_test is true - resolved_output_base = Path(custom_output_directory) if custom_output_directory else default_output_base - resolved_output_directory = resolved_output_base / project_name - resolved_output_path = resolved_output_directory / nwbfile_path - # Remove symlink placed at the default_output_directory if this will hold real data - if resolved_output_directory == default_output_directory and default_output_directory.is_symlink(): - default_output_directory.unlink() + try: - resolved_output_path.parent.mkdir(exist_ok=True, parents=True) # Ensure all parent directories exist + # add a subdirectory to a filepath if stub_test is true + resolved_output_base = Path(custom_output_directory) if custom_output_directory else default_output_base + resolved_output_directory = resolved_output_base / project_name + resolved_output_path = resolved_output_directory / nwbfile_path - resolved_source_data = replace_none_with_nan( - info["source_data"], resolve_references(get_custom_converter(info["interfaces"]).get_source_schema()) - ) + # Remove symlink placed at the default_output_directory if this will hold real data + if resolved_output_directory == default_output_directory and default_output_directory.is_symlink(): + default_output_directory.unlink() - converter = instantiate_custom_converter(resolved_source_data, info["interfaces"]) + resolved_output_path.parent.mkdir(exist_ok=True, parents=True) # Ensure all parent directories exist - def update_conversion_progress(message): - update_dict = dict(request_id=request_id, **message) - if (url) or not run_stub_test: - requests.post(url=url, json=update_dict) - else: - announcer.announce(update_dict) + resolved_source_data = replace_none_with_nan( + info["source_data"], resolve_references(get_custom_converter(info["interfaces"]).get_source_schema()) + ) - progress_bar_options = dict( - mininterval=0, - on_progress_update=update_conversion_progress, - ) + converter = instantiate_custom_converter(resolved_source_data, info["interfaces"]) - # Assume all interfaces have the same conversion options for now - available_options = converter.get_conversion_options_schema() - options = {interface: {} for interface in info["source_data"]} + def update_conversion_progress(message): + update_dict = dict(request_id=request_id, **message) + if (url) or not run_stub_test: + requests.post(url=url, json=update_dict) + else: + announcer.announce(update_dict) - for interface in options: - available_opts = available_options.get("properties").get(interface).get("properties", {}) + progress_bar_options = dict( + mininterval=0, + on_progress_update=update_conversion_progress, + ) - # Specify if stub test - if run_stub_test: - if available_opts.get("stub_test"): - options[interface]["stub_test"] = True + # Assume all interfaces have the same conversion options for now + available_options = converter.get_conversion_options_schema() + options = {interface: {} for interface in info["source_data"]} - # Specify if iterator options are available - elif available_opts.get("iterator_opts"): - options[interface]["iterator_opts"] = dict( - display_progress=True, - progress_bar_class=TQDMProgressSubscriber, - progress_bar_options=progress_bar_options, - ) + for interface in options: + available_opts = available_options.get("properties").get(interface).get("properties", {}) - # Ensure Ophys NaN values are resolved - resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) + # Specify if stub test + if run_stub_test: + if available_opts.get("stub_test"): + options[interface]["stub_test"] = True - ecephys_metadata = resolved_metadata.get("Ecephys") + # Specify if iterator options are available + elif available_opts.get("iterator_opts"): + options[interface]["iterator_opts"] = dict( + display_progress=True, + progress_bar_class=TQDMProgressSubscriber, + progress_bar_options=progress_bar_options, + ) - if ecephys_metadata: + # Ensure Ophys NaN values are resolved + resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) - # Quick fix to remove units - has_units = "Units" in ecephys_metadata + ecephys_metadata = resolved_metadata.get("Ecephys") - if has_units: + if ecephys_metadata: - ## NOTE: Currently do not allow editing units properties - # shared_units_columns = ecephys_metadata["UnitColumns"] - # for interface_name, interface_unit_results in ecephys_metadata["Units"].items(): - # interface = converter.data_interface_objects[interface_name] + # Quick fix to remove units + has_units = "Units" in ecephys_metadata - # update_sorting_properties_from_table_as_json( - # interface, - # unit_table_json=interface_unit_results, - # unit_column_info=shared_units_columns, - # ) + if has_units: - # ecephys_metadata["UnitProperties"] = [ - # {"name": entry["name"], "description": entry["description"]} for entry in shared_units_columns - # ] + ## NOTE: Currently do not allow editing units properties + # shared_units_columns = ecephys_metadata["UnitColumns"] + # for interface_name, interface_unit_results in ecephys_metadata["Units"].items(): + # interface = converter.data_interface_objects[interface_name] - del ecephys_metadata["Units"] - del ecephys_metadata["UnitColumns"] + # update_sorting_properties_from_table_as_json( + # interface, + # unit_table_json=interface_unit_results, + # unit_column_info=shared_units_columns, + # ) - has_electrodes = "Electrodes" in ecephys_metadata - if has_electrodes: + # ecephys_metadata["UnitProperties"] = [ + # {"name": entry["name"], "description": entry["description"]} for entry in shared_units_columns + # ] - shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] + del ecephys_metadata["Units"] + del ecephys_metadata["UnitColumns"] - for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): - interface = converter.data_interface_objects[interface_name] + has_electrodes = "Electrodes" in ecephys_metadata + if has_electrodes: - update_recording_properties_from_table_as_json( - interface, - electrode_table_json=interface_electrode_results, - electrode_column_info=shared_electrode_columns, - ) + shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] - ecephys_metadata["Electrodes"] = [ - {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns - ] + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): + interface = converter.data_interface_objects[interface_name] - del ecephys_metadata["ElectrodeColumns"] + update_recording_properties_from_table_as_json( + interface, + electrode_table_json=interface_electrode_results, + electrode_column_info=shared_electrode_columns, + ) - # Actually run the conversion - converter.run_conversion( - metadata=resolved_metadata, - nwbfile_path=resolved_output_path, - overwrite=info.get("overwrite", False), - conversion_options=options, - ) + ecephys_metadata["Electrodes"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns + ] + + del ecephys_metadata["ElectrodeColumns"] + + + # Actually run the conversion + converter.run_conversion( + metadata=resolved_metadata, + nwbfile_path=resolved_output_path, + overwrite=info.get("overwrite", False), + conversion_options=options, + ) + + # Create a symlink between the fake data and custom data + if not resolved_output_directory == default_output_directory: + if default_output_directory.exists(): + # If default default_output_directory is not a symlink, delete all contents and create a symlink there + if not default_output_directory.is_symlink(): + rmtree(default_output_directory) + + # If the location is already a symlink, but points to a different output location + # remove the existing symlink before creating a new one + elif ( + default_output_directory.is_symlink() + and default_output_directory.readlink() is not resolved_output_directory + ): + default_output_directory.unlink() - # Create a symlink between the fake data and custom data - if not resolved_output_directory == default_output_directory: - if default_output_directory.exists(): - # If default default_output_directory is not a symlink, delete all contents and create a symlink there - if not default_output_directory.is_symlink(): - rmtree(default_output_directory) + # Create a pointer to the actual conversion outputs + if not default_output_directory.exists(): + os.symlink(resolved_output_directory, default_output_directory) - # If the location is already a symlink, but points to a different output location - # remove the existing symlink before creating a new one - elif ( - default_output_directory.is_symlink() - and default_output_directory.readlink() is not resolved_output_directory - ): - default_output_directory.unlink() + return dict(file=str(resolved_output_path)) + - # Create a pointer to the actual conversion outputs - if not default_output_directory.exists(): - os.symlink(resolved_output_directory, default_output_directory) + except Exception as e: + if log_url: + requests.post(url=log_url, json=dict( + header=f"Conversion failed for {project_name} — {nwbfile_path} (convert_to_nwb)", + inputs=dict(info=info), + traceback=traceback.format_exc(), + type="error" + )) - return dict(file=str(resolved_output_path)) + raise e def convert_all_to_nwb( @@ -825,6 +842,7 @@ def convert_all_to_nwb( files: List[dict], request_id: Optional[str], max_workers: int = 1, + log_url: Optional[str] = None, ) -> List[str]: from tqdm_publisher import TQDMProgressSubscriber @@ -854,6 +872,7 @@ def on_progress_update(message): request_id=request_id, **file_info, ), + log_url=log_url, ) ) From c4a48a2ef50806e354f57d9d80e8bbae0c7f1a2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 23:38:53 +0000 Subject: [PATCH 10/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/neuroconv.py | 4 ++-- pyflask/app.py | 5 ++-- pyflask/manageNeuroconv/info/__init__.py | 7 +----- pyflask/manageNeuroconv/manage_neuroconv.py | 26 ++++++++++----------- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 3fda7dabec..e1748a2d9e 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -111,13 +111,13 @@ class Convert(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): log_url = f"{request.url_root}log" - + try: url = f"{request.url_root}neuroconv/announce" return convert_all_to_nwb( - url, + url, **neuroconv_api.payload, log_url=log_url, ) diff --git a/pyflask/app.py b/pyflask/app.py index 153b413278..bb8fe9852d 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -108,13 +108,13 @@ def get_species(): return species_map + @api.route("/log") class Log(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - payload = api.payload type = payload["type"] header = payload["header"] @@ -122,13 +122,14 @@ def post(self): traceback = payload["traceback"] message = f"{header}\n{'-'*len(header)}\n\n{json.dumps(inputs, indent=2)}\n\n{traceback}\n" - selected_logger = getattr(api.logger, type) + selected_logger = getattr(api.logger, type) selected_logger(message) except Exception as exception: if notBadRequestException(exception): api.abort(500, str(exception)) + @api.route("/server_shutdown", endpoint="shutdown") class Shutdown(Resource): def get(self): diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index 1229ca6892..9b35ca3510 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -1,8 +1,3 @@ -from .urls import ( - resource_path, - GUIDE_ROOT_FOLDER, - STUB_SAVE_FOLDER_PATH, - CONVERSION_SAVE_FOLDER_PATH -) +from .urls import resource_path, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH from .sse import announcer, format_sse diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index e99354e29d..858349c1d2 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -690,9 +690,9 @@ def get_interface_alignment(info: dict) -> dict: def convert_to_nwb( - info: dict, - log_url=None, - ) -> str: + info: dict, + log_url=None, +) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" from neuroconv import NWBConverter @@ -710,7 +710,6 @@ def convert_to_nwb( default_output_directory = default_output_base / project_name run_stub_test = info.get("stub_test", False) - try: # add a subdirectory to a filepath if stub_test is true @@ -762,7 +761,6 @@ def update_conversion_progress(message): progress_bar_options=progress_bar_options, ) - # Ensure Ophys NaN values are resolved resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) @@ -796,7 +794,6 @@ def update_conversion_progress(message): has_electrodes = "Electrodes" in ecephys_metadata if has_electrodes: - shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): @@ -832,7 +829,6 @@ def update_conversion_progress(message): del ecephys_metadata["ElectrodeColumns"] - # Actually run the conversion converter.run_conversion( metadata=resolved_metadata, @@ -861,16 +857,18 @@ def update_conversion_progress(message): os.symlink(resolved_output_directory, default_output_directory) return dict(file=str(resolved_output_path)) - except Exception as e: if log_url: - requests.post(url=log_url, json=dict( - header=f"Conversion failed for {project_name} — {nwbfile_path} (convert_to_nwb)", - inputs=dict(info=info), - traceback=traceback.format_exc(), - type="error" - )) + requests.post( + url=log_url, + json=dict( + header=f"Conversion failed for {project_name} — {nwbfile_path} (convert_to_nwb)", + inputs=dict(info=info), + traceback=traceback.format_exc(), + type="error", + ), + ) raise e From 885e1e0d10e0f1aea189199e1114b691a45b99be Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 14:44:34 +0000 Subject: [PATCH 11/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/app.py | 8 ++------ pyflask/manageNeuroconv/__init__.py | 2 +- pyflask/manageNeuroconv/info/__init__.py | 1 - pyflask/manageNeuroconv/manage_neuroconv.py | 10 +++++----- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pyflask/app.py b/pyflask/app.py index f372f4a8d3..a75fb88bbb 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -3,6 +3,7 @@ import json import multiprocessing import sys +from datetime import datetime from logging import DEBUG, Formatter from logging.handlers import RotatingFileHandler from os import getpid, kill @@ -12,8 +13,6 @@ from urllib.parse import unquote from errorHandlers import notBadRequestException -from datetime import datetime - # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() @@ -23,12 +22,9 @@ from flask import Flask, request, send_file, send_from_directory from flask_cors import CORS from flask_restx import Api, Resource - -from apis import startup_api, neuroconv_api, data_api - from manageNeuroconv.info import ( - GUIDE_ROOT_FOLDER, CONVERSION_SAVE_FOLDER_PATH, + GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, resource_path, ) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index e9c9b82736..841e18397e 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -1,8 +1,8 @@ from .info import CONVERSION_SAVE_FOLDER_PATH, STUB_SAVE_FOLDER_PATH from .manage_neuroconv import ( autocomplete_format_string, - convert_to_nwb, convert_all_to_nwb, + convert_to_nwb, generate_dataset, generate_test_data, get_all_converter_info, diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index c939d0efe4..edde04113c 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -1,5 +1,4 @@ from .sse import announcer, format_sse - from .urls import ( CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 51bd436a6e..24c9b11526 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -6,12 +6,11 @@ import math import os import re +import traceback from datetime import datetime from pathlib import Path - -from typing import Any, Dict, List, Optional, Union -import traceback from shutil import copytree, rmtree +from typing import Any, Dict, List, Optional, Union from .info import ( CONVERSION_SAVE_FOLDER_PATH, @@ -703,9 +702,9 @@ def convert_to_nwb( ) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" + import requests from neuroconv import NWBConverter from tqdm_publisher import TQDMProgressSubscriber - import requests url = info.get("url", None) request_id = info.get("request_id", None) @@ -889,9 +888,10 @@ def convert_all_to_nwb( log_url: Optional[str] = None, ) -> List[str]: - from tqdm_publisher import TQDMProgressSubscriber from concurrent.futures import ProcessPoolExecutor, as_completed + from tqdm_publisher import TQDMProgressSubscriber + def on_progress_update(message): message["progress_bar_id"] = request_id # Ensure request_id matches announcer.announce( From b42cc0aec39e134f6c659bbbf9ca7ef6a39d25a2 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 30 May 2024 16:33:29 -0700 Subject: [PATCH 12/23] Update neuroconv.py --- src/pyflask/namespaces/neuroconv.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pyflask/namespaces/neuroconv.py b/src/pyflask/namespaces/neuroconv.py index f298a6a13a..77b8d90124 100644 --- a/src/pyflask/namespaces/neuroconv.py +++ b/src/pyflask/namespaces/neuroconv.py @@ -4,7 +4,7 @@ from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import ( autocomplete_format_string, - convert_to_nwb, + convert_all_to_nwb, get_all_converter_info, get_all_interface_info, get_interface_alignment, @@ -75,9 +75,16 @@ def post(self): class Convert(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): - return convert_to_nwb(neuroconv_namespace.payload) - + + log_url = f"{request.url_root}log" + url = f"{request.url_root}neuroconv/announce/progress" + return convert_all_to_nwb( + url, + **neuroconv_namespace.payload, + log_url=log_url, + ) + @neuroconv_namespace.route("/alignment") class Alignment(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) From dcf1815183321ed802cf8f73d1cd61402ab0f7d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 23:33:43 +0000 Subject: [PATCH 13/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../frontend/core/components/pages/Page.js | 565 +++++++++--------- .../options/GuidedInspectorPage.js | 537 ++++++++--------- .../components/pages/inspect/InspectPage.js | 340 +++++------ .../frontend/core/components/pages/utils.js | 154 ++--- .../core/components/utils/progress.js | 240 ++++---- 5 files changed, 918 insertions(+), 918 deletions(-) diff --git a/src/electron/frontend/core/components/pages/Page.js b/src/electron/frontend/core/components/pages/Page.js index 5b3a8b7307..7f107be2d9 100644 --- a/src/electron/frontend/core/components/pages/Page.js +++ b/src/electron/frontend/core/components/pages/Page.js @@ -1,283 +1,282 @@ -import { LitElement, html } from "lit"; -import { run } from "./guided-mode/options/utils.js"; -import { get, save } from "../../progress/index.js"; - -import { dismissNotification, notify } from "../../dependencies.js"; -import { isStorybook } from "../../globals.js"; - -import { randomizeElements, mapSessions, merge } from "./utils"; - -import { resolveMetadata } from "./guided-mode/data/utils.js"; -import Swal from "sweetalert2"; -import { createProgressPopup } from "../utils/progress.js"; - -export class Page extends LitElement { - // static get styles() { - // return useGlobalStyles( - // componentCSS, - // (sheet) => sheet.href && sheet.href.includes("bootstrap"), - // this.shadowRoot - // ); - // } - - info = { globalState: {} }; - - constructor(info = {}) { - super(); - Object.assign(this.info, info); - } - - createRenderRoot() { - return this; - } - - query = (input) => { - return (this.shadowRoot ?? this).querySelector(input); - }; - - onSet = () => {}; // User-defined function - - set = (info, rerender = true) => { - if (info) { - Object.assign(this.info, info); - this.onSet(); - if (rerender) this.requestUpdate(); - } - }; - - #notifications = []; - - dismiss = (notification) => { - if (notification) dismissNotification(notification); - else { - this.#notifications.forEach((notification) => dismissNotification(notification)); - this.#notifications = []; - } - }; - - notify = (...args) => { - const ref = notify(...args); - this.#notifications.push(ref); - return ref; - }; - - to = async (transition) => { - // Otherwise note unsaved updates if present - if ( - this.unsavedUpdates || - ("states" in this.info && - transition === 1 && // Only ensure save for standard forward progression - !this.info.states.saved) - ) { - if (transition === 1) - await this.save(); // Save before a single forward transition - else { - await Swal.fire({ - title: "You have unsaved data on this page.", - text: "Would you like to save your changes?", - icon: "warning", - showCancelButton: true, - confirmButtonColor: "#3085d6", - confirmButtonText: "Save and Continue", - cancelButtonText: "Ignore Changes", - }).then(async (result) => { - if (result && result.isConfirmed) await this.save(); - }); - } - } - - return await this.onTransition(transition); - }; - - onTransition = () => {}; // User-defined function - updatePages = () => {}; // User-defined function - beforeSave = () => {}; // User-defined function - - save = async (overrides, runBeforeSave = true) => { - if (runBeforeSave) await this.beforeSave(); - save(this, overrides); - if ("states" in this.info) this.info.states.saved = true; - this.unsavedUpdates = false; - }; - - load = (datasetNameToResume = new URLSearchParams(window.location.search).get("project")) => - (this.info.globalState = get(datasetNameToResume)); - - addSession({ subject, session, info }) { - if (!this.info.globalState.results[subject]) this.info.globalState.results[subject] = {}; - if (this.info.globalState.results[subject][session]) - throw new Error(`Session ${subject}/${session} already exists.`); - info = this.info.globalState.results[subject][session] = info ?? {}; - if (!info.metadata) info.metadata = {}; - if (!info.source_data) info.source_data = {}; - return info; - } - - removeSession({ subject, session }) { - delete this.info.globalState.results[subject][session]; - } - - mapSessions = (callback, data = this.info.globalState.results) => mapSessions(callback, data); - - async convert({ preview } = {}) { - const key = preview ? "preview" : "conversion"; - - delete this.info.globalState[key]; // Clear the preview results - - if (preview) { - const stubs = await this.runConversions({ stub_test: true }, undefined, { - title: "Creating conversion preview for all sessions...", - }); - this.info.globalState[key] = { stubs }; - } else { - this.info.globalState[key] = await this.runConversions({}, true, { title: "Running all conversions" }); - } - - this.unsavedUpdates = true; - - // Indicate conversion has run successfully - const { desyncedData } = this.info.globalState; - if (!desyncedData) this.info.globalState.desyncedData = {}; - - if (desyncedData) { - desyncedData[key] = false; - await this.save({}, false); - } - } - - async runConversions(conversionOptions = {}, toRun, options = {}) { - let original = toRun; - if (!Array.isArray(toRun)) toRun = this.mapSessions(); - - // Filter the sessions to run - if (typeof original === "number") - toRun = randomizeElements(toRun, original); // Grab a random set of sessions - else if (typeof original === "string") toRun = toRun.filter(({ subject }) => subject === original); - else if (typeof original === "function") toRun = toRun.filter(original); - - const results = {}; - - const swalOpts = await createProgressPopup({ title: `Running conversion`, ...options }); - - const { close: closeProgressPopup } = swalOpts; - const fileConfiguration = [] - - for (let info of toRun) { - const { subject, session, globalState = this.info.globalState } = info; - const file = `sub-${subject}/sub-${subject}_ses-${session}.nwb`; - - const { conversion_output_folder, name, SourceData, alignment } = globalState.project; - - const sessionResults = globalState.results[subject][session]; - - const sourceDataCopy = structuredClone(sessionResults.source_data); - - // Resolve the correct session info from all of the metadata for this conversion - const sessionInfo = { - ...sessionResults, - metadata: resolveMetadata(subject, session, globalState), - source_data: merge(SourceData, sourceDataCopy), - }; - - - const payload = { - output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, - project_name: name, - nwbfile_path: file, - overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) - ...sessionInfo, // source_data and metadata are passed in here - ...conversionOptions, // Any additional conversion options override the defaults - - interfaces: globalState.interfaces, - }; - - fileConfiguration.push(payload); - } - - const conversionResults = await run( - `neuroconv/convert`, - { - files: fileConfiguration, - max_workers: 2, // TODO: Make this configurable and confirm default value - request_id: swalOpts.id, - }, - { - title: "Running the conversion", - onError: () => "Conversion failed with current metadata. Please try again.", - ...swalOpts, - } - ).catch(async (error) => { - let message = error.message; - - if (message.includes("The user aborted a request.")) { - this.notify("Conversion was cancelled.", "warning"); - throw error; - } - - this.notify(message, "error"); - await closeProgressPopup(); - throw error; - }); - - conversionResults.forEach((info) => { - const { file } = info; - const fileName = file.split("/").pop(); - const [subject, session] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1); - const subRef = results[subject] ?? (results[subject] = {}); - subRef[session] = info; - }); - - await closeProgressPopup(); - - return results; - } - - // NOTE: Until the shadow DOM is supported in Storybook, we can't use this render function how we'd intend to. - addPage = (id, subpage) => { - if (!this.info.pages) this.info.pages = {}; - this.info.pages[id] = subpage; - this.updatePages(); - }; - - checkSyncState = async (info = this.info, sync = info.sync) => { - if (!sync) return; - if (isStorybook) return; - - const { desyncedData } = info.globalState; - - return Promise.all( - sync.map((k) => { - if (desyncedData?.[k] !== false) { - if (k === "conversion") return this.convert(); - else if (k === "preview") return this.convert({ preview: true }); - } - }) - ); - }; - - updateSections = () => { - const dashboard = document.querySelector("nwb-dashboard"); - dashboard.updateSections({ sidebar: true, main: true }, this.info.globalState); - }; - - #unsaved = false; - get unsavedUpdates() { - return this.#unsaved; - } - - set unsavedUpdates(value) { - this.#unsaved = !!value; - if (value === "conversions") this.info.globalState.desyncedData = { preview: true, conversion: true }; - } - - // NOTE: Make sure you call this explicitly if a child class overwrites this AND data is updated - updated() { - this.unsavedUpdates = false; - } - - render() { - return html``; - } -} - -customElements.get("nwbguide-page") || customElements.define("nwbguide-page", Page); +import { LitElement, html } from "lit"; +import { run } from "./guided-mode/options/utils.js"; +import { get, save } from "../../progress/index.js"; + +import { dismissNotification, notify } from "../../dependencies.js"; +import { isStorybook } from "../../globals.js"; + +import { randomizeElements, mapSessions, merge } from "./utils"; + +import { resolveMetadata } from "./guided-mode/data/utils.js"; +import Swal from "sweetalert2"; +import { createProgressPopup } from "../utils/progress.js"; + +export class Page extends LitElement { + // static get styles() { + // return useGlobalStyles( + // componentCSS, + // (sheet) => sheet.href && sheet.href.includes("bootstrap"), + // this.shadowRoot + // ); + // } + + info = { globalState: {} }; + + constructor(info = {}) { + super(); + Object.assign(this.info, info); + } + + createRenderRoot() { + return this; + } + + query = (input) => { + return (this.shadowRoot ?? this).querySelector(input); + }; + + onSet = () => {}; // User-defined function + + set = (info, rerender = true) => { + if (info) { + Object.assign(this.info, info); + this.onSet(); + if (rerender) this.requestUpdate(); + } + }; + + #notifications = []; + + dismiss = (notification) => { + if (notification) dismissNotification(notification); + else { + this.#notifications.forEach((notification) => dismissNotification(notification)); + this.#notifications = []; + } + }; + + notify = (...args) => { + const ref = notify(...args); + this.#notifications.push(ref); + return ref; + }; + + to = async (transition) => { + // Otherwise note unsaved updates if present + if ( + this.unsavedUpdates || + ("states" in this.info && + transition === 1 && // Only ensure save for standard forward progression + !this.info.states.saved) + ) { + if (transition === 1) + await this.save(); // Save before a single forward transition + else { + await Swal.fire({ + title: "You have unsaved data on this page.", + text: "Would you like to save your changes?", + icon: "warning", + showCancelButton: true, + confirmButtonColor: "#3085d6", + confirmButtonText: "Save and Continue", + cancelButtonText: "Ignore Changes", + }).then(async (result) => { + if (result && result.isConfirmed) await this.save(); + }); + } + } + + return await this.onTransition(transition); + }; + + onTransition = () => {}; // User-defined function + updatePages = () => {}; // User-defined function + beforeSave = () => {}; // User-defined function + + save = async (overrides, runBeforeSave = true) => { + if (runBeforeSave) await this.beforeSave(); + save(this, overrides); + if ("states" in this.info) this.info.states.saved = true; + this.unsavedUpdates = false; + }; + + load = (datasetNameToResume = new URLSearchParams(window.location.search).get("project")) => + (this.info.globalState = get(datasetNameToResume)); + + addSession({ subject, session, info }) { + if (!this.info.globalState.results[subject]) this.info.globalState.results[subject] = {}; + if (this.info.globalState.results[subject][session]) + throw new Error(`Session ${subject}/${session} already exists.`); + info = this.info.globalState.results[subject][session] = info ?? {}; + if (!info.metadata) info.metadata = {}; + if (!info.source_data) info.source_data = {}; + return info; + } + + removeSession({ subject, session }) { + delete this.info.globalState.results[subject][session]; + } + + mapSessions = (callback, data = this.info.globalState.results) => mapSessions(callback, data); + + async convert({ preview } = {}) { + const key = preview ? "preview" : "conversion"; + + delete this.info.globalState[key]; // Clear the preview results + + if (preview) { + const stubs = await this.runConversions({ stub_test: true }, undefined, { + title: "Creating conversion preview for all sessions...", + }); + this.info.globalState[key] = { stubs }; + } else { + this.info.globalState[key] = await this.runConversions({}, true, { title: "Running all conversions" }); + } + + this.unsavedUpdates = true; + + // Indicate conversion has run successfully + const { desyncedData } = this.info.globalState; + if (!desyncedData) this.info.globalState.desyncedData = {}; + + if (desyncedData) { + desyncedData[key] = false; + await this.save({}, false); + } + } + + async runConversions(conversionOptions = {}, toRun, options = {}) { + let original = toRun; + if (!Array.isArray(toRun)) toRun = this.mapSessions(); + + // Filter the sessions to run + if (typeof original === "number") + toRun = randomizeElements(toRun, original); // Grab a random set of sessions + else if (typeof original === "string") toRun = toRun.filter(({ subject }) => subject === original); + else if (typeof original === "function") toRun = toRun.filter(original); + + const results = {}; + + const swalOpts = await createProgressPopup({ title: `Running conversion`, ...options }); + + const { close: closeProgressPopup } = swalOpts; + const fileConfiguration = []; + + for (let info of toRun) { + const { subject, session, globalState = this.info.globalState } = info; + const file = `sub-${subject}/sub-${subject}_ses-${session}.nwb`; + + const { conversion_output_folder, name, SourceData, alignment } = globalState.project; + + const sessionResults = globalState.results[subject][session]; + + const sourceDataCopy = structuredClone(sessionResults.source_data); + + // Resolve the correct session info from all of the metadata for this conversion + const sessionInfo = { + ...sessionResults, + metadata: resolveMetadata(subject, session, globalState), + source_data: merge(SourceData, sourceDataCopy), + }; + + const payload = { + output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder, + project_name: name, + nwbfile_path: file, + overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite) + ...sessionInfo, // source_data and metadata are passed in here + ...conversionOptions, // Any additional conversion options override the defaults + + interfaces: globalState.interfaces, + }; + + fileConfiguration.push(payload); + } + + const conversionResults = await run( + `neuroconv/convert`, + { + files: fileConfiguration, + max_workers: 2, // TODO: Make this configurable and confirm default value + request_id: swalOpts.id, + }, + { + title: "Running the conversion", + onError: () => "Conversion failed with current metadata. Please try again.", + ...swalOpts, + } + ).catch(async (error) => { + let message = error.message; + + if (message.includes("The user aborted a request.")) { + this.notify("Conversion was cancelled.", "warning"); + throw error; + } + + this.notify(message, "error"); + await closeProgressPopup(); + throw error; + }); + + conversionResults.forEach((info) => { + const { file } = info; + const fileName = file.split("/").pop(); + const [subject, session] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1); + const subRef = results[subject] ?? (results[subject] = {}); + subRef[session] = info; + }); + + await closeProgressPopup(); + + return results; + } + + // NOTE: Until the shadow DOM is supported in Storybook, we can't use this render function how we'd intend to. + addPage = (id, subpage) => { + if (!this.info.pages) this.info.pages = {}; + this.info.pages[id] = subpage; + this.updatePages(); + }; + + checkSyncState = async (info = this.info, sync = info.sync) => { + if (!sync) return; + if (isStorybook) return; + + const { desyncedData } = info.globalState; + + return Promise.all( + sync.map((k) => { + if (desyncedData?.[k] !== false) { + if (k === "conversion") return this.convert(); + else if (k === "preview") return this.convert({ preview: true }); + } + }) + ); + }; + + updateSections = () => { + const dashboard = document.querySelector("nwb-dashboard"); + dashboard.updateSections({ sidebar: true, main: true }, this.info.globalState); + }; + + #unsaved = false; + get unsavedUpdates() { + return this.#unsaved; + } + + set unsavedUpdates(value) { + this.#unsaved = !!value; + if (value === "conversions") this.info.globalState.desyncedData = { preview: true, conversion: true }; + } + + // NOTE: Make sure you call this explicitly if a child class overwrites this AND data is updated + updated() { + this.unsavedUpdates = false; + } + + render() { + return html``; + } +} + +customElements.get("nwbguide-page") || customElements.define("nwbguide-page", Page); diff --git a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js index f1cc066267..a48c4a46b5 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js @@ -1,268 +1,269 @@ -import { html } from "lit"; -import { Page } from "../../Page.js"; - -import { unsafeSVG } from "lit/directives/unsafe-svg.js"; -import folderOpenSVG from "../../../../../assets/icons/folder_open.svg?raw"; - -import { electron } from "../../../../../utils/electron.js"; -import { getSharedPath, removeFilePaths, truncateFilePaths } from "../../../preview/NWBFilePreview.js"; -const { ipcRenderer } = electron; -import { until } from "lit/directives/until.js"; -import { run } from "./utils.js"; -import { InspectorList, InspectorLegend } from "../../../preview/inspector/InspectorList.js"; -import { getStubArray } from "./GuidedStubPreview.js"; -import { InstanceManager } from "../../../InstanceManager.js"; -import { getMessageType } from "../../../../validation/index.js"; - -import { Button } from "../../../Button"; - -import { download } from "../../inspect/utils.js"; -import { createProgressPopup } from "../../../utils/progress.js"; -import { resolve } from "../../../../promises"; - -const filter = (list, toFilter) => { - return list.filter((item) => { - return Object.entries(toFilter) - .map(([key, strOrArray]) => { - return Array.isArray(strOrArray) - ? strOrArray.map((str) => item[key].includes(str)) - : item[key].includes(strOrArray); - }) - .flat() - .every(Boolean); - }); -}; - -const emptyMessage = "No issues detected in these files!"; - -export class GuidedInspectorPage extends Page { - constructor(...args) { - super(...args); - this.style.height = "100%"; // Fix main section - - Object.assign(this.style, { - display: "grid", - gridTemplateRows: "calc(100% - 120px) 1fr", - rowGap: "10px", - }); - } - - workflow = { - multiple_sessions: {}, - }; - - headerButtons = [ - new Button({ - label: "JSON", - primary: true, - }), - - new Button({ - label: "Text", - primary: true, - }), - ]; - - header = { - subtitle: `The NWB Inspector has scanned your files for adherence to best practices.`, - controls: () => [ - ...this.headerButtons, - html` { - if (ipcRenderer) - ipcRenderer.send( - "showItemInFolder", - getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file)) - ); - }} - >${unsafeSVG(folderOpenSVG)}`, - ], - }; - - // NOTE: We may want to trigger this whenever (1) this page is visited AND (2) data has been changed. - footer = {}; - - #toggleRendered; - #rendered; - #updateRendered = (force) => - force || this.#rendered === true - ? (this.#rendered = new Promise( - (resolve) => (this.#toggleRendered = () => resolve((this.#rendered = true))) - )) - : this.#rendered; - - get rendered() { - return resolve(this.#rendered, () => true); - } - - getStatus = (list) => { - return list.reduce((acc, messageInfo) => { - const res = getMessageType(messageInfo); - if (acc === "error") return acc; - else return res; - }, "valid"); - }; - - updated() { - const [downloadJSONButton, downloadTextButton] = this.headerButtons; - - downloadJSONButton.onClick = () => - download("nwb-inspector-report.json", { - header: this.report.header, - messages: this.report.messages, - }); - - downloadTextButton.onClick = () => download("nwb-inspector-report.txt", this.report.text); - } - - render() { - this.#updateRendered(true); - - const { globalState } = this.info; - const { stubs, inspector } = globalState.preview; - - const legendProps = { multiple: this.workflow.multiple_sessions.value }; - - const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user - const title = "Inspecting your file"; - - const fileArr = Object.entries(stubs) - .map(([subject, v]) => - Object.entries(v).map(([session, info]) => { - return { subject, session, info }; - }) - ) - .flat(); - return html` - ${until( - (async () => { - if (fileArr.length <= 1) { - this.report = inspector; - - if (!this.report) { - const result = await run( - "neuroconv/inspect_file", - { nwbfile_path: fileArr[0].info.file, ...options }, - { title } - ).catch((error) => { - this.notify(error.message, "error"); - return null; - }) - .finally(() => closeProgressPopup()) - - if (!result) return "Failed to generate inspector report."; - - this.report = globalState.preview.inspector = { - ...result, - messages: removeFilePaths(result.messages), - }; - } - - if (!inspector) await this.save(); - - const items = this.report.messages; - - const list = new InspectorList({ items, emptyMessage }); - - Object.assign(list.style, { - height: "100%", - }); - - return html`${list}${new InspectorLegend(legendProps)}`; - } - - const path = getSharedPath(fileArr.map(({ info }) => info.file)); - - this.report = inspector; - if (!this.report) { - const swalOpts = await createProgressPopup({ title: `${title}s` }); - - const { close: closeProgressPopup } = swalOpts; - - const result = await run( - "neuroconv/inspect_folder", - { path, ...options, request_id: swalOpts.id }, - swalOpts - ).catch(async (error) => { - this.notify(error.message, "error"); - await closeProgressPopup(); - return null; - }); - - if (!result) return "Failed to generate inspector report."; - - await closeProgressPopup(); - - this.report = globalState.preview.inspector = { - ...result, - messages: truncateFilePaths(result.messages, path), - }; - } - - if (!inspector) await this.save(); - - const messages = this.report.messages; - const items = truncateFilePaths(messages, path); - - const _instances = fileArr.map(({ subject, session, info }) => { - const file_path = [`sub-${subject}`, `sub-${subject}_ses-${session}`]; - const filtered = removeFilePaths(filter(items, { file_path })); - - const display = () => new InspectorList({ items: filtered, emptyMessage }); - display.status = this.getStatus(filtered); - - return { - subject, - session, - display, - }; - }); - - const instances = _instances.reduce((acc, { subject, session, display }) => { - const subLabel = `sub-${subject}`; - if (!acc[`sub-${subject}`]) acc[subLabel] = {}; - acc[subLabel][`ses-${session}`] = display; - return acc; - }, {}); - - Object.keys(instances).forEach((subLabel) => { - // const subItems = filter(items, { file_path: `${subLabel}${nodePath.sep}${subLabel}_ses-` }); // NOTE: This will not run on web-only now - const subItems = filter(items, { file_path: `${subLabel}_ses-` }); // NOTE: This will not run on web-only now - const path = getSharedPath(subItems.map((item) => item.file_path)); - const filtered = truncateFilePaths(subItems, path); - - const display = () => new InspectorList({ items: filtered, emptyMessage }); - display.status = this.getStatus(filtered); - - instances[subLabel] = { - ["All Files"]: display, - ...instances[subLabel], - }; - }); - - const allDisplay = () => new InspectorList({ items, emptyMessage }); - allDisplay.status = this.getStatus(items); - - const allInstances = { - ["All Files"]: allDisplay, - ...instances, - }; - - const manager = new InstanceManager({ - instances: allInstances, - }); - - return html`${manager}${new InspectorLegend(legendProps)}`; - })().finally(() => { - this.#toggleRendered(); - }), - "Loading inspector report..." - )} - `; - } -} - -customElements.get("nwbguide-guided-inspector-page") || - customElements.define("nwbguide-guided-inspector-page", GuidedInspectorPage); +import { html } from "lit"; +import { Page } from "../../Page.js"; + +import { unsafeSVG } from "lit/directives/unsafe-svg.js"; +import folderOpenSVG from "../../../../../assets/icons/folder_open.svg?raw"; + +import { electron } from "../../../../../utils/electron.js"; +import { getSharedPath, removeFilePaths, truncateFilePaths } from "../../../preview/NWBFilePreview.js"; +const { ipcRenderer } = electron; +import { until } from "lit/directives/until.js"; +import { run } from "./utils.js"; +import { InspectorList, InspectorLegend } from "../../../preview/inspector/InspectorList.js"; +import { getStubArray } from "./GuidedStubPreview.js"; +import { InstanceManager } from "../../../InstanceManager.js"; +import { getMessageType } from "../../../../validation/index.js"; + +import { Button } from "../../../Button"; + +import { download } from "../../inspect/utils.js"; +import { createProgressPopup } from "../../../utils/progress.js"; +import { resolve } from "../../../../promises"; + +const filter = (list, toFilter) => { + return list.filter((item) => { + return Object.entries(toFilter) + .map(([key, strOrArray]) => { + return Array.isArray(strOrArray) + ? strOrArray.map((str) => item[key].includes(str)) + : item[key].includes(strOrArray); + }) + .flat() + .every(Boolean); + }); +}; + +const emptyMessage = "No issues detected in these files!"; + +export class GuidedInspectorPage extends Page { + constructor(...args) { + super(...args); + this.style.height = "100%"; // Fix main section + + Object.assign(this.style, { + display: "grid", + gridTemplateRows: "calc(100% - 120px) 1fr", + rowGap: "10px", + }); + } + + workflow = { + multiple_sessions: {}, + }; + + headerButtons = [ + new Button({ + label: "JSON", + primary: true, + }), + + new Button({ + label: "Text", + primary: true, + }), + ]; + + header = { + subtitle: `The NWB Inspector has scanned your files for adherence to best practices.`, + controls: () => [ + ...this.headerButtons, + html` { + if (ipcRenderer) + ipcRenderer.send( + "showItemInFolder", + getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file)) + ); + }} + >${unsafeSVG(folderOpenSVG)}`, + ], + }; + + // NOTE: We may want to trigger this whenever (1) this page is visited AND (2) data has been changed. + footer = {}; + + #toggleRendered; + #rendered; + #updateRendered = (force) => + force || this.#rendered === true + ? (this.#rendered = new Promise( + (resolve) => (this.#toggleRendered = () => resolve((this.#rendered = true))) + )) + : this.#rendered; + + get rendered() { + return resolve(this.#rendered, () => true); + } + + getStatus = (list) => { + return list.reduce((acc, messageInfo) => { + const res = getMessageType(messageInfo); + if (acc === "error") return acc; + else return res; + }, "valid"); + }; + + updated() { + const [downloadJSONButton, downloadTextButton] = this.headerButtons; + + downloadJSONButton.onClick = () => + download("nwb-inspector-report.json", { + header: this.report.header, + messages: this.report.messages, + }); + + downloadTextButton.onClick = () => download("nwb-inspector-report.txt", this.report.text); + } + + render() { + this.#updateRendered(true); + + const { globalState } = this.info; + const { stubs, inspector } = globalState.preview; + + const legendProps = { multiple: this.workflow.multiple_sessions.value }; + + const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user + const title = "Inspecting your file"; + + const fileArr = Object.entries(stubs) + .map(([subject, v]) => + Object.entries(v).map(([session, info]) => { + return { subject, session, info }; + }) + ) + .flat(); + return html` + ${until( + (async () => { + if (fileArr.length <= 1) { + this.report = inspector; + + if (!this.report) { + const result = await run( + "neuroconv/inspect_file", + { nwbfile_path: fileArr[0].info.file, ...options }, + { title } + ) + .catch((error) => { + this.notify(error.message, "error"); + return null; + }) + .finally(() => closeProgressPopup()); + + if (!result) return "Failed to generate inspector report."; + + this.report = globalState.preview.inspector = { + ...result, + messages: removeFilePaths(result.messages), + }; + } + + if (!inspector) await this.save(); + + const items = this.report.messages; + + const list = new InspectorList({ items, emptyMessage }); + + Object.assign(list.style, { + height: "100%", + }); + + return html`${list}${new InspectorLegend(legendProps)}`; + } + + const path = getSharedPath(fileArr.map(({ info }) => info.file)); + + this.report = inspector; + if (!this.report) { + const swalOpts = await createProgressPopup({ title: `${title}s` }); + + const { close: closeProgressPopup } = swalOpts; + + const result = await run( + "neuroconv/inspect_folder", + { path, ...options, request_id: swalOpts.id }, + swalOpts + ).catch(async (error) => { + this.notify(error.message, "error"); + await closeProgressPopup(); + return null; + }); + + if (!result) return "Failed to generate inspector report."; + + await closeProgressPopup(); + + this.report = globalState.preview.inspector = { + ...result, + messages: truncateFilePaths(result.messages, path), + }; + } + + if (!inspector) await this.save(); + + const messages = this.report.messages; + const items = truncateFilePaths(messages, path); + + const _instances = fileArr.map(({ subject, session, info }) => { + const file_path = [`sub-${subject}`, `sub-${subject}_ses-${session}`]; + const filtered = removeFilePaths(filter(items, { file_path })); + + const display = () => new InspectorList({ items: filtered, emptyMessage }); + display.status = this.getStatus(filtered); + + return { + subject, + session, + display, + }; + }); + + const instances = _instances.reduce((acc, { subject, session, display }) => { + const subLabel = `sub-${subject}`; + if (!acc[`sub-${subject}`]) acc[subLabel] = {}; + acc[subLabel][`ses-${session}`] = display; + return acc; + }, {}); + + Object.keys(instances).forEach((subLabel) => { + // const subItems = filter(items, { file_path: `${subLabel}${nodePath.sep}${subLabel}_ses-` }); // NOTE: This will not run on web-only now + const subItems = filter(items, { file_path: `${subLabel}_ses-` }); // NOTE: This will not run on web-only now + const path = getSharedPath(subItems.map((item) => item.file_path)); + const filtered = truncateFilePaths(subItems, path); + + const display = () => new InspectorList({ items: filtered, emptyMessage }); + display.status = this.getStatus(filtered); + + instances[subLabel] = { + ["All Files"]: display, + ...instances[subLabel], + }; + }); + + const allDisplay = () => new InspectorList({ items, emptyMessage }); + allDisplay.status = this.getStatus(items); + + const allInstances = { + ["All Files"]: allDisplay, + ...instances, + }; + + const manager = new InstanceManager({ + instances: allInstances, + }); + + return html`${manager}${new InspectorLegend(legendProps)}`; + })().finally(() => { + this.#toggleRendered(); + }), + "Loading inspector report..." + )} + `; + } +} + +customElements.get("nwbguide-guided-inspector-page") || + customElements.define("nwbguide-guided-inspector-page", GuidedInspectorPage); diff --git a/src/electron/frontend/core/components/pages/inspect/InspectPage.js b/src/electron/frontend/core/components/pages/inspect/InspectPage.js index 905e108c00..08f3d090c3 100644 --- a/src/electron/frontend/core/components/pages/inspect/InspectPage.js +++ b/src/electron/frontend/core/components/pages/inspect/InspectPage.js @@ -1,170 +1,170 @@ -import { html } from "lit"; -import { Page } from "../Page.js"; -import { onThrow } from "../../../errors"; -import { Button } from "../../Button.js"; - -import { run } from "../guided-mode/options/utils.js"; -import { Modal } from "../../Modal"; -import { getSharedPath, truncateFilePaths } from "../../preview/NWBFilePreview.js"; -import { InspectorList, InspectorLegend } from "../../preview/inspector/InspectorList.js"; -import { download } from "./utils"; -import { createProgressPopup } from "../../utils/progress.js"; - -import { ready } from "../../../../../../schemas/dandi-upload.schema"; -import { JSONSchemaForm } from "../../JSONSchemaForm.js"; - -export class InspectPage extends Page { - constructor(...args) { - super(...args); - } - - header = { - title: "NWB File Validation", - subtitle: "Inspect NWB files using the NWB Inspector.", - }; - - inspect = async (paths, kwargs = {}, options = {}) => { - const swalOpts = await createProgressPopup({ - title: "Inspecting selected filesystem entries.", - ...options, - }); - - const { close: closeProgressPopup } = swalOpts; - const result = await run("neuroconv/inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch( - async (error) => { - this.notify(error.message, "error"); - await closeProgressPopup(); - throw error; - } - ); - - await closeProgressPopup(); - - if (typeof result === "string") return result; - - const { messages } = result; - - if (!messages.length) return this.notify("No messages received from the NWB Inspector"); - - return result; - }; - - showReport = async (value) => { - if (!value) { - const message = "Please provide filesystem entries to inspect."; - onThrow(message); - throw new Error(message); - } - - const legend = new InspectorLegend(); - - const kwargs = {}; - const nJobs = this.form.inputs["n_jobs"].value; - if (nJobs) kwargs.n_jobs = nJobs; - - const result = await this.inspect(value, kwargs); - - const messages = result.messages; - - const items = truncateFilePaths(messages, getSharedPath(messages.map((item) => item.file_path))); - - const list = new InspectorList({ items }); - - // const buttons = document.createElement('div') - // buttons.style.display = 'flex' - // buttons.style.gap = '10px' - - const downloadJSONButton = new Button({ - label: "JSON", - primary: true, - onClick: () => - download("nwb-inspector-report.json", { - header: result.header, - messages: result.messages, - }), - }); - - const downloadTextButton = new Button({ - label: "Text", - primary: true, - onClick: async () => { - download("nwb-inspector-report.txt", result.text); - }, - }); - - const modal = new Modal({ - header: value.length === 1 ? value : `Selected Filesystem Entries`, - controls: [downloadJSONButton, downloadTextButton], - footer: legend, - }); - - const container = document.createElement("div"); - Object.assign(container.style, { - display: "flex", - flexDirection: "column", - justifyContent: "space-between", - padding: "20px", - }); - - container.append(list); - - modal.append(container); - document.body.append(modal); - - modal.toggle(true); - }; - - form = new JSONSchemaForm({ - schema: { - properties: { - filesystem_paths: { - title: false, - type: "array", - items: { - type: "string", - format: ["file", "directory"], - multiple: true, - }, - }, - n_jobs: { - type: "number", - title: "Job Count", - description: "Number of parallel jobs to run. Leave blank to use all available cores.", - min: 1, - step: 1, - }, - }, - }, - showLabel: true, - onThrow, - }); - - updated() { - const urlFilePath = new URL(document.location).searchParams.get("file"); - - if (urlFilePath) { - this.showReport(urlFilePath); - this.form.inputs["filesystem_paths"].value = urlFilePath; - } - - ready.cpus.then(({ number_of_jobs }) => { - const nJobsInput = this.form.inputs["n_jobs"]; - nJobsInput.schema.max = number_of_jobs.max; - }); - } - - render() { - const button = new Button({ - label: "Start Inspection", - onClick: async () => this.showReport(this.form.inputs["filesystem_paths"].value), - }); - - return html` - ${this.form} -

- ${button} - `; - } -} - -customElements.get("nwbguide-inspect-page") || customElements.define("nwbguide-inspect-page", InspectPage); +import { html } from "lit"; +import { Page } from "../Page.js"; +import { onThrow } from "../../../errors"; +import { Button } from "../../Button.js"; + +import { run } from "../guided-mode/options/utils.js"; +import { Modal } from "../../Modal"; +import { getSharedPath, truncateFilePaths } from "../../preview/NWBFilePreview.js"; +import { InspectorList, InspectorLegend } from "../../preview/inspector/InspectorList.js"; +import { download } from "./utils"; +import { createProgressPopup } from "../../utils/progress.js"; + +import { ready } from "../../../../../../schemas/dandi-upload.schema"; +import { JSONSchemaForm } from "../../JSONSchemaForm.js"; + +export class InspectPage extends Page { + constructor(...args) { + super(...args); + } + + header = { + title: "NWB File Validation", + subtitle: "Inspect NWB files using the NWB Inspector.", + }; + + inspect = async (paths, kwargs = {}, options = {}) => { + const swalOpts = await createProgressPopup({ + title: "Inspecting selected filesystem entries.", + ...options, + }); + + const { close: closeProgressPopup } = swalOpts; + const result = await run("neuroconv/inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch( + async (error) => { + this.notify(error.message, "error"); + await closeProgressPopup(); + throw error; + } + ); + + await closeProgressPopup(); + + if (typeof result === "string") return result; + + const { messages } = result; + + if (!messages.length) return this.notify("No messages received from the NWB Inspector"); + + return result; + }; + + showReport = async (value) => { + if (!value) { + const message = "Please provide filesystem entries to inspect."; + onThrow(message); + throw new Error(message); + } + + const legend = new InspectorLegend(); + + const kwargs = {}; + const nJobs = this.form.inputs["n_jobs"].value; + if (nJobs) kwargs.n_jobs = nJobs; + + const result = await this.inspect(value, kwargs); + + const messages = result.messages; + + const items = truncateFilePaths(messages, getSharedPath(messages.map((item) => item.file_path))); + + const list = new InspectorList({ items }); + + // const buttons = document.createElement('div') + // buttons.style.display = 'flex' + // buttons.style.gap = '10px' + + const downloadJSONButton = new Button({ + label: "JSON", + primary: true, + onClick: () => + download("nwb-inspector-report.json", { + header: result.header, + messages: result.messages, + }), + }); + + const downloadTextButton = new Button({ + label: "Text", + primary: true, + onClick: async () => { + download("nwb-inspector-report.txt", result.text); + }, + }); + + const modal = new Modal({ + header: value.length === 1 ? value : `Selected Filesystem Entries`, + controls: [downloadJSONButton, downloadTextButton], + footer: legend, + }); + + const container = document.createElement("div"); + Object.assign(container.style, { + display: "flex", + flexDirection: "column", + justifyContent: "space-between", + padding: "20px", + }); + + container.append(list); + + modal.append(container); + document.body.append(modal); + + modal.toggle(true); + }; + + form = new JSONSchemaForm({ + schema: { + properties: { + filesystem_paths: { + title: false, + type: "array", + items: { + type: "string", + format: ["file", "directory"], + multiple: true, + }, + }, + n_jobs: { + type: "number", + title: "Job Count", + description: "Number of parallel jobs to run. Leave blank to use all available cores.", + min: 1, + step: 1, + }, + }, + }, + showLabel: true, + onThrow, + }); + + updated() { + const urlFilePath = new URL(document.location).searchParams.get("file"); + + if (urlFilePath) { + this.showReport(urlFilePath); + this.form.inputs["filesystem_paths"].value = urlFilePath; + } + + ready.cpus.then(({ number_of_jobs }) => { + const nJobsInput = this.form.inputs["n_jobs"]; + nJobsInput.schema.max = number_of_jobs.max; + }); + } + + render() { + const button = new Button({ + label: "Start Inspection", + onClick: async () => this.showReport(this.form.inputs["filesystem_paths"].value), + }); + + return html` + ${this.form} +

+ ${button} + `; + } +} + +customElements.get("nwbguide-inspect-page") || customElements.define("nwbguide-inspect-page", InspectPage); diff --git a/src/electron/frontend/core/components/pages/utils.js b/src/electron/frontend/core/components/pages/utils.js index e14d86b1f7..0947b2690f 100644 --- a/src/electron/frontend/core/components/pages/utils.js +++ b/src/electron/frontend/core/components/pages/utils.js @@ -1,77 +1,77 @@ -export const randomizeIndex = (count) => Math.floor(count * Math.random()); - -export const randomizeElements = (array, count) => { - if (count > array.length) throw new Error("Array size cannot be smaller than expected random numbers count."); - const result = []; - const guardian = new Set(); - while (result.length < count) { - const index = randomizeIndex(array.length); - if (guardian.has(index)) continue; - const element = array[index]; - guardian.add(index); - result.push(element); - } - return result; -}; - -const isObject = (item) => { - return item && typeof item === "object" && !Array.isArray(item); -}; - -export const setUndefinedIfNotDeclared = (schemaProps, resolved) => { - if ("properties" in schemaProps) schemaProps = schemaProps.properties; - for (const prop in schemaProps) { - const propInfo = schemaProps[prop]?.properties; - if (propInfo) setUndefinedIfNotDeclared(propInfo, resolved[prop]); - else if (!(prop in resolved)) resolved[prop] = undefined; - } -}; - -export const isPrivate = (k) => k.slice(0, 2) === "__"; - -export const sanitize = (item, condition = isPrivate) => { - if (isObject(item)) { - for (const [k, value] of Object.entries(item)) { - if (condition(k, value)) delete item[k]; - else sanitize(value, condition); - } - } else if (Array.isArray(item)) item.forEach((value) => sanitize(value, condition)); - - return item; -}; - -export function merge(toMerge = {}, target = {}, mergeOptions = {}) { - // Deep merge objects - for (const [k, value] of Object.entries(toMerge)) { - const targetValue = target[k]; - // if (isPrivate(k)) continue; - const arrayMergeMethod = mergeOptions.arrays; - if (arrayMergeMethod && Array.isArray(value) && Array.isArray(targetValue)) { - if (arrayMergeMethod === "append") - target[k] = [...targetValue, ...value]; // Append array entries together - else { - target[k] = targetValue.map((targetItem, i) => merge(value[i], targetItem, mergeOptions)); // Merge array entries - } - } else if (value === undefined) { - delete target[k]; // Remove matched values - // if (mergeOptions.remove !== false) delete target[k]; // Remove matched values - } else if (isObject(value)) { - if (isObject(targetValue)) target[k] = merge(value, targetValue, mergeOptions); - else { - if (mergeOptions.clone) - target[k] = merge(value, {}, mergeOptions); // Replace primitive values - else target[k] = value; // Replace object values - } - } else target[k] = value; // Replace primitive values - } - - return target; -} - -export function mapSessions(callback = (value) => value, toIterate = {}) { - return Object.entries(toIterate) - .map(([subject, sessions]) => { - return Object.entries(sessions).map(([session, info], i) => callback({ subject, session, info }, i)); - }) - .flat(2); -} +export const randomizeIndex = (count) => Math.floor(count * Math.random()); + +export const randomizeElements = (array, count) => { + if (count > array.length) throw new Error("Array size cannot be smaller than expected random numbers count."); + const result = []; + const guardian = new Set(); + while (result.length < count) { + const index = randomizeIndex(array.length); + if (guardian.has(index)) continue; + const element = array[index]; + guardian.add(index); + result.push(element); + } + return result; +}; + +const isObject = (item) => { + return item && typeof item === "object" && !Array.isArray(item); +}; + +export const setUndefinedIfNotDeclared = (schemaProps, resolved) => { + if ("properties" in schemaProps) schemaProps = schemaProps.properties; + for (const prop in schemaProps) { + const propInfo = schemaProps[prop]?.properties; + if (propInfo) setUndefinedIfNotDeclared(propInfo, resolved[prop]); + else if (!(prop in resolved)) resolved[prop] = undefined; + } +}; + +export const isPrivate = (k) => k.slice(0, 2) === "__"; + +export const sanitize = (item, condition = isPrivate) => { + if (isObject(item)) { + for (const [k, value] of Object.entries(item)) { + if (condition(k, value)) delete item[k]; + else sanitize(value, condition); + } + } else if (Array.isArray(item)) item.forEach((value) => sanitize(value, condition)); + + return item; +}; + +export function merge(toMerge = {}, target = {}, mergeOptions = {}) { + // Deep merge objects + for (const [k, value] of Object.entries(toMerge)) { + const targetValue = target[k]; + // if (isPrivate(k)) continue; + const arrayMergeMethod = mergeOptions.arrays; + if (arrayMergeMethod && Array.isArray(value) && Array.isArray(targetValue)) { + if (arrayMergeMethod === "append") + target[k] = [...targetValue, ...value]; // Append array entries together + else { + target[k] = targetValue.map((targetItem, i) => merge(value[i], targetItem, mergeOptions)); // Merge array entries + } + } else if (value === undefined) { + delete target[k]; // Remove matched values + // if (mergeOptions.remove !== false) delete target[k]; // Remove matched values + } else if (isObject(value)) { + if (isObject(targetValue)) target[k] = merge(value, targetValue, mergeOptions); + else { + if (mergeOptions.clone) + target[k] = merge(value, {}, mergeOptions); // Replace primitive values + else target[k] = value; // Replace object values + } + } else target[k] = value; // Replace primitive values + } + + return target; +} + +export function mapSessions(callback = (value) => value, toIterate = {}) { + return Object.entries(toIterate) + .map(([subject, sessions]) => { + return Object.entries(sessions).map(([session, info], i) => callback({ subject, session, info }, i)); + }) + .flat(2); +} diff --git a/src/electron/frontend/core/components/utils/progress.js b/src/electron/frontend/core/components/utils/progress.js index 3e9696d4ba..9d525057ac 100644 --- a/src/electron/frontend/core/components/utils/progress.js +++ b/src/electron/frontend/core/components/utils/progress.js @@ -1,120 +1,120 @@ -import { openProgressSwal } from "../pages/guided-mode/options/utils.js"; -import { ProgressBar } from "../ProgressBar"; -import { baseUrl } from "../../server/globals"; -import { createRandomString } from "../forms/utils"; - -export const createProgressPopup = async (options, tqdmCallback) => { - const cancelController = new AbortController(); - - if (!("showCancelButton" in options)) { - options.showCancelButton = true; - options.customClass = { actions: "swal-conversion-actions" }; - } - - const popup = await openProgressSwal(options, (result) => { - if (!result.isConfirmed) cancelController.abort(); - }); - - let elements = {}; - popup.hideLoading(); - const element = (elements.container = popup.getHtmlContainer()); - element.innerText = ""; - - Object.assign(element.style, { - marginTop: "5px", - }); - - const container = document.createElement("div"); - Object.assign(container.style, { - textAlign: "left", - display: "flex", - flexDirection: "column", - overflow: "hidden", - width: "100%", - gap: "5px", - }); - element.append(container); - - const bars = {}; - - const getBar = (id, large = false) => { - if (!bars[id]) { - const bar = new ProgressBar({ size: large ? undefined : "small" }); - bars[id] = bar; - container.append(bar); - } - return bars[id]; - }; - - const globalSymbol = Symbol("global"); - - elements.progress = getBar(globalSymbol, true); - - elements.bars = bars; - - const commonReturnValue = { swal: popup, fetch: { signal: cancelController.signal }, elements, ...options }; - - // Provide a default callback - let lastUpdate; - - const id = createRandomString(); - - const onProgressMessage = ({ data }) => { - const parsed = JSON.parse(data); - const { request_id, ...update } = parsed; - console.warn("parsed", parsed); - - if (request_id && request_id !== id) return; - lastUpdate = Date.now(); - - const _barId = parsed.progress_bar_id; - const barId = id === _barId ? globalSymbol : _barId; - const bar = getBar(barId); - if (!tqdmCallback) bar.format = parsed.format_dict; - else tqdmCallback(update); - }; - - progressHandler.addEventListener("message", onProgressMessage); - - const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - - const close = async () => { - if (lastUpdate) { - // const timeSinceLastUpdate = now - lastUpdate; - const animationLeft = 1000; // ProgressBar.animationDuration - timeSinceLastUpdate; // Add 100ms to ensure the animation has time to complete - if (animationLeft) await sleep(animationLeft); - } - - popup.close(); - - progressHandler.removeEventListener("message", onProgressMessage); - } - - return { ...commonReturnValue, id, close }; -}; - -const progressEventsUrl = new URL("/neuroconv/events/progress", baseUrl).href; - -class ProgressHandler { - constructor(props) { - const { url, callbacks, ...otherProps } = props; - - const source = (this.source = new EventSource(url)); - Object.assign(this, otherProps); - - source.addEventListener("error", this.onerror(), false); - - source.addEventListener("open", () => this.onopen(), false); - - source.addEventListener("message", (event) => this.onmessage(event), false); - } - - onopen = () => {}; - onmessage = () => {}; - onerror = () => {}; - - addEventListener = (...args) => this.source.addEventListener(...args); - removeEventListener = (...args) => this.source.removeEventListener(...args); -} - -export const progressHandler = new ProgressHandler({ url: progressEventsUrl }); +import { openProgressSwal } from "../pages/guided-mode/options/utils.js"; +import { ProgressBar } from "../ProgressBar"; +import { baseUrl } from "../../server/globals"; +import { createRandomString } from "../forms/utils"; + +export const createProgressPopup = async (options, tqdmCallback) => { + const cancelController = new AbortController(); + + if (!("showCancelButton" in options)) { + options.showCancelButton = true; + options.customClass = { actions: "swal-conversion-actions" }; + } + + const popup = await openProgressSwal(options, (result) => { + if (!result.isConfirmed) cancelController.abort(); + }); + + let elements = {}; + popup.hideLoading(); + const element = (elements.container = popup.getHtmlContainer()); + element.innerText = ""; + + Object.assign(element.style, { + marginTop: "5px", + }); + + const container = document.createElement("div"); + Object.assign(container.style, { + textAlign: "left", + display: "flex", + flexDirection: "column", + overflow: "hidden", + width: "100%", + gap: "5px", + }); + element.append(container); + + const bars = {}; + + const getBar = (id, large = false) => { + if (!bars[id]) { + const bar = new ProgressBar({ size: large ? undefined : "small" }); + bars[id] = bar; + container.append(bar); + } + return bars[id]; + }; + + const globalSymbol = Symbol("global"); + + elements.progress = getBar(globalSymbol, true); + + elements.bars = bars; + + const commonReturnValue = { swal: popup, fetch: { signal: cancelController.signal }, elements, ...options }; + + // Provide a default callback + let lastUpdate; + + const id = createRandomString(); + + const onProgressMessage = ({ data }) => { + const parsed = JSON.parse(data); + const { request_id, ...update } = parsed; + console.warn("parsed", parsed); + + if (request_id && request_id !== id) return; + lastUpdate = Date.now(); + + const _barId = parsed.progress_bar_id; + const barId = id === _barId ? globalSymbol : _barId; + const bar = getBar(barId); + if (!tqdmCallback) bar.format = parsed.format_dict; + else tqdmCallback(update); + }; + + progressHandler.addEventListener("message", onProgressMessage); + + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + const close = async () => { + if (lastUpdate) { + // const timeSinceLastUpdate = now - lastUpdate; + const animationLeft = 1000; // ProgressBar.animationDuration - timeSinceLastUpdate; // Add 100ms to ensure the animation has time to complete + if (animationLeft) await sleep(animationLeft); + } + + popup.close(); + + progressHandler.removeEventListener("message", onProgressMessage); + }; + + return { ...commonReturnValue, id, close }; +}; + +const progressEventsUrl = new URL("/neuroconv/events/progress", baseUrl).href; + +class ProgressHandler { + constructor(props) { + const { url, callbacks, ...otherProps } = props; + + const source = (this.source = new EventSource(url)); + Object.assign(this, otherProps); + + source.addEventListener("error", this.onerror(), false); + + source.addEventListener("open", () => this.onopen(), false); + + source.addEventListener("message", (event) => this.onmessage(event), false); + } + + onopen = () => {}; + onmessage = () => {}; + onerror = () => {}; + + addEventListener = (...args) => this.source.addEventListener(...args); + removeEventListener = (...args) => this.source.removeEventListener(...args); +} + +export const progressHandler = new ProgressHandler({ url: progressEventsUrl }); From 5fff8728517ccdf7ab2d6208cafea96b20309583 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 23:33:57 +0000 Subject: [PATCH 14/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyflask/namespaces/neuroconv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pyflask/namespaces/neuroconv.py b/src/pyflask/namespaces/neuroconv.py index 77b8d90124..24b16f3077 100644 --- a/src/pyflask/namespaces/neuroconv.py +++ b/src/pyflask/namespaces/neuroconv.py @@ -75,7 +75,7 @@ def post(self): class Convert(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): - + log_url = f"{request.url_root}log" url = f"{request.url_root}neuroconv/announce/progress" @@ -84,7 +84,8 @@ def post(self): **neuroconv_namespace.payload, log_url=log_url, ) - + + @neuroconv_namespace.route("/alignment") class Alignment(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) From 07c8d00a8ad3b5e7db535a1cadc94aede15d31d6 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 31 May 2024 07:44:48 -0700 Subject: [PATCH 15/23] Update GuidedInspectorPage.js --- .../guided-mode/options/GuidedInspectorPage.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js index a48c4a46b5..9a2809017b 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js @@ -140,18 +140,20 @@ export class GuidedInspectorPage extends Page { (async () => { if (fileArr.length <= 1) { this.report = inspector; + if (!this.report) { + + const result = await run( "neuroconv/inspect_file", { nwbfile_path: fileArr[0].info.file, ...options }, { title } ) - .catch((error) => { - this.notify(error.message, "error"); - return null; - }) - .finally(() => closeProgressPopup()); + .catch((error) => { + this.notify(error.message, "error"); + return null; + }) if (!result) return "Failed to generate inspector report."; @@ -188,13 +190,12 @@ export class GuidedInspectorPage extends Page { swalOpts ).catch(async (error) => { this.notify(error.message, "error"); - await closeProgressPopup(); return null; - }); + }) + .finally(() => closeProgressPopup()); if (!result) return "Failed to generate inspector report."; - await closeProgressPopup(); this.report = globalState.preview.inspector = { ...result, From 4e07964f698c207afa4a43f8c3f57de8575308d8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 14:45:06 +0000 Subject: [PATCH 16/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../options/GuidedInspectorPage.js | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js index 9a2809017b..036712a9fa 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js @@ -140,20 +140,16 @@ export class GuidedInspectorPage extends Page { (async () => { if (fileArr.length <= 1) { this.report = inspector; - if (!this.report) { - - const result = await run( "neuroconv/inspect_file", { nwbfile_path: fileArr[0].info.file, ...options }, { title } - ) - .catch((error) => { + ).catch((error) => { this.notify(error.message, "error"); return null; - }) + }); if (!result) return "Failed to generate inspector report."; @@ -188,15 +184,15 @@ export class GuidedInspectorPage extends Page { "neuroconv/inspect_folder", { path, ...options, request_id: swalOpts.id }, swalOpts - ).catch(async (error) => { - this.notify(error.message, "error"); - return null; - }) - .finally(() => closeProgressPopup()); + ) + .catch(async (error) => { + this.notify(error.message, "error"); + return null; + }) + .finally(() => closeProgressPopup()); if (!result) return "Failed to generate inspector report."; - this.report = globalState.preview.inspector = { ...result, messages: truncateFilePaths(result.messages, path), From 837b71d8229f529aa839da0c5e0daf1ca47c2c43 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 31 May 2024 12:09:51 -0700 Subject: [PATCH 17/23] Update pipelines.test.ts --- tests/e2e/pipelines.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index c775916bc4..bf38df0b90 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -119,6 +119,7 @@ describe('Run example pipelines', () => { await dashboard.next() await dashboard.page.rendered const id = dashboard.page.info.id + takeScreenshot(join('test-pipelines', 'conversion', toMatch, id)) if (id === '//conversion') break // Conversion page is the last page } catch (e) { await toHome() From 3234e760168992b1e741f5604d2a4487615557fe Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 31 May 2024 12:42:22 -0700 Subject: [PATCH 18/23] Update pipelines.test.ts --- tests/e2e/pipelines.test.ts | 182 +++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 85 deletions(-) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index bf38df0b90..5b4acfaea0 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -24,121 +24,133 @@ beforeAll(() => initTests({ screenshots: false, data: false })) describe('Run example pipelines', () => { - test('Ensure test data is present', () => { - expect(existsSync(testGINPath)).toBe(true) - }) + test('Ensure test data is present', () => { + expect(existsSync(testGINPath)).toBe(true) + }) - test('Ensure number of example pipelines starts at zero', async () => { - const nPipelines = await evaluate(() => document.getElementById('guided-div-resume-progress-cards').children.length) - expect(nPipelines).toBe(0) - }) + test('Ensure number of example pipelines starts at zero', async () => { + const nPipelines = await evaluate(() => document.getElementById('guided-div-resume-progress-cards').children.length) + expect(nPipelines).toBe(0) + }) - // NOTE: The following code is dependent on the presence of test data on the user's computer - pipelineDescribeFn('Generate and run pipeline from YAML file', () => { + // NOTE: The following code is dependent on the presence of test data on the user's computer + pipelineDescribeFn('Generate and run pipeline from YAML file', () => { - test('All example pipelines are created', async ( ) => { + test('All example pipelines are created', async () => { - const result = await evaluate(async (testGINPath) => { + const result = await evaluate(async (testGINPath) => { - // Transition to settings page - const dashboard = document.querySelector('nwb-dashboard') - dashboard.sidebar.select('settings') - await new Promise(resolve => setTimeout(resolve, 200)) + // Transition to settings page + const dashboard = document.querySelector('nwb-dashboard') + dashboard.sidebar.select('settings') + await new Promise(resolve => setTimeout(resolve, 200)) - const page = dashboard.page + const page = dashboard.page - // Open relevant accordion - const accordion = page.form.accordions['developer'] - accordion.toggle(true) + // Open relevant accordion + const accordion = page.form.accordions['developer'] + accordion.toggle(true) - // Generate example pipelines - const folderInput = page.form.getFormElement(["developer", "testing_data_folder"]) - folderInput.updateData(testGINPath) + // Generate example pipelines + const folderInput = page.form.getFormElement(["developer", "testing_data_folder"]) + folderInput.updateData(testGINPath) - const button = folderInput.nextSibling - const results = await button.onClick() + const button = folderInput.nextSibling + const results = await button.onClick() - page.save() - page.dismiss() // Dismiss any notifications + page.save() + page.dismiss() // Dismiss any notifications - return results + return results - }, testGINPath) + }, testGINPath) - await sleep(500) // Wait for notification to dismiss + await sleep(500) // Wait for notification to dismiss - const allPipelineNames = Object.keys(examplePipelines).reverse() + const allPipelineNames = Object.keys(examplePipelines).reverse() - expect(result).toEqual(allPipelineNames.map(name => { return {name, success: true} })) + expect(result).toEqual(allPipelineNames.map(name => { return { name, success: true } })) - await takeScreenshot(join('test-pipelines', 'created')) + await takeScreenshot(join('test-pipelines', 'created')) - // Transition back to the conversions page and count pipelines - const renderedPipelineNames = await evaluate(async () => { - const dashboard = document.querySelector('nwb-dashboard') - dashboard.sidebar.select('/') - await new Promise(resolve => setTimeout(resolve, 200)) - const pipelineCards = document.getElementById('guided-div-resume-progress-cards').children - return Array.from(pipelineCards).map(card => card.info.project.name) - }) + // Transition back to the conversions page and count pipelines + const renderedPipelineNames = await evaluate(async () => { + const dashboard = document.querySelector('nwb-dashboard') + dashboard.sidebar.select('/') + await new Promise(resolve => setTimeout(resolve, 200)) + const pipelineCards = document.getElementById('guided-div-resume-progress-cards').children + return Array.from(pipelineCards).map(card => card.info.project.name) + }) - await takeScreenshot(join('test-pipelines', 'list'), 500) + await takeScreenshot(join('test-pipelines', 'list'), 500) - // Assert all the pipelines are present - expect(renderedPipelineNames.sort()).toEqual(allPipelineNames.map(header).sort()) + // Assert all the pipelines are present + expect(renderedPipelineNames.sort()).toEqual(allPipelineNames.map(header).sort()) - }) + }) - for (let pipeline in examplePipelines) { + for (let pipeline in examplePipelines) { - const pipelineParsed = header(pipeline) - const info = examplePipelines[pipeline] - const describeFn = info.test === false ? describe.skip : describe + const pipelineParsed = header(pipeline) + const info = examplePipelines[pipeline] + const describeFn = info.test === false ? describe.skip : describe - describeFn(`${pipelineParsed}`, async ( ) => { + describeFn(`${pipelineParsed}`, async () => { - test(`Full conversion completed`, async () => { - const wasFound = await evaluate(async (toMatch) => { - const dashboard = document.querySelector('nwb-dashboard') - const pipelines = document.getElementById('guided-div-resume-progress-cards').children - const found = Array.from(pipelines).find(card => card.info.project.name === toMatch) - if (found) { - found.querySelector('button')!.click() - - const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) - const toHome = () => dashboard.page.to('/') - - // Update this with a while loop to advance through the pipeline until back at the home page - while (document.querySelector('nwb-dashboard').page.info.id !== '//'){ - await sleep(100) - try { - await dashboard.next() - await dashboard.page.rendered - const id = dashboard.page.info.id - takeScreenshot(join('test-pipelines', 'conversion', toMatch, id)) - if (id === '//conversion') break // Conversion page is the last page - } catch (e) { - await toHome() - return e.message // Return error - } - } - - await toHome() - - return true - } - }, pipelineParsed) - - expect(wasFound).toBe(true) + test(`Full conversion completed`, async () => { + + // Start pipeline + const started = await evaluate(async (toMatch) => { + // const dashboard = document.querySelector('nwb-dashboard') + // if (dashboard.page.info.id !== '//') await dashboard.page.to('/') // Go back to home page + const pipelines = document.getElementById('guided-div-resume-progress-cards').children + const found = Array.from(pipelines).find(card => card.info.project.name === toMatch) + if (found) found.querySelector('button')!.click() + return !!found + }, pipelineParsed) + + expect(started).toBe(true) + + let pageId; + + while (pageId !== '/') { + pageId = await evaluate(async () => { + const dashboard = document.querySelector('nwb-dashboard') + + const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + + // Update this with a while loop to advance through the pipeline until back at the home page + await sleep(100) + + try { + await dashboard.next() + await dashboard.page.rendered + } catch (e) { await dashboard.page.to('/') } // Go back to the home page if error + + return dashboard.page.info.id }) - }) + takeScreenshot(join('test-pipelines', 'conversion', pipelineParsed, pageId)) - } + // Stop the pipeline. Conversion page is the last page + if (pageId === '//conversion') { + pageId = await evaluate(async () => { + const dashboard = document.querySelector('nwb-dashboard') + await dashboard.page.to('/') + return dashboard.page.info.id + }) - }) + expect(pageId).toBe('/') // Will break while loop if failed + + } + } + + }) + }) + } + }) }) From 7576a77401a815911d2c5f70dd2ef63b5b0b1b3b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 19:42:39 +0000 Subject: [PATCH 19/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/e2e/pipelines.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 5b4acfaea0..486adb4083 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -145,7 +145,7 @@ describe('Run example pipelines', () => { }) expect(pageId).toBe('/') // Will break while loop if failed - + } } From 8febb0eeb1ae1097bb9354ff201ecd11a8ba82fd Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 31 May 2024 14:17:09 -0700 Subject: [PATCH 20/23] Update synced data check and awaiting screenshot --- src/electron/frontend/core/components/pages/Page.js | 11 ++++------- tests/e2e/pipelines.test.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/electron/frontend/core/components/pages/Page.js b/src/electron/frontend/core/components/pages/Page.js index 7f107be2d9..e883814ef7 100644 --- a/src/electron/frontend/core/components/pages/Page.js +++ b/src/electron/frontend/core/components/pages/Page.js @@ -136,13 +136,10 @@ export class Page extends LitElement { this.unsavedUpdates = true; // Indicate conversion has run successfully - const { desyncedData } = this.info.globalState; - if (!desyncedData) this.info.globalState.desyncedData = {}; - - if (desyncedData) { - desyncedData[key] = false; - await this.save({}, false); - } + let { desyncedData } = this.info.globalState; + if (!desyncedData) desyncedData = this.info.globalState.desyncedData = {}; + desyncedData[key] = false; + await this.save({}, false); } async runConversions(conversionOptions = {}, toRun, options = {}) { diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 5b4acfaea0..7a4aeb2f2f 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -129,15 +129,18 @@ describe('Run example pipelines', () => { try { await dashboard.next() await dashboard.page.rendered - } catch (e) { await dashboard.page.to('/') } // Go back to the home page if error + } catch (e) { + await dashboard.page.to('/') + } // Go back to the home page if error return dashboard.page.info.id }) - takeScreenshot(join('test-pipelines', 'conversion', pipelineParsed, pageId)) + await takeScreenshot(join('test-pipelines', 'conversion', pipelineParsed, pageId)) // Stop the pipeline. Conversion page is the last page if (pageId === '//conversion') { + pageId = await evaluate(async () => { const dashboard = document.querySelector('nwb-dashboard') await dashboard.page.to('/') @@ -145,6 +148,7 @@ describe('Run example pipelines', () => { }) expect(pageId).toBe('/') // Will break while loop if failed + break } } From 54ea4c99e21a872c6f016b1472312945e5991f2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 21:19:07 +0000 Subject: [PATCH 21/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/e2e/pipelines.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 7a4aeb2f2f..1c1645f78d 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -129,8 +129,8 @@ describe('Run example pipelines', () => { try { await dashboard.next() await dashboard.page.rendered - } catch (e) { - await dashboard.page.to('/') + } catch (e) { + await dashboard.page.to('/') } // Go back to the home page if error return dashboard.page.info.id @@ -149,7 +149,7 @@ describe('Run example pipelines', () => { expect(pageId).toBe('/') // Will break while loop if failed break - + } } From cbf63434cd77afd9a12b5040a836c2b06bebccf1 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 31 May 2024 14:43:16 -0700 Subject: [PATCH 22/23] Take screenshots during long-running events --- tests/e2e/pipelines.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 7a4aeb2f2f..73dc24df11 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -118,7 +118,10 @@ describe('Run example pipelines', () => { let pageId; while (pageId !== '/') { - pageId = await evaluate(async () => { + + + + const promise = evaluate(async () => { const dashboard = document.querySelector('nwb-dashboard') const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) @@ -136,6 +139,16 @@ describe('Run example pipelines', () => { return dashboard.page.info.id }) + let nImages = 0 + const intervalId = setInterval(async () => { + if (pageId) await takeScreenshot(join('test-pipelines', 'conversion', pipelineParsed, `${pageId}-after-${nImages++}`)) + }, 1000) + + pageId = await promise.catch(e => { + console.error(e) + }) + .finally(() => clearInterval(intervalId)) + await takeScreenshot(join('test-pipelines', 'conversion', pipelineParsed, pageId)) // Stop the pipeline. Conversion page is the last page From 1753aac91c7013e1c80fee58cc32ec4ed9484476 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 21:43:41 +0000 Subject: [PATCH 23/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/e2e/pipelines.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 0c3a64a144..290ebd10a8 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -119,7 +119,7 @@ describe('Run example pipelines', () => { while (pageId !== '/') { - + const promise = evaluate(async () => { const dashboard = document.querySelector('nwb-dashboard')