diff --git a/CMakeLists.txt b/CMakeLists.txt index bff07b059a..e5cbe6daaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,9 +461,11 @@ set(SIMPLNX_HDRS ${SIMPLNX_SOURCE_DIR}/Parameters/util/DynamicTableInfo.hpp ${SIMPLNX_SOURCE_DIR}/Parameters/util/ReadCSVData.hpp + ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineFilter.hpp ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineNode.hpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Pipeline.hpp ${SIMPLNX_SOURCE_DIR}/Pipeline/PipelineFilter.hpp + ${SIMPLNX_SOURCE_DIR}/Pipeline/PlaceholderFilter.hpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/AbstractPipelineMessage.hpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/FilterPreflightMessage.hpp @@ -664,9 +666,11 @@ set(SIMPLNX_SRCS ${SIMPLNX_SOURCE_DIR}/Parameters/util/ReadCSVData.cpp ${SIMPLNX_SOURCE_DIR}/Parameters/util/DynamicTableInfo.cpp + ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineFilter.cpp ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineNode.cpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Pipeline.cpp ${SIMPLNX_SOURCE_DIR}/Pipeline/PipelineFilter.cpp + ${SIMPLNX_SOURCE_DIR}/Pipeline/PlaceholderFilter.cpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/AbstractPipelineMessage.cpp ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/FilterPreflightMessage.cpp diff --git a/src/Plugins/SimplnxCore/CMakeLists.txt b/src/Plugins/SimplnxCore/CMakeLists.txt index 18f980ca44..8023a7b851 100644 --- a/src/Plugins/SimplnxCore/CMakeLists.txt +++ b/src/Plugins/SimplnxCore/CMakeLists.txt @@ -368,18 +368,6 @@ string(HEX ${PYTHON_PLUGIN_TEMPLATE} PYTHON_PLUGIN_TEMPLATE) string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE ${PYTHON_PLUGIN_TEMPLATE}) string(APPEND PYTHON_PLUGIN_TEMPLATE "0x00") -if(WINDOWS) - file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginTemplate.bat" PYTHON_PLUGIN_TEMPLATE_BAT) - string(HEX ${PYTHON_PLUGIN_TEMPLATE_BAT} PYTHON_PLUGIN_TEMPLATE_BAT) - string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE_BAT ${PYTHON_PLUGIN_TEMPLATE_BAT}) - string(APPEND PYTHON_PLUGIN_TEMPLATE_BAT "0x00") -else() - file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginTemplate.sh" PYTHON_PLUGIN_TEMPLATE_BAT) - string(HEX ${PYTHON_PLUGIN_TEMPLATE_BAT} PYTHON_PLUGIN_TEMPLATE_BAT) - string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE_BAT ${PYTHON_PLUGIN_TEMPLATE_BAT}) - string(APPEND PYTHON_PLUGIN_TEMPLATE_BAT "0x00") -endif() - cmpConfigureFileWithMD5Check(CONFIGURED_TEMPLATE_PATH "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.in.hpp" GENERATED_FILE_PATH "${${PLUGIN_NAME}_BINARY_DIR}/generated/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.hpp") diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.cpp index f434961b4d..3aabb40867 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.cpp @@ -35,7 +35,6 @@ Result<> GeneratePythonSkeleton::operator()() } else { - return nx::core::WritePythonPluginFiles(m_InputValues->pluginOutputDir, m_InputValues->pluginName, m_InputValues->pluginName, "Description", m_InputValues->filterNames, - m_InputValues->createBatchShellScript, m_InputValues->anacondaEnvName); + return nx::core::WritePythonPluginFiles(m_InputValues->pluginOutputDir, m_InputValues->pluginName, m_InputValues->pluginName, "Description", m_InputValues->filterNames); } } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.hpp index 12cdc5072b..cc12e92604 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.hpp @@ -20,8 +20,6 @@ struct SIMPLNXCORE_EXPORT GeneratePythonSkeletonInputValues std::string pluginName; std::string pluginHumanName; std::string filterNames; - bool createBatchShellScript; - std::string anacondaEnvName; }; /** diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.cpp index 10bc6555e3..7b208d263b 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.cpp @@ -66,15 +66,10 @@ Parameters GeneratePythonSkeletonFilter::parameters() const params.insert( std::make_unique(k_PluginFilterNames, "Filter Names (comma-separated)", "The names of filters that will be created, separated by commas (,).", "FirstFilter,SecondFilter")); - params.insertLinkableParameter( - std::make_unique(k_CreateBatchFile_Key, "Create Anaconda Init Batch/Shell Script", "Generates a script file that can be used to export needed environment variables", false)); - params.insert(std::make_unique(k_AnacondaEnvName_Key, "Anaconda Environment Name", "The name of the Anaconda environment.", "nxpython")); - params.linkParameters(k_UseExistingPlugin_Key, k_PluginName_Key, false); params.linkParameters(k_UseExistingPlugin_Key, k_PluginHumanName_Key, false); params.linkParameters(k_UseExistingPlugin_Key, k_PluginOutputDirectory_Key, false); params.linkParameters(k_UseExistingPlugin_Key, k_PluginInputDirectory_Key, true); - params.linkParameters(k_CreateBatchFile_Key, k_AnacondaEnvName_Key, true); return params; } @@ -99,20 +94,41 @@ IFilter::PreflightResult GeneratePythonSkeletonFilter::preflightImpl(const DataS auto filterList = StringUtilities::split(filterNames, ','); + std::stringstream preflightUpdatedValue; + std::string pluginPath = fmt::format("{}{}{}", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName); if(useExistingPlugin) { pluginPath = pluginInputDir.string(); } + std::string fullPath = fmt::format("{}{}{}{}Plugin.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + if(std::filesystem::exists({fullPath})) + { + fullPath = "[REPLACE]: " + fullPath; + } + else + { + fullPath = "[New]: " + fullPath; + } + preflightUpdatedValue << fullPath << '\n'; - std::stringstream preflightUpdatedValue; + fullPath = fmt::format("{}{}{}{}__init__.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator}); + if(std::filesystem::exists({fullPath})) + { + fullPath = "[REPLACE]: " + fullPath; + } + else + { + fullPath = "[New]: " + fullPath; + } + preflightUpdatedValue << fullPath << '\n'; for(const auto& filterName : filterList) { - std::string fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName); + fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName); if(std::filesystem::exists({fullPath})) { - fullPath = "[EXISTS]: " + fullPath; + fullPath = "[REPLACE]: " + fullPath; } else { @@ -139,8 +155,6 @@ Result<> GeneratePythonSkeletonFilter::executeImpl(DataStructure& dataStructure, inputValues.pluginName = filterArgs.value(k_PluginName_Key); inputValues.pluginHumanName = filterArgs.value(k_PluginHumanName_Key); inputValues.filterNames = filterArgs.value(k_PluginFilterNames); - inputValues.createBatchShellScript = filterArgs.value(k_CreateBatchFile_Key); - inputValues.anacondaEnvName = filterArgs.value(k_AnacondaEnvName_Key); return GeneratePythonSkeleton(dataStructure, messageHandler, shouldCancel, &inputValues)(); } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.hpp index 1e125394bf..d062e4b582 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.hpp @@ -29,8 +29,6 @@ class SIMPLNXCORE_EXPORT GeneratePythonSkeletonFilter : public IFilter static inline constexpr StringLiteral k_PluginHumanName_Key = "plugin_human_name"; static inline constexpr StringLiteral k_PluginInputDirectory_Key = "plugin_input_directory"; static inline constexpr StringLiteral k_PluginOutputDirectory_Key = "plugin_output_directory"; - static inline constexpr StringLiteral k_CreateBatchFile_Key = "create_batch_shell_script"; - static inline constexpr StringLiteral k_AnacondaEnvName_Key = "anaconda_env_name"; static inline constexpr StringLiteral k_PluginFilterNames = "filter_names"; /** diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonFilterTemplate.py b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonFilterTemplate.py index 1f1cc848fa..5068ed61ef 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonFilterTemplate.py +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonFilterTemplate.py @@ -2,11 +2,6 @@ import simplnx as nx class #PYTHON_FILTER_NAME#: - """ - This section should contain the 'keys' that store each parameter. The value of the key should be snake_case. The name - of the value should be ALL_CAPITOL_KEY - """ - TEST_KEY = 'test' # ----------------------------------------------------------------------------- # These methods should not be edited @@ -59,35 +54,91 @@ def default_tags(self) -> List[str]: :rtype: list """ return ['python', '#PYTHON_FILTER_HUMAN_NAME#'] - + + + """ + This section should contain the 'keys' that store each parameter. The value of the key should be snake_case. The name + of the value should be ALL_CAPITOL_KEY + """ + ARRAY_PATH_KEY = 'output_array_path' + NUM_TUPLES_KEY = 'num_tuples' + def parameters(self) -> nx.Parameters: - """This function defines the parameters that are needed by the filter. Parameters collect the values from the user - or through a pipeline file. + """This function defines the parameters that are needed by the filter. Parameters collect the values from the user interface + and pack them up into a dictionary for use in the preflight and execute methods. """ params = nx.Parameters() - params.insert(nx.Float64Parameter(#PYTHON_FILTER_NAME#.TEST_KEY, 'Test', '', 0.0)) + params.insert(nx.ArrayCreationParameter(#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY, 'Created Array', 'Array storing the data', nx.DataPath())) + + params.insert(nx.UInt64Parameter(#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY, 'Num Tuples', 'The number of tuples the array will have', 0)) return params def preflight_impl(self, data_structure: nx.DataStructure, args: dict, message_handler: nx.IFilter.MessageHandler, should_cancel: nx.AtomicBoolProxy) -> nx.IFilter.PreflightResult: """This method preflights the filter and should ensure that all inputs are sanity checked as best as possible. Array - sizes can be checked if the arrays are actually know at preflight time. Some filters will not be able to report output - array sizes during preflight (segmentation filters for example). + sizes can be checked if the array sizes are actually known at preflight time. Some filters will not be able to report output + array sizes during preflight (segmentation filters for example). If in doubt, set the tuple dimensions of an array to [1]. :returns: :rtype: nx.IFilter.PreflightResult """ - value: float = args[#PYTHON_FILTER_NAME#.TEST_KEY] - message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Preflight: {value}')) - return nx.IFilter.PreflightResult() + + # Extract the values from the user interface from the 'args' + data_array_path: nx.DataPath = args[#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY] + num_tuples: int = args[#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY] + + # Create an OutputActions object to hold any DataStructure modifications that we are going to make + output_actions = nx.OutputActions() + + # Create the Errors and Warnings Lists to commuicate back to the user if anything has gone wrong + # errors = [] + # warnings = [] + # preflight_values = [] + + # Validate that the number of tuples > 0, otherwise return immediately with an error message + if num_tuples == 0: + return nx.IFilter.PreflightResult(nx.OutputActions(), [nx.Error(-65020, "The number of tuples should be at least 1.")]) + + # Append a "CreateArrayAction" + output_actions.append_action(nx.CreateArrayAction(nx.DataType.float32, [num_tuples], [1], data_array_path)) + + # Send back any messages that will appear in the "Output" widget in the UI. This is optional. + message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f"Creating array at: '{data_array_path.to_string('/')}' with {num_tuples} tuples")) + + # Return the output_actions so the changes are reflected in the User Interface. + return nx.IFilter.PreflightResult(output_actions=output_actions, errors=None, warnings=None, preflight_values=None) def execute_impl(self, data_structure: nx.DataStructure, args: dict, message_handler: nx.IFilter.MessageHandler, should_cancel: nx.AtomicBoolProxy) -> nx.IFilter.ExecuteResult: """ This method actually executes the filter algorithm and reports results. :returns: :rtype: nx.IFilter.ExecuteResult """ + # Extract the values from the user interface from the 'args' + # This is basically repeated from the preflight because the variables are scoped to the method() + data_array_path: nx.DataPath = args[#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY] + num_tuples: int = args[#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY] + + # At this point the array has been allocated with the proper number of tuples and components. And we can access + # the data array through a numpy view. + data_array_view = data_structure[data_array_path].npview() + # Now you can go off and use numpy or anything else that can use a numpy view to modify the data + # or use the data in another calculation. Any operation that works on the numpy view in-place + # has an immediate effect within the DataStructure + + # ----------------------------------------------------------------------------- + # If you want to send back progress on your filter, you can use the message_handler + # ----------------------------------------------------------------------------- + message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Information Message: Num_Tuples = {num_tuples}')) + + # ----------------------------------------------------------------------------- + # If you have a long running process, check the should_cancel to see if the user cancelled the filter + # ----------------------------------------------------------------------------- + if not should_cancel: + return nx.Result() + - value: float = args[#PYTHON_FILTER_NAME#.TEST_KEY] - message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Execute: {value}')) return nx.Result() + + + diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginInitTemplate.py b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginInitTemplate.py index 3cd33c9511..d0a232247e 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginInitTemplate.py +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginInitTemplate.py @@ -1,9 +1,16 @@ + +""" +Insert documentation here for #PLUGIN_NAME# +""" from #PLUGIN_NAME#.Plugin import #PLUGIN_NAME# -#PLUGIN_IMPORT_CODE## FILTER_INCLUDE_INSERT +__all__ = ['#PLUGIN_NAME#', 'get_plugin'] -def get_plugin(): - return #PLUGIN_NAME#() +""" +This section conditionally tries to import each filter +""" -__all__ = [#PLUGIN_FILTER_LIST#] # FILTER_NAME_INSERT +#PLUGIN_IMPORT_CODE# +def get_plugin(): + return #PLUGIN_NAME#() diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp index e73e41ec6b..727c4f5ac6 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp @@ -37,13 +37,4 @@ inline const std::string PluginInitPythonFile() return {k_PluginInitPythonFileCharArray}; } -// clang-format off -static const char k_PluginBatchFileCharArray[] = {@PYTHON_PLUGIN_TEMPLATE_BAT@}; -// clang-format on - -inline const std::string PluginBatchFile() -{ - return {k_PluginBatchFileCharArray}; -} - }; // namespace nx::core \ No newline at end of file diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.bat b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.bat deleted file mode 100644 index 374c5147b5..0000000000 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.bat +++ /dev/null @@ -1,27 +0,0 @@ -:: This batch file can be run to activate a specific python virtual environment -:: and set the necessary envionment variables to load the DREAM3D-NX Plugin - -:: If you launch an Anaconda Prompt you can then run this batch file to -:: activate your environment and set the needed variables. If you have installed -:: your anaconda environment into a non-default location then you will need to -:: modify the below line with this command instead, being sure to substitue -:: the proper path to the environment. -:: conda activate --prefix C:/conda3/envs/pyd3d - -:: conda activate @ANACONDA_ENV_NAME@ - -:: ---------------------------------------------------------------------------- -:: The first variable to set is the PYTHONPATH which should point to a -:: directory or directories that contains the top level python plugin -:: folders. If you need to list mulitple directories to search for plugins -:: you can use the ";" character to separate those paths. - -set PYTHONPATH=@PYTHONPATH@ - -:: ---------------------------------------------------------------------------- -:: The next variable is a list of all plugins that you would like to -:: load when DREAM3D-NX (or NXRunner) is run. If you have multiple plugins that -:: you would like to load, list them all separated by a ";" character. - -set SIMPLNX_PYTHON_PLUGINS=@SIMPLNX_PYTHON_PLUGINS@ - diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.py b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.py index 9d443d32e2..adfdfd605d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.py +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.py @@ -2,26 +2,51 @@ Insert documentation here. """ -#PLUGIN_IMPORT_CODE## FILTER_INCLUDE_INSERT +_filters = [] + +""" +This section conditionally tries to import each filter +""" + +#PLUGIN_IMPORT_CODE# import simplnx as nx class #PLUGIN_NAME#: + """ + This class defines the plugin's basic information. + """ def __init__(self) -> None: pass def id(self) -> nx.Uuid: + """This returns the UUID of the filter. Each Plugin has a unique UUID value. DO NOT change this. + :return: The Plugins's Uuid value + :rtype: string + """ return nx.Uuid('#PLUGIN_UUID#') def name(self) -> str: + """The returns the name of plugin. DO NOT Change this + :return: The name of the plugin + :rtype: string + """ return '#PLUGIN_NAME#' def description(self) -> str: + """This returns the description of the plugin. Feel free to edit this. + :return: The plugin's descriptive text + :rtype: string + """ return '#PLUGIN_SHORT_NAME#' def vendor(self) -> str: + """This returns the name of the organization that is writing the plugin. Feel free to edit this. + :return: The plugin's organization + :rtype: string + """ return '#PLUGIN_DESCRIPTION#' def get_filters(self): - return [#PLUGIN_FILTER_LIST#] # FILTER_NAME_INSERT + return _filters diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.sh b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.sh deleted file mode 100644 index dd631a57aa..0000000000 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -# This batch file can be run to activate a specific python virtual environment -# and set the necessary envionment variables to load the DREAM3D-NX Plugin - -# If you launch an Anaconda Prompt you can then run this batch file to -# activate your environment and set the needed variables. If you have installed -# your anaconda environment into a non-default location then you will need to -# modify the below line with this command instead, being sure to substitue -# the proper path to the environment. -# conda activate --prefix /opt/local/conda3/envs/pyd3d - -conda activate @ANACONDA_ENV_NAME@ - -# ---------------------------------------------------------------------------- -# The first variable to set is the PYTHONPATH which should point to a -# directory or directories that contains the top level python plugin -# folders. If you need to list mulitple directories to search for plugins -# you can use the ":" character to separate those paths. - -export PYTHONPATH=@PYTHONPATH@ - -# ---------------------------------------------------------------------------- -# The next variable is a list of all plugins that you would like to -# load when DREAM3D-NX (or NXRunner) is run. If you have multiple plugins that -# you would like to load, list them all separated by a ":" character. - -export SIMPLNX_PYTHON_PLUGINS=@SIMPLNX_PYTHON_PLUGINS@ diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp index a3a02062e8..316478ef9d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplateFile.hpp @@ -81,30 +81,28 @@ inline Result<> InsertFilterNameInPluginFiles(const std::filesystem::path& plugi // Create the output file by opening the same file for OVER WRITE. std::ofstream outFile = std::ofstream(initPyPath.string(), std::ios_base::binary | std::ios_base::trunc); + std::string filterMarkerLine = fmt::format("# FILTER_START: {}", filterName); + std::string lastMarkerLine = "def get_plugin():"; + std::string filterImportToken = fmt::format("from {0}.{1} import {1}", pluginName, filterName); bool insertToken = true; for(auto& line : lines) { - // If the line is the exact same as the generated import statement mark false - if(line == filterImportToken) + if(line == filterMarkerLine) { insertToken = false; } - // If we hit the include marker comment, then possibly write the newly generated token - if(nx::core::StringUtilities::starts_with(line, k_FilterIncludeInsertToken)) + if(line == lastMarkerLine && insertToken) { - if(insertToken) - { - outFile << filterImportToken << '\n'; - } + outFile << "# FILTER_START: " << filterName << "\n" + << "try:\n" + << " from " << pluginName << "." << filterName << " import " << filterName << "\n" + << " __all__.append('" << filterName << "')\n" + << "except ImportError:\n" + << " pass\n" + << "# FILTER_END: " << filterName << "\n\n"; } - // Do we need to append the filter to the list of filters - if(nx::core::StringUtilities::contains(line, k_FilterNameInsertToken) && insertToken) - { - line = nx::core::StringUtilities::replace(line, "__all__ = [", fmt::format("__all__ = ['{}', ", filterName)); - } - outFile << line << '\n'; // Write the line out to the file } outFile.close(); @@ -125,6 +123,9 @@ inline Result<> InsertFilterNameInPluginFiles(const std::filesystem::path& plugi // Create the output file by opening the same file for OVER WRITE. std::ofstream outFile = std::ofstream(pluginPyPath.string(), std::ios_base::binary | std::ios_base::trunc); + std::string filterMarkerLine = fmt::format("# FILTER_START: {}", filterName); + std::string lastMarkerLine = "import simplnx as nx"; + std::string filterImportToken = fmt::format("from {0}.{1} import {1}", pluginName, filterName); std::string filterInsertToken = fmt::format("'{}'", filterName); @@ -132,24 +133,20 @@ inline Result<> InsertFilterNameInPluginFiles(const std::filesystem::path& plugi for(auto& line : lines) { // If the line is the exact same as the generated import statement mark false - if(line == filterImportToken) + if(line == filterMarkerLine) { insertToken = false; } - // If we hit the include marker comment, then possibly write the newly generated token - if(nx::core::StringUtilities::starts_with(line, k_FilterIncludeInsertToken)) - { - if(insertToken) - { - outFile << filterImportToken << '\n'; - } - } - // Do we need to append the filter to the list of filters - if(nx::core::StringUtilities::contains(line, k_FilterNameInsertToken) && insertToken) + if(line == lastMarkerLine && insertToken) { - line = nx::core::StringUtilities::replace(line, "return [", fmt::format("return [{}, ", filterName)); + outFile << "# FILTER_START: " << filterName << "\n" + << "try:\n" + << " from " << pluginName << "." << filterName << " import " << filterName << "\n" + << " _filters_.append(" << filterName << ")\n" + << "except ImportError:\n" + << " pass\n" + << "# FILTER_END: " << filterName << "\n\n"; } - outFile << line << '\n'; // Write the line out to the file } outFile.close(); @@ -279,13 +276,19 @@ inline std::string GeneratePythonPlugin(const std::string& pluginName, const std auto filterList = StringUtilities::split(pluginFilterList, ','); content = StringUtilities::replace(content, "#PLUGIN_FILTER_LIST#", fmt::format("{}", fmt::join(filterList, ", "))); - std::string importStatements; + std::stringstream ss; + for(const auto& name : filterList) { - importStatements.append(fmt::format("from {}.{} import {}\n", pluginName, name, name)); + ss << "# FILTER_START: " << name << "\n" + << "try:\n" + << " from " << pluginName << "." << name << " import " << name << "\n" + << " _filters.append(" << name << ")\n" + << "except ImportError:\n" + << " pass\n" + << "# FILTER_END: " << name << "\n\n"; } - - content = StringUtilities::replace(content, "#PLUGIN_IMPORT_CODE#", importStatements); + content = StringUtilities::replace(content, "#PLUGIN_IMPORT_CODE#", ss.str()); return content; } @@ -301,7 +304,7 @@ inline std::string GeneratePythonPlugin(const std::string& pluginName, const std * @return */ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirectory, const std::string& pluginName, const std::string& pluginShortName, const std::string& pluginDescription, - const std::string& pluginFilterList, bool createBatchShellScript, const std::string& anacondaEnvName) + const std::string& pluginFilterList) { auto pluginRootPath = outputDirectory / pluginName; @@ -336,40 +339,6 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect } } - if(createBatchShellScript) - { -#ifdef __WIN32__ - outputPath = pluginRootPath / "init_evn.bat"; -#else - outputPath = pluginRootPath / "init_evn.sh"; -#endif - AtomicFile initTempFile(outputPath.string()); - auto creationResult = initTempFile.getResult(); - if(creationResult.invalid()) - { - return creationResult; - } - { - // Scope this so that the file closes first before we then 'commit' with the atomic file - std::ofstream fout(initTempFile.tempFilePath(), std::ios_base::out | std::ios_base::binary); - if(!fout.is_open()) - { - return MakeErrorResult(-74100, fmt::format("Error creating and opening output file at path: {}", initTempFile.tempFilePath().string())); - } - std::string content = PluginBatchFile(); - - content = StringUtilities::replace(content, "@PYTHONPATH@", outputDirectory.string()); - content = StringUtilities::replace(content, "@SIMPLNX_PYTHON_PLUGINS@", pluginName); - content = StringUtilities::replace(content, "@ANACONDA_ENV_NAME@", anacondaEnvName); - - fout << content; - } - if(!initTempFile.commit()) - { - return initTempFile.getResult(); - } - } - // Write the __init__.py file outputPath = pluginRootPath / "__init__.py"; { @@ -403,13 +372,18 @@ inline Result<> WritePythonPluginFiles(const std::filesystem::path& outputDirect aList.append("'get_plugin'"); content = StringUtilities::replace(content, "#PLUGIN_FILTER_LIST#", aList); - std::string importStatements; + std::stringstream ss; for(const auto& name : filterList) { - importStatements.append(fmt::format("from {}.{} import {}\n", pluginName, name, name)); + ss << "# FILTER_START: " << name << "\n" + << "try:\n" + << " from " << pluginName << "." << name << " import " << name << "\n" + << " __all__.append('" << name << "')\n" + << "except ImportError:\n" + << " pass\n" + << "# FILTER_END: " << name << "\n\n"; } - - content = StringUtilities::replace(content, "#PLUGIN_IMPORT_CODE#", importStatements); + content = StringUtilities::replace(content, "#PLUGIN_IMPORT_CODE#", ss.str()); fout << content; } diff --git a/src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp b/src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp index cd71051270..f3c5a316fe 100644 --- a/src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp +++ b/src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp @@ -278,6 +278,91 @@ nx::core::DataPath CreateDataPath(std::string_view path) return result.value(); } +class ManualImportFinder +{ +public: + bool insert(const fs::path& path) + { + if(containsPath(path)) + { + return false; + } + std::string modName = GetModuleNameFromPath(path); + if(containsModule(modName)) + { + return false; + } + m_ModuleToPathMap.insert({modName, path}); + m_PathToModuleMap.insert({path, modName}); + return true; + } + + void removePath(const fs::path& path) + { + if(!containsPath(path)) + { + return; + } + std::string modName = GetModuleNameFromPath(path); + m_ModuleToPathMap.erase(modName); + m_PathToModuleMap.erase(path); + } + + void removeModule(const std::string& modName) + { + if(!containsModule(modName)) + { + return; + } + fs::path modPath = m_ModuleToPathMap.at(modName); + m_ModuleToPathMap.erase(modName); + m_PathToModuleMap.erase(modPath); + } + + void clear() + { + m_ModuleToPathMap.clear(); + m_PathToModuleMap.clear(); + } + + bool containsPath(const fs::path& path) const + { + return m_PathToModuleMap.count(path) > 0; + } + + bool containsModule(const std::string& modName) const + { + return m_ModuleToPathMap.count(modName) > 0; + } + + py::object findSpec(const std::string& fullname, py::object path, py::object target) const + { + if(!containsModule(fullname)) + { + return py::none(); + } + + fs::path modPath = m_ModuleToPathMap.at(fullname); + + bool isPackage = modPath.extension() != ".py"; + + auto importLibUtil = py::module_::import("importlib.util"); + fs::path initPyPath = isPackage ? modPath / "__init__.py" : modPath; + py::object submoduleSearchLocations = isPackage ? py::list() : py::object(py::none()); + auto spec = importLibUtil.attr("spec_from_file_location")(fullname, initPyPath, py::arg("submodule_search_locations") = submoduleSearchLocations); + return spec; + } + +private: + static std::string GetModuleNameFromPath(const fs::path& path) + { + return path.stem().string(); + } + + std::map m_ModuleToPathMap; + std::map m_PathToModuleMap; +}; + PYBIND11_MODULE(simplnx, mod) { auto* internals = new Internals(); @@ -546,16 +631,33 @@ PYBIND11_MODULE(simplnx, mod) } return self.removeData(pathConversionResult.value()); }); + + dataStructure.def( + "exists", + [](const DataStructure& self, std::string_view path) { + auto convertedPath = DataPath::FromString(path); + if(!convertedPath) + { + return false; + } + return self.containsData(convertedPath.value()); + }, + "Returns true if there is a DataStructure object at the given path", "path"_a); + dataStructure.def("exists", py::overload_cast(&DataStructure::containsData, py::const_), "Returns true if there is a DataStructure object at the given path", "path"_a); + dataStructure.def("__contains__", py::overload_cast(&DataStructure::containsData, py::const_), "Returns true if there is a DataStructure object at the given path", "path"_a); dataStructure.def("hierarchy_to_str", [](DataStructure& self) { std::stringstream ss; self.exportHierarchyAsText(ss); return ss.str(); }); - dataStructure.def("hierarchy_to_graphviz", [](DataStructure& self) { - std::stringstream ss; - self.exportHierarchyAsGraphViz(ss); - return ss.str(); - }); + dataStructure.def( + "hierarchy_to_graphviz", + [](DataStructure& self) { + std::stringstream ss; + self.exportHierarchyAsGraphViz(ss); + return ss.str(); + }, + "Returns the DataStructure hierarchy expressed in the 'dot' language. Use a GraphViz package to render."); dataStructure.def("get_children", [](DataStructure& self, nx::core::DataPath& parentPath) { if(parentPath.empty()) { @@ -734,6 +836,8 @@ PYBIND11_MODULE(simplnx, mod) py::class_> attributeMatrix(mod, "AttributeMatrix"); attributeMatrix.def("resize_tuples", &AttributeMatrix::resizeTuples, "Resize the tuples with the given shape"); + attributeMatrix.def_property_readonly("tuple_shape", &AttributeMatrix::getShape, "Returns the Tuple dimensions of the AttributeMatrix"); + attributeMatrix.def_property_readonly("size", &AttributeMatrix::getNumTuples, "Returns the total number of tuples"); py::class_> iArray(mod, "IArray"); iArray.def_property_readonly("tuple_shape", &IArray::getTupleShape); @@ -1126,6 +1230,17 @@ PYBIND11_MODULE(simplnx, mod) filterMessage.def_readwrite("type", &IFilter::Message::type); filterMessage.def_readwrite("message", &IFilter::Message::message); + py::class_ progressMessage(filter, "ProgressMessage"); + progressMessage.def(py::init([](std::string message, int32 progress) { + IFilter::ProgressMessage progressMessage; + progressMessage.type = IFilter::Message::Type::Progress; + progressMessage.message = std::move(message); + progressMessage.progress = progress; + return progressMessage; + }), + "message"_a, "progress"_a); + progressMessage.def_readwrite("progress", &IFilter::ProgressMessage::progress); + py::class_ messageHandler(filter, "MessageHandler"); messageHandler.def(py::init<>()); messageHandler.def_readwrite("callback", &IFilter::MessageHandler::m_Callback); @@ -1133,6 +1248,7 @@ PYBIND11_MODULE(simplnx, mod) py::class_ preflightValue(filter, "PreflightValue"); preflightValue.def(py::init<>()); + preflightValue.def(py::init(), "name"_a, "value"_a); preflightValue.def_readwrite("name", &IFilter::PreflightValue::name); preflightValue.def_readwrite("value", &IFilter::PreflightValue::value); @@ -1256,6 +1372,28 @@ PYBIND11_MODULE(simplnx, mod) "set_args", [internals](PipelineFilter& self, py::dict& args) { self.setArguments(ConvertDictToArgs(*internals, self.getParameters(), args)); }, "args"_a); pipelineFilter.def( "get_filter", [](PipelineFilter& self) { return self.getFilter(); }, py::return_value_policy::reference_internal); + pipelineFilter.def( + "name", + [](const PipelineFilter& self) { + const IFilter* filter = self.getFilter(); + if(filter == nullptr) + { + throw std::runtime_error("PipelineFilter doesn't contain a filter (nullptr)"); + } + return filter->name(); + }, + "Returns the C++ name of the filter"); + pipelineFilter.def( + "human_name", + [](const PipelineFilter& self) { + const IFilter* filter = self.getFilter(); + if(filter == nullptr) + { + throw std::runtime_error("PipelineFilter doesn't contain a filter (nullptr)"); + } + return filter->humanName(); + }, + "Returns the human facing name of the filter"); py::class_ pyFilter(mod, "PyFilter"); pyFilter.def(py::init<>([](py::object object) { return std::make_unique(std::move(object)); })); @@ -1346,4 +1484,15 @@ PYBIND11_MODULE(simplnx, mod) }); mod.def("reload_python_plugins", [internals]() { internals->reloadPythonPlugins(); }); + mod.def("unload_python_plugins", [internals]() { internals->unloadPythonPlugins(); }); + + py::class_ manualImportFinder(mod, "ManualImportFinder"); + manualImportFinder.def(py::init<>()); + manualImportFinder.def("find_spec", &ManualImportFinder::findSpec, "fullname"_a, "path"_a = py::none(), "target"_a = py::none()); + manualImportFinder.def("insert", &ManualImportFinder::insert, "path"_a); + manualImportFinder.def("remove_path", &ManualImportFinder::removePath, "path"_a); + manualImportFinder.def("remove_module", &ManualImportFinder::removeModule, "mod_name"_a); + manualImportFinder.def("clear", &ManualImportFinder::clear); + manualImportFinder.def("contains_path", &ManualImportFinder::containsPath, "path"_a); + manualImportFinder.def("contains_module", &ManualImportFinder::containsModule, "mod_name"_a); } diff --git a/src/Plugins/TestOne/src/TestOne/Filters/ExampleFilter1.cpp b/src/Plugins/TestOne/src/TestOne/Filters/ExampleFilter1.cpp index 2e345fd760..3869279e13 100644 --- a/src/Plugins/TestOne/src/TestOne/Filters/ExampleFilter1.cpp +++ b/src/Plugins/TestOne/src/TestOne/Filters/ExampleFilter1.cpp @@ -72,8 +72,8 @@ Parameters ExampleFilter1::parameters() const params.insert(std::make_unique(k_InputDir_Key, "Input Directory", "Example input directory help text", "Data", FileSystemPathParameter::ExtensionsType{}, FileSystemPathParameter::PathType::InputDir)); params.insert(std::make_unique(k_InputFile_Key, "Input File", "Example input file help text", "/opt/local/bin/ninja", FileSystemPathParameter::ExtensionsType{}, - FileSystemPathParameter::PathType::InputFile, true)); - params.insert(std::make_unique(k_OutputDir_Key, "Ouptut Directory", "Example output directory help text", "Output Data", FileSystemPathParameter::ExtensionsType{}, + FileSystemPathParameter::PathType::InputFile, false)); + params.insert(std::make_unique(k_OutputDir_Key, "Output Directory", "Example output directory help text", "Output Data", FileSystemPathParameter::ExtensionsType{}, FileSystemPathParameter::PathType::OutputDir)); params.insert(std::make_unique(k_OutputFile_Key, "Output File", "Example output file help text", "", FileSystemPathParameter::ExtensionsType{}, FileSystemPathParameter::PathType::OutputFile)); diff --git a/src/nxrunner/CMakeLists.txt b/src/nxrunner/CMakeLists.txt index dd6e4efbe6..4f093386ab 100644 --- a/src/nxrunner/CMakeLists.txt +++ b/src/nxrunner/CMakeLists.txt @@ -35,7 +35,7 @@ if(SIMPLNX_EMBED_PYTHON) target_link_libraries(nxrunner PRIVATE - pybind11::embed + NxPythonEmbed ) endif() diff --git a/src/nxrunner/src/nxrunner.cpp b/src/nxrunner/src/nxrunner.cpp index 209271b5a2..8de816928d 100644 --- a/src/nxrunner/src/nxrunner.cpp +++ b/src/nxrunner/src/nxrunner.cpp @@ -16,6 +16,8 @@ #include #if SIMPLNX_EMBED_PYTHON +#include "NxPythonEmbed/NxPythonEmbed.hpp" + #include #endif @@ -491,22 +493,6 @@ Result<> SetLogFile(const Argument& argument) std::filesystem::path filepath(argument.value); return cliOut.setLogFile(filepath); } - -#if SIMPLNX_EMBED_PYTHON -std::vector GetPythonPluginList() -{ - auto* var = std::getenv("SIMPLNX_PYTHON_PLUGINS"); - if(var == nullptr) - { - return {}; - } -#if defined(Q_OS_WIN) - return nx::core::StringUtilities::split(var, ';'); -#else - return nx::core::StringUtilities::split(var, ':'); -#endif -} -#endif } // namespace int main(int argc, char* argv[]) @@ -565,60 +551,36 @@ int main(int argc, char* argv[]) LoadApp(); #if SIMPLNX_EMBED_PYTHON - { - constexpr const char k_PYTHONHOME[] = "PYTHONHOME"; - constexpr const char k_CONDA_PREFIX[] = "CONDA_PREFIX"; - - std::string condaPrefix; - char* condaPrefixPtr = getenv(k_CONDA_PREFIX); - if(condaPrefixPtr != nullptr) - { - condaPrefix = condaPrefixPtr; - std::cout << "CONDA_PREFIX=" << condaPrefix << std::endl; - } + nx::python::OutputCallback outputCallback = [](const std::string& message) { std::cout << message << "\n"; }; - std::string pythonHome; - char* pythonHomePtr = getenv(k_PYTHONHOME); - if(pythonHomePtr != nullptr) - { - pythonHome = pythonHomePtr; - std::cout << "PYTHONHOME=" << pythonHome << std::endl; - } + nx::python::SetupPythonEnvironmentVars(outputCallback); - if(pythonHome.empty() && !condaPrefix.empty()) - { - std::string envVar = fmt::format("{}={}", k_PYTHONHOME, condaPrefix); - putenv(envVar.data()); - std::cout << envVar << std::endl; - } - } + std::set pythonPlugins = nx::python::GetPythonPluginListFromEnvironment(); py::scoped_interpreter guard{}; - try - { - auto cx = py::module_::import(SIMPLNX_PYTHON_MODULE); - - auto pythonPlugins = GetPythonPluginList(); - - fmt::print("Loading Python plugins: {}\n", pythonPlugins); + nx::python::PluginLoadErrorCallback pluginLoadErrorCallback = [](const nx::python::PluginLoadErrorInfo& errorInfo) { + std::string exceptionType = nx::python::ExceptionTypeToString(errorInfo.type); + std::string text = fmt::format("{} exception while while attempting to import '{}': ", exceptionType, errorInfo.pluginName); + std::cout << text << "\n"; + std::cout << errorInfo.message << "\n"; + }; + nx::python::PythonErrorCallback pythonErrorCallback = [](const nx::python::PythonErrorInfo& errorInfo) { + std::string exceptionType = nx::python::ExceptionTypeToString(errorInfo.type); + std::string text = fmt::format("{} exception while importing plugins: ", exceptionType); + std::cout << text; + std::cout << errorInfo.message; + }; - for(const auto& pluginName : pythonPlugins) - { - fmt::print("Attempting to load Python plugin: '{}'\n", pluginName); - auto mod = py::module_::import(pluginName.c_str()); - cx.attr("load_python_plugin")(mod); - auto pluginPath = mod.attr("__file__").cast(); - fmt::print("Successfully loaded Python plugin '{}' from '{}'\n", pluginName, pluginPath); - } - } catch(const py::error_already_set& exception) + try { - fmt::print("Python exception while importing plugins: {}\n", exception.what()); - return 1; + auto manualImportFinder = nx::python::ManualImportFinderHolder::Create(); + manualImportFinder.addToMetaPath(); + nx::python::LoadPythonPlugins(pythonPlugins, outputCallback, pluginLoadErrorCallback, pythonErrorCallback); } catch(const std::exception& exception) { - fmt::print("C++ exception while importing plugins: {}\n", exception.what()); - return 1; + std::cout << "Aborting python plugin loading due to exception: \n"; + std::cout << exception.what(); } #endif diff --git a/src/simplnx/Pipeline/AbstractPipelineFilter.cpp b/src/simplnx/Pipeline/AbstractPipelineFilter.cpp new file mode 100644 index 0000000000..eb0afaad68 --- /dev/null +++ b/src/simplnx/Pipeline/AbstractPipelineFilter.cpp @@ -0,0 +1,17 @@ +#include "AbstractPipelineFilter.hpp" + +using namespace nx::core; + +AbstractPipelineFilter::~AbstractPipelineFilter() noexcept = default; + +AbstractPipelineNode::NodeType AbstractPipelineFilter::getType() const +{ + return NodeType::Filter; +} + +void AbstractPipelineFilter::setIndex(int32 index) +{ + m_Index = index; +} + +AbstractPipelineFilter::AbstractPipelineFilter() = default; diff --git a/src/simplnx/Pipeline/AbstractPipelineFilter.hpp b/src/simplnx/Pipeline/AbstractPipelineFilter.hpp new file mode 100644 index 0000000000..f7931b8a40 --- /dev/null +++ b/src/simplnx/Pipeline/AbstractPipelineFilter.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include "simplnx/Pipeline/AbstractPipelineNode.hpp" + +namespace nx::core +{ +/** + * @class AbstractPipelineFilter + * @brief Base class for PipelineFilter and PlaceholderFilter + */ +class SIMPLNX_EXPORT AbstractPipelineFilter : public AbstractPipelineNode +{ +public: + enum class FilterType + { + Filter, + Placeholder + }; + + /** + * @brief + */ + ~AbstractPipelineFilter() noexcept override; + + /** + * @brief Returns the node type for quick type checking. + * @return NodeType + */ + NodeType getType() const override; + + /** + * @brief Sets the index of this filter in an executing pipeline + * @param index + */ + void setIndex(int32 index); + + /** + * @brief Returns the type of filter of this node (filter or placeholder) + * @return AbstractPipelineFilter::FilterType + */ + virtual FilterType getFilterType() const = 0; + +protected: + /** + * @brief + */ + AbstractPipelineFilter(); + + int32 m_Index = 0; +}; +} // namespace nx::core diff --git a/src/simplnx/Pipeline/AbstractPipelineNode.cpp b/src/simplnx/Pipeline/AbstractPipelineNode.cpp index 85fcb25af2..1709f27d68 100644 --- a/src/simplnx/Pipeline/AbstractPipelineNode.cpp +++ b/src/simplnx/Pipeline/AbstractPipelineNode.cpp @@ -21,7 +21,7 @@ AbstractPipelineNode::AbstractPipelineNode(Pipeline* parent) { } -AbstractPipelineNode::~AbstractPipelineNode() = default; +AbstractPipelineNode::~AbstractPipelineNode() noexcept = default; Pipeline* AbstractPipelineNode::getParentPipeline() const { diff --git a/src/simplnx/Pipeline/AbstractPipelineNode.hpp b/src/simplnx/Pipeline/AbstractPipelineNode.hpp index 62261405eb..88a0c6417d 100644 --- a/src/simplnx/Pipeline/AbstractPipelineNode.hpp +++ b/src/simplnx/Pipeline/AbstractPipelineNode.hpp @@ -83,7 +83,7 @@ class SIMPLNX_EXPORT AbstractPipelineNode Filter }; - virtual ~AbstractPipelineNode(); + virtual ~AbstractPipelineNode() noexcept; /** * @brief Returns the node type for quick type checking. diff --git a/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.cpp b/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.cpp index 3402e93249..ef7c316745 100644 --- a/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.cpp +++ b/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.cpp @@ -8,7 +8,7 @@ PipelineNodeObserver::PipelineNodeObserver() { } -PipelineNodeObserver::~PipelineNodeObserver() +PipelineNodeObserver::~PipelineNodeObserver() noexcept { stopObservingNode(); } diff --git a/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.hpp b/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.hpp index a1c67bcc29..2ca8fa736a 100644 --- a/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.hpp +++ b/src/simplnx/Pipeline/Messaging/PipelineNodeObserver.hpp @@ -39,7 +39,7 @@ class SIMPLNX_EXPORT PipelineNodeObserver */ PipelineNodeObserver(PipelineNodeObserver&& other) = delete; - virtual ~PipelineNodeObserver(); + virtual ~PipelineNodeObserver() noexcept; /** * @brief Returns a pointer to the observed pipeline node. Returns nullptr diff --git a/src/simplnx/Pipeline/Pipeline.cpp b/src/simplnx/Pipeline/Pipeline.cpp index bb08ab6e6b..6e696d14e2 100644 --- a/src/simplnx/Pipeline/Pipeline.cpp +++ b/src/simplnx/Pipeline/Pipeline.cpp @@ -8,6 +8,7 @@ #include "simplnx/Pipeline/Messaging/NodeRemovedMessage.hpp" #include "simplnx/Pipeline/Messaging/PipelineNodeMessage.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/Pipeline/PlaceholderFilter.hpp" #include @@ -83,7 +84,7 @@ Pipeline::Pipeline(Pipeline&& other) noexcept resetCollectionParent(); } -Pipeline::~Pipeline() +Pipeline::~Pipeline() noexcept { for(auto& node : m_Collection) { @@ -590,8 +591,13 @@ bool Pipeline::contains(const Uuid& id) const switch(nodeType) { case NodeType::Filter: { - const auto& filterNode = dynamic_cast(*node); - if(id == filterNode.getFilter()->uuid()) + const auto& filterNode = dynamic_cast(*node); + if(filterNode.getFilterType() == AbstractPipelineFilter::FilterType::Placeholder) + { + continue; + } + const auto& pipelineFilter = dynamic_cast(filterNode); + if(id == pipelineFilter.getFilter()->uuid()) { return true; } @@ -610,6 +616,34 @@ bool Pipeline::contains(const Uuid& id) const return false; } +bool Pipeline::containsPlaceholder() const +{ + for(const auto& node : *this) + { + NodeType nodeType = node->getType(); + switch(nodeType) + { + case NodeType::Filter: { + const auto& filterNode = dynamic_cast(*node); + if(filterNode.getFilterType() == AbstractPipelineFilter::FilterType::Placeholder) + { + return true; + } + break; + } + case NodeType::Pipeline: { + const auto& pipeline = dynamic_cast(*node); + if(pipeline.containsPlaceholder()) + { + return true; + } + break; + } + } + } + return false; +} + bool Pipeline::push_front(std::shared_ptr node) { return insertAt(begin(), std::move(node)); @@ -740,12 +774,12 @@ nlohmann::json Pipeline::toJsonImpl() const return CreatePipelineJson(m_Name, std::move(jsonArray)); } -Result Pipeline::FromJson(const nlohmann::json& json) +Result Pipeline::FromJson(const nlohmann::json& json, bool allowPlaceholderFilters) { - return FromJson(json, Application::Instance()->getFilterList()); + return FromJson(json, Application::Instance()->getFilterList(), allowPlaceholderFilters); } -Result Pipeline::FromJson(const nlohmann::json& json, FilterList* filterList) +Result Pipeline::FromJson(const nlohmann::json& json, FilterList* filterList, bool allowPlaceholderFilters) { if(!json.contains(k_PipelineNameKey.view())) { @@ -771,14 +805,21 @@ Result Pipeline::FromJson(const nlohmann::json& json, FilterList* filt { warnings.push_back(std::move(warning)); } + if(filterResult.invalid()) { - Result result{nonstd::make_unexpected(std::move(filterResult.errors()))}; - result.warnings() = std::move(warnings); - return result; + if(!allowPlaceholderFilters) + { + Result result{nonstd::make_unexpected(std::move(filterResult.errors()))}; + result.warnings() = std::move(warnings); + return result; + } + pipeline.push_back(PlaceholderFilter::Create(item)); + } + else + { + pipeline.push_back(std::move(filterResult.value())); } - - pipeline.push_back(std::move(filterResult.value())); } Result result{std::move(pipeline)}; @@ -787,13 +828,13 @@ Result Pipeline::FromJson(const nlohmann::json& json, FilterList* filt return result; } -Result Pipeline::FromFile(const std::filesystem::path& path) +Result Pipeline::FromFile(const std::filesystem::path& path, bool allowPlaceholderFilters) { auto app = Application::Instance(); - return FromFile(path, app->getFilterList()); + return FromFile(path, app->getFilterList(), allowPlaceholderFilters); } -Result Pipeline::FromFile(const std::filesystem::path& path, FilterList* filterList) +Result Pipeline::FromFile(const std::filesystem::path& path, FilterList* filterList, bool allowPlaceholderFilters) { std::ifstream file(path); @@ -812,7 +853,7 @@ Result Pipeline::FromFile(const std::filesystem::path& path, FilterLis return MakeErrorResult(-2, exception.what()); } - return FromJson(pipelineJson, filterList); + return FromJson(pipelineJson, filterList, allowPlaceholderFilters); } void Pipeline::onNotify(AbstractPipelineNode* node, const std::shared_ptr& msg) diff --git a/src/simplnx/Pipeline/Pipeline.hpp b/src/simplnx/Pipeline/Pipeline.hpp index 2765d797cf..7b8b184bb0 100644 --- a/src/simplnx/Pipeline/Pipeline.hpp +++ b/src/simplnx/Pipeline/Pipeline.hpp @@ -41,7 +41,7 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN * @param json * @return */ - static Result FromJson(const nlohmann::json& json); + static Result FromJson(const nlohmann::json& json, bool allowPlaceholderFilters = false); /** * @brief Constructs a Pipeline from json and the given filter list. @@ -49,14 +49,14 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN * @param filterList * @return */ - static Result FromJson(const nlohmann::json& json, FilterList* filterList); + static Result FromJson(const nlohmann::json& json, FilterList* filterList, bool allowPlaceholderFilters = false); /** * @brief Constructs a Pipeline from a JSON file. * @param path * @return Result */ - static Result FromFile(const std::filesystem::path& path); + static Result FromFile(const std::filesystem::path& path, bool allowPlaceholderFilters = false); /** * @brief Constructs a Pipeline from a JSON file with the given FilterList. @@ -64,7 +64,7 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN * @param filterList * @return Result */ - static Result FromFile(const std::filesystem::path& path, FilterList* filterList); + static Result FromFile(const std::filesystem::path& path, FilterList* filterList, bool allowPlaceholderFilters = false); /** * @brief Attempts to read a SIMPL json pipeline and convert to a simplnx Pipeline. @@ -115,7 +115,7 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN */ Pipeline(Pipeline&& other) noexcept; - ~Pipeline() override; + ~Pipeline() noexcept override; /** * @brief Returns the node type for quick type checking. @@ -438,6 +438,12 @@ class SIMPLNX_EXPORT Pipeline : public AbstractPipelineNode, protected PipelineN */ bool contains(const Uuid& id) const; + /** + * @brief Returns true if the pipeline contains a placeholder filter. + * @return + */ + bool containsPlaceholder() const; + /** * @brief Inserts the specified pipeline node to the front of the pipeline * segment. diff --git a/src/simplnx/Pipeline/PipelineFilter.cpp b/src/simplnx/Pipeline/PipelineFilter.cpp index 89657661b7..9abf792cbc 100644 --- a/src/simplnx/Pipeline/PipelineFilter.cpp +++ b/src/simplnx/Pipeline/PipelineFilter.cpp @@ -74,18 +74,13 @@ std::unique_ptr PipelineFilter::Create(const FilterHandle& handl } PipelineFilter::PipelineFilter(IFilter::UniquePointer&& filter, const Arguments& args) -: AbstractPipelineNode() +: AbstractPipelineFilter() , m_Filter(std::move(filter)) , m_Arguments(args) { } -PipelineFilter::~PipelineFilter() = default; - -AbstractPipelineNode::NodeType PipelineFilter::getType() const -{ - return NodeType::Filter; -} +PipelineFilter::~PipelineFilter() noexcept = default; std::string PipelineFilter::getName() const { @@ -127,11 +122,6 @@ void PipelineFilter::setArguments(const Arguments& args) m_Arguments = args; } -void PipelineFilter::setIndex(int32 index) -{ - m_Index = index; -} - const std::string& PipelineFilter::getComments() const { return m_Comments; @@ -720,3 +710,8 @@ Result> PipelineFilter::FromSIMPLJson(const nloh return {std::move(pipelineFilter), std::move(warnings)}; } + +AbstractPipelineFilter::FilterType PipelineFilter::getFilterType() const +{ + return FilterType::Filter; +} diff --git a/src/simplnx/Pipeline/PipelineFilter.hpp b/src/simplnx/Pipeline/PipelineFilter.hpp index 742a368067..a6599f5c94 100644 --- a/src/simplnx/Pipeline/PipelineFilter.hpp +++ b/src/simplnx/Pipeline/PipelineFilter.hpp @@ -1,7 +1,7 @@ #pragma once #include "simplnx/Filter/IFilter.hpp" -#include "simplnx/Pipeline/AbstractPipelineNode.hpp" +#include "simplnx/Pipeline/AbstractPipelineFilter.hpp" #include @@ -16,7 +16,7 @@ class FilterList; * IFilter object. The node keeps track of the resulting DataStructure as well * as error and warning messages. */ -class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineNode +class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineFilter { public: using WarningsChangedSignal = nod::signal)>; @@ -75,13 +75,7 @@ class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineNode */ PipelineFilter(IFilter::UniquePointer&& filter, const Arguments& args = {}); - ~PipelineFilter() override; - - /** - * @brief Returns the node type for quick type checking. - * @return NodeType - */ - NodeType getType() const override; + ~PipelineFilter() noexcept override; /** * @brief Returns the filter's human label. @@ -113,12 +107,6 @@ class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineNode */ void setArguments(const Arguments& args); - /** - * @brief Sets the index of this filter in an executing pipeline - * @param index - */ - void setIndex(int32 index); - /** * @brief Gets the node's filter comments. */ @@ -201,6 +189,12 @@ class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineNode */ void renamePathArgs(const RenamedPaths& renamedPaths); + /** + * @brief Returns the type of filter of this node (filter or placeholder) + * @return AbstractPipelineFilter::FilterType + */ + FilterType getFilterType() const override; + protected: /** * @brief Returns implementation-specific json value for the node. @@ -236,7 +230,6 @@ class SIMPLNX_EXPORT PipelineFilter : public AbstractPipelineNode private: IFilter::UniquePointer m_Filter; Arguments m_Arguments; - int32 m_Index = 0; std::string m_Comments; std::vector m_Warnings; std::vector m_Errors; diff --git a/src/simplnx/Pipeline/PlaceholderFilter.cpp b/src/simplnx/Pipeline/PlaceholderFilter.cpp new file mode 100644 index 0000000000..6f5b5111b0 --- /dev/null +++ b/src/simplnx/Pipeline/PlaceholderFilter.cpp @@ -0,0 +1,58 @@ +#include "PlaceholderFilter.hpp" + +using namespace nx::core; + +std::unique_ptr PlaceholderFilter::Create(nlohmann::json json) +{ + return std::make_unique(std::move(json)); +} + +PlaceholderFilter::PlaceholderFilter() = default; + +PlaceholderFilter::PlaceholderFilter(nlohmann::json json) +: AbstractPipelineFilter() +, m_Json(std::move(json)) +{ +} + +PlaceholderFilter::~PlaceholderFilter() noexcept = default; + +bool PlaceholderFilter::preflight(DataStructure& data, const std::atomic_bool& shouldCancel) +{ + return true; +} + +bool PlaceholderFilter::preflight(DataStructure& data, RenamedPaths& renamedPaths, const std::atomic_bool& shouldCancel, bool allowRenaming) +{ + return true; +} + +bool PlaceholderFilter::execute(DataStructure& data, const std::atomic_bool& shouldCancel) +{ + return true; +} + +std::unique_ptr PlaceholderFilter::deepCopy() const +{ + return std::make_unique(m_Json); +} + +nlohmann::json PlaceholderFilter::toJsonImpl() const +{ + return m_Json; +} + +std::string PlaceholderFilter::getName() const +{ + return "Placeholder Filter"; +} + +const nlohmann::json& PlaceholderFilter::getJson() const +{ + return m_Json; +} + +AbstractPipelineFilter::FilterType PlaceholderFilter::getFilterType() const +{ + return FilterType::Placeholder; +} diff --git a/src/simplnx/Pipeline/PlaceholderFilter.hpp b/src/simplnx/Pipeline/PlaceholderFilter.hpp new file mode 100644 index 0000000000..eb3bb14be9 --- /dev/null +++ b/src/simplnx/Pipeline/PlaceholderFilter.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include "simplnx/Pipeline/AbstractPipelineFilter.hpp" + +#include + +namespace nx::core +{ +/** + * @class PlaceholderFilter + * @brief PlaceholderFilter acts as a placeholder for filters that weren't able + * to be created successfully from json. The original json is stored in case the + * filter is able to be restored later. + */ +class SIMPLNX_EXPORT PlaceholderFilter : public AbstractPipelineFilter +{ +public: + /** + * @brief + * @return std::unique_ptr + */ + static std::unique_ptr Create(nlohmann::json json); + + /** + * @brief + */ + PlaceholderFilter(); + + /** + * @brief + */ + PlaceholderFilter(nlohmann::json json); + + /** + * @brief + */ + ~PlaceholderFilter() noexcept override; + + /** + * @brief Attempts to preflight the node using the provided DataStructure. + * Returns true if preflighting succeeded. Otherwise, this returns false. + * @param data + * @param shouldCancel + * @return bool + */ + bool preflight(DataStructure& data, const std::atomic_bool& shouldCancel) override; + + /** + * @brief Attempts to preflight the node using the provided DataStructure. + * Returns true if preflighting succeeded. Otherwise, this returns false. + * @param data + * @param renamedPaths Collection of renamed output paths. + * @param shouldCancel + * @param allowRenaming + * @return bool + */ + bool preflight(DataStructure& data, RenamedPaths& renamedPaths, const std::atomic_bool& shouldCancel, bool allowRenaming) override; + + /** + * @brief Attempts to execute the node using the provided DataStructure. + * Returns true if execution succeeded. Otherwise, this returns false. + * @param data + * @param shouldCancel + * @return bool + */ + bool execute(DataStructure& data, const std::atomic_bool& shouldCancel) override; + + /** + * @brief Creates and returns a unique pointer to a copy of the node. + * @return std::unique_ptr + */ + std::unique_ptr deepCopy() const override; + + /** + * @brief Returns the filter's human label. + * @return std::string + */ + std::string getName() const override; + + /** + * @brief Returns the original json for this filter. + * @return const nlohmann::json& + */ + const nlohmann::json& getJson() const; + + /** + * @brief Returns the type of filter of this node (filter or placeholder) + * @return AbstractPipelineFilter::FilterType + */ + FilterType getFilterType() const override; + +protected: + /** + * @brief Returns implementation-specific json value for the node. + * This method should only be called from toJson(). + * @return + */ + nlohmann::json toJsonImpl() const override; + +private: + nlohmann::json m_Json; +}; +} // namespace nx::core diff --git a/src/simplnx/Utilities/Parsing/HDF5/IO/AttributeIO.hpp b/src/simplnx/Utilities/Parsing/HDF5/IO/AttributeIO.hpp index fd39167246..5c9953561a 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/IO/AttributeIO.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/IO/AttributeIO.hpp @@ -168,7 +168,7 @@ class SIMPLNX_EXPORT AttributeIO Result<> writeVector(const DimsVector& dims, const std::vector& vector); AttributeIO& operator=(const AttributeIO& other) = delete; - AttributeIO& operator=(AttributeIO&& other) noexcept = default; + AttributeIO& operator=(AttributeIO&& other) noexcept = delete; protected: /** diff --git a/src/simplnx/Utilities/Parsing/HDF5/Writers/AttributeWriter.hpp b/src/simplnx/Utilities/Parsing/HDF5/Writers/AttributeWriter.hpp index 2662a6c1c7..062aa261e8 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/Writers/AttributeWriter.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/Writers/AttributeWriter.hpp @@ -208,7 +208,7 @@ class SIMPLNX_EXPORT AttributeWriter } AttributeWriter& operator=(const AttributeWriter& other) = delete; - AttributeWriter& operator=(AttributeWriter&& other) noexcept = default; + AttributeWriter& operator=(AttributeWriter&& other) noexcept = delete; protected: /** diff --git a/src/simplnx/Utilities/Parsing/HDF5/Writers/DatasetWriter.hpp b/src/simplnx/Utilities/Parsing/HDF5/Writers/DatasetWriter.hpp index 7a36298be4..13190916cc 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/Writers/DatasetWriter.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/Writers/DatasetWriter.hpp @@ -207,7 +207,7 @@ class SIMPLNX_EXPORT DatasetWriter : public ObjectWriter IdType getPListId() const; DatasetWriter& operator=(const DatasetWriter& other) = delete; - DatasetWriter& operator=(DatasetWriter&& other) noexcept = default; + DatasetWriter& operator=(DatasetWriter&& other) noexcept = delete; protected: /** diff --git a/test/PipelineSaveTest.cpp b/test/PipelineSaveTest.cpp index 9046191615..d235fba5a7 100644 --- a/test/PipelineSaveTest.cpp +++ b/test/PipelineSaveTest.cpp @@ -1,5 +1,8 @@ #include "simplnx/Core/Application.hpp" #include "simplnx/Pipeline/Pipeline.hpp" +#include "simplnx/Pipeline/PlaceholderFilter.hpp" + +#include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/unit_test/simplnx_test_dirs.hpp" #include @@ -7,6 +10,88 @@ using namespace nx::core; +namespace +{ +class TestFilter : public IFilter +{ +public: + static constexpr Uuid k_ID = *Uuid::FromString("d01ad5c2-6897-4380-89f1-6d0760b78a22"); + + TestFilter() = default; + ~TestFilter() override = default; + + std::string name() const override + { + return "TestFilter"; + } + + std::string className() const override + { + return "TestFilter"; + } + + nx::core::Uuid uuid() const override + { + return k_ID; + } + + std::string humanName() const override + { + return "Test Filter"; + } + + std::vector defaultTags() const override + { + return {}; + } + + nx::core::Parameters parameters() const override + { + return Parameters(); + } + + UniquePointer clone() const override + { + return std::make_unique(); + } + +protected: + PreflightResult preflightImpl(const nx::core::DataStructure& data, const nx::core::Arguments& args, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const override + { + return {}; + } + nx::core::Result<> executeImpl(nx::core::DataStructure& data, const nx::core::Arguments& args, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, + const std::atomic_bool& shouldCancel) const override + { + return {}; + } +}; + +class TestPlugin : public AbstractPlugin +{ +public: + static constexpr AbstractPlugin::IdType k_ID = *Uuid::FromString("09721f74-c282-44ac-ada7-1db1a23435dd"); + + TestPlugin() + : AbstractPlugin(k_ID, "TestPlugin", "", "") + { + addFilter([]() { return std::make_unique(); }); + } + ~TestPlugin() override = default; + + TestPlugin(const TestPlugin&) = delete; + TestPlugin(TestPlugin&&) = delete; + + TestPlugin& operator=(const TestPlugin&) = delete; + TestPlugin& operator=(TestPlugin&&) = delete; + + SIMPLMapType getSimplToSimplnxMap() const override + { + return {}; + } +}; +} // namespace + TEST_CASE("Save Filters To Json") { auto app = Application::GetOrCreateInstance(); @@ -63,3 +148,44 @@ TEST_CASE("Save Pipeline To Json") INFO(fmt::format("The pipeline containing every filter did not serialize to json properly!")) REQUIRE_NOTHROW(pipeline.toJson()); } + +TEST_CASE("PlaceholderFilter") +{ + auto app = Application::GetOrCreateInstance(); + FilterList* filterList = app->getFilterList(); + filterList->addPlugin(std::make_shared(std::make_shared())); + REQUIRE(filterList->containsPlugin(TestPlugin::k_ID)); + auto filterHandles = filterList->getFilterHandles(); + auto testFilterIter = std::find_if(filterHandles.cbegin(), filterHandles.cend(), [](const FilterHandle& handle) { return handle.getFilterId() == TestFilter::k_ID; }); + REQUIRE(testFilterIter != filterHandles.cend()); + + Pipeline pipeline; + pipeline.insertAt(0, *testFilterIter); + + nlohmann::json pipelineJson = pipeline.toJson(); + + filterList->removePlugin(TestPlugin::k_ID); + REQUIRE(!filterList->containsPlugin(TestPlugin::k_ID)); + + auto placeholderPipelineResult = Pipeline::FromJson(pipelineJson, true); + SIMPLNX_RESULT_REQUIRE_VALID(placeholderPipelineResult); + + Pipeline placeholderPipeline = std::move(placeholderPipelineResult.value()); + REQUIRE(placeholderPipeline.size() == 1); + REQUIRE(placeholderPipeline.containsPlaceholder()); + + AbstractPipelineNode* node = placeholderPipeline.at(0); + REQUIRE(node != nullptr); + REQUIRE(node->getType() == AbstractPipelineNode::NodeType::Filter); + + auto* filterNode = dynamic_cast(node); + REQUIRE(filterNode != nullptr); + REQUIRE(filterNode->getFilterType() == AbstractPipelineFilter::FilterType::Placeholder); + + auto* placeholderFilter = dynamic_cast(filterNode); + REQUIRE(placeholderFilter != nullptr); + + nlohmann::json placeholderPipelineJson = placeholderPipeline.toJson(); + + REQUIRE(placeholderPipelineJson == pipelineJson); +} diff --git a/wrapping/python/CMakeLists.txt b/wrapping/python/CMakeLists.txt index 8a371ee162..c99fac44fb 100644 --- a/wrapping/python/CMakeLists.txt +++ b/wrapping/python/CMakeLists.txt @@ -10,6 +10,19 @@ target_link_libraries(CxPybind pybind11::headers ) +add_library(NxPythonEmbed INTERFACE) + +target_include_directories(NxPythonEmbed + INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/NxPythonEmbed +) + +target_link_libraries(NxPythonEmbed + INTERFACE + simplnx + pybind11::embed +) + pybind11_add_module(simplnxpy "${SimplnxCore_SOURCE_DIR}/wrapping/python/simplnxpy.cpp") target_compile_definitions(simplnxpy PUBLIC PYBIND11_DETAILED_ERROR_MESSAGES) diff --git a/wrapping/python/CxPybind/CxPybind/CxPybind.hpp b/wrapping/python/CxPybind/CxPybind/CxPybind.hpp index 28c87f85ca..3a4fe6571b 100644 --- a/wrapping/python/CxPybind/CxPybind/CxPybind.hpp +++ b/wrapping/python/CxPybind/CxPybind/CxPybind.hpp @@ -267,6 +267,8 @@ class Internals void reloadPythonPlugins(); + void unloadPythonPlugins(); + std::vector> getPythonPlugins() { std::vector> plugins; @@ -718,6 +720,16 @@ inline void Internals::reloadPythonPlugins() } } +inline void Internals::unloadPythonPlugins() +{ + FilterList* filterList = m_App->getFilterList(); + for(auto&& [id, plugin] : m_PythonPlugins) + { + filterList->removePlugin(id); + } + m_PythonPlugins.clear(); +} + /** * @brief Convenience function for binding parameter constructors that folow the standard signature: * const std::string& name, const std::string& humanName, const std::string& helpText, const ValueType& defaultValue diff --git a/wrapping/python/NxPythonEmbed/NxPythonEmbed/NxPythonEmbed.hpp b/wrapping/python/NxPythonEmbed/NxPythonEmbed/NxPythonEmbed.hpp new file mode 100644 index 0000000000..cab15b5c81 --- /dev/null +++ b/wrapping/python/NxPythonEmbed/NxPythonEmbed/NxPythonEmbed.hpp @@ -0,0 +1,307 @@ +#pragma once + +#include "simplnx/SimplnxPython.hpp" +#include "simplnx/Utilities/StringUtilities.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nx::python +{ +namespace py = pybind11; + +class PyOutputRedirect +{ +public: + PyOutputRedirect() + { + auto sys = py::module_::import("sys"); + m_Stdout = sys.attr("stdout"); + m_Stderr = sys.attr("stderr"); + auto stringio = py::module_::import("io").attr("StringIO"); + m_StdoutBuffer = stringio(); + m_StderrBuffer = stringio(); + sys.attr("stdout") = m_StdoutBuffer; + sys.attr("stderr") = m_StderrBuffer; + } + + PyOutputRedirect(const PyOutputRedirect&) = delete; + PyOutputRedirect(PyOutputRedirect&&) noexcept = default; + + PyOutputRedirect& operator=(const PyOutputRedirect&) = delete; + PyOutputRedirect& operator=(PyOutputRedirect&&) noexcept = default; + + ~PyOutputRedirect() noexcept + { + try + { + auto sys = py::module_::import("sys"); + if(m_Stdout) + { + sys.attr("stdout") = m_Stdout; + } + if(m_Stderr) + { + sys.attr("stderr") = m_Stderr; + } + } catch(py::error_already_set& pyException) + { + pyException.discard_as_unraisable(__func__); + } + } + + std::string getStdoutString() + { + m_StdoutBuffer.attr("seek")(0); + return py::str(m_StdoutBuffer.attr("read")()); + } + + std::string getStderrString() + { + m_StderrBuffer.attr("seek")(0); + return py::str(m_StderrBuffer.attr("read")()); + } + +private: + py::object m_Stdout; + py::object m_Stderr; + py::object m_StdoutBuffer; + py::object m_StderrBuffer; +}; + +inline py::module_ LoadPythonModuleFromString(const std::string& modName, std::string_view strData, const std::string& filename = "") +{ + auto importLibUtil = py::module_::import("importlib.util"); + auto spec = importLibUtil.attr("spec_from_loader")(modName, py::none()); + auto mod = importLibUtil.attr("module_from_spec")(spec); + auto builtins = py::module_::import("builtins"); + auto code = builtins.attr("compile")(strData, filename, "exec"); + builtins.attr("exec")(code, mod.attr("__dict__")); + auto sys = py::module_::import("sys"); + sys.attr("modules")[modName.c_str()] = mod; + return mod; +} + +class ManualImportFinderHolder +{ +public: + ManualImportFinderHolder() = default; + + static ManualImportFinderHolder Create() + { + auto simplnxModule = py::module_::import(SIMPLNX_PYTHON_MODULE); + return ManualImportFinderHolder(simplnxModule.attr("ManualImportFinder")()); + } + + bool isNone() const + { + return m_Finder.is_none(); + } + + const py::object& getPyObject() const + { + return m_Finder; + } + + void insert(const std::string& pluginPath) + { + m_Finder.attr("insert")(pluginPath); + } + + void addToMetaPath() + { + auto sys = py::module_::import("sys"); + py::list(sys.attr("meta_path")).append(m_Finder); + } + + static ManualImportFinderHolder GetFromMetaPath() + { + auto sys = py::module_::import("sys"); + auto simplnxModule = py::module_::import(SIMPLNX_PYTHON_MODULE); + py::handle manualImportFinderType = simplnxModule.attr("ManualImportFinder"); + auto metaPath = py::list(sys.attr("meta_path")); + auto iter = std::find_if(metaPath.begin(), metaPath.end(), [manualImportFinderType](py::handle item) { return py::isinstance(item, manualImportFinderType); }); + if(iter == metaPath.end()) + { + throw std::runtime_error("ManualImportFinder is not in sys.meta_path"); + } + return ManualImportFinderHolder(py::reinterpret_borrow(*iter)); + } + +private: + ManualImportFinderHolder(py::object obj) + : m_Finder(std::move(obj)) + { + } + + py::object m_Finder; +}; + +inline std::string GetPythonPluginNameFromPath(const std::filesystem::path& path) +{ + return path.stem().string(); +} + +inline std::set GetPythonPluginListFromEnvironment() +{ + auto* var = std::getenv("SIMPLNX_PYTHON_PLUGINS"); + if(var == nullptr) + { + return {}; + } + +#if defined(_WIN32) + std::vector pluginList = nx::core::StringUtilities::split(var, ';'); +#else + std::vector pluginList = nx::core::StringUtilities::split(var, ':'); +#endif + + return std::set(std::make_move_iterator(pluginList.begin()), std::make_move_iterator(pluginList.end())); +} + +using OutputCallback = std::function; + +inline void SetupPythonEnvironmentVars(OutputCallback outputCallback = {}) +{ + constexpr const char k_PYTHONHOME[] = "PYTHONHOME"; + constexpr const char k_CONDA_PREFIX[] = "CONDA_PREFIX"; + + std::string condaPrefix; + char* condaPrefixPtr = std::getenv(k_CONDA_PREFIX); + if(condaPrefixPtr != nullptr) + { + condaPrefix = condaPrefixPtr; + if(outputCallback) + { + outputCallback(fmt::format("CONDA_PREFIX={}\n", condaPrefix)); + } + } + + std::string pythonHome; + char* pythonHomePtr = std::getenv(k_PYTHONHOME); + if(pythonHomePtr != nullptr) + { + pythonHome = pythonHomePtr; + if(outputCallback) + { + outputCallback(fmt::format("PYTHONHOME={}\n", pythonHome)); + } + } + + if(pythonHome.empty() && !condaPrefix.empty()) + { + std::string envVar = fmt::format("{}={}", k_PYTHONHOME, condaPrefix); + putenv(envVar.data()); + if(outputCallback) + { + outputCallback(envVar); + } + } +} + +enum class ExceptionType +{ + Cpp, + Python +}; + +inline std::string ExceptionTypeToString(ExceptionType type) +{ + switch(type) + { + case nx::python::ExceptionType::Cpp: { + return "C++"; + } + case nx::python::ExceptionType::Python: { + return "Python"; + } + } + return ""; +} + +struct PluginLoadErrorInfo +{ + ExceptionType type; + std::string pluginName; + std::string message; +}; +using PluginLoadErrorCallback = std::function; + +struct PythonErrorInfo +{ + ExceptionType type; + std::string message; +}; +using PythonErrorCallback = std::function; + +inline std::vector LoadPythonPlugins(const std::set& pythonPlugins, OutputCallback outputCallback = {}, PluginLoadErrorCallback pluginLoadErrorCallback = {}, + PythonErrorCallback pythonErrorCallback = {}) +{ + std::vector loadedPythonPlugins; + try + { + auto simplnxModule = py::module_::import(SIMPLNX_PYTHON_MODULE); + + ManualImportFinderHolder manualImportFinder = ManualImportFinderHolder::GetFromMetaPath(); + + if(outputCallback) + { + outputCallback(fmt::format("Loading Python plugins: {}", pythonPlugins)); + } + + for(const auto& pluginPath : pythonPlugins) + { + std::string pluginName = GetPythonPluginNameFromPath(pluginPath); + if(outputCallback) + { + outputCallback(fmt::format("Attempting to load Python plugin: '{}' from '{}'", pluginName, pluginPath)); + } + try + { + manualImportFinder.insert(pluginPath); + auto mod = py::module_::import(pluginName.c_str()); + simplnxModule.attr("load_python_plugin")(mod); + if(outputCallback) + { + outputCallback(fmt::format("Successfully loaded Python plugin '{}' from '{}'", pluginName, pluginPath)); + } + } catch(const py::error_already_set& exception) + { + if(pluginLoadErrorCallback) + { + pluginLoadErrorCallback(PluginLoadErrorInfo{ExceptionType::Python, pluginName, exception.what()}); + } + } catch(const std::exception& exception) + { + if(pluginLoadErrorCallback) + { + pluginLoadErrorCallback(PluginLoadErrorInfo{ExceptionType::Cpp, pluginName, exception.what()}); + } + } + loadedPythonPlugins.push_back(pluginName); + } + } catch(const py::error_already_set& exception) + { + if(pythonErrorCallback) + { + pythonErrorCallback(PythonErrorInfo{ExceptionType::Python, exception.what()}); + } + } catch(const std::exception& exception) + { + if(pythonErrorCallback) + { + pythonErrorCallback(PythonErrorInfo{ExceptionType::Cpp, exception.what()}); + } + } + return loadedPythonPlugins; +} +} // namespace nx::python diff --git a/wrapping/python/docs/generate_sphinx_docs.cpp b/wrapping/python/docs/generate_sphinx_docs.cpp index 48102e0c55..976e1e460d 100644 --- a/wrapping/python/docs/generate_sphinx_docs.cpp +++ b/wrapping/python/docs/generate_sphinx_docs.cpp @@ -693,13 +693,13 @@ void GeneratePythonRstFiles() IFilter::UniquePointer filter = filterListPtr->createFilter(filterHandle); // auto plugin = filterListPtr->getPlugin(filterHandle); - rstStream << filterClassName << "\n"; - rstStream << GenerateUnderline(filterClassName.size(), '-') << "\n\n"; + rstStream << filter->humanName() << "\n"; + rstStream << GenerateUnderline(filter->humanName().size(), '-') << "\n\n"; rstStream << ".. index:: pair: Filter Human Names; " << filter->humanName() << "\n"; rstStream << ".. index:: pair: Filter Class Names; " << filter->className() << "\n"; rstStream << "\n"; - rstStream << "- **UI Name**: " << filter->humanName() << "\n\n"; + // rstStream << "- **UI Name**: " << filter->humanName() << "\n\n"; rstStream << ".. _" << filterClassName << ":\n"; diff --git a/wrapping/python/docs/source/DataObjects.rst b/wrapping/python/docs/source/DataObjects.rst index e484cd0b2b..0959e33467 100644 --- a/wrapping/python/docs/source/DataObjects.rst +++ b/wrapping/python/docs/source/DataObjects.rst @@ -406,3 +406,136 @@ Geometry ---------- Please see the :ref:`Geometry` documentation. + +Pipeline +-------- + + The Pipeline object holds a collections of filters. This collection can come from loading a .d3dpipeline file, + or from programmatically appending filters into a `nx.Pipeline` object. + +.. attention:: + + This API is still in development so expect some changes + + +.. py:class:: Pipeline + + This class holds a DREAM3D-NX pipeline which consists of a number of Filter instances. + + :ivar dtype: The type of Data stored in the DataStore + + .. py:method:: from_file(file_path) + + :ivar file_path: PathLike: The filepath to the input pipeline file + + .. py:method:: to_file(name, output_file_path) + + :ivar name: str: The name of the pipeline. Can be different from the file name + :ivar output_file_path: PathLike: The filepath to the output pipeline file + + .. py:method:: execute(data_structure) + + :ivar data_structure: nx.DataStructure: + :return: The result of executing the pipeline + :rtype: nx.IFilter.ExecuteResult + + .. py:method:: size() + + :return: The number of filters in the pipeline + + .. py:method:: insert(index, filter, parameters) + + Inserts a new filter at the index specified with the specified argument dictionary + + :ivar index: The index to insert the filter at. (Zero based indexing) + :ivar filter: The filter to insert + :ivar parameters: Dictionary: The dictionary of arguments (parameters) that the filter will use when it is executed. + + .. py:method:: append(filter, parameters) + + :ivar filter: nx.IFilter: The filter to append to the pipeline + :ivar parameters: Dictionary: The dictionary of arguments (parameters) that the filter will use when it is executed. + + .. py:method:: clear() + + Removes all filters from the pipeline + + .. py:method:: remove(index) + + Removes a filter at the given index (Zero based indexing) + + .. code:: python + + # Shows modifying a pipeline that is read in from disk + # Create the DataStructure instance + data_structure = nx.DataStructure() + # Read the pipeline file + pipeline = nx.Pipeline().from_file( 'Pipelines/lesson_2.d3dpipeline') + + create_data_array_args:dict = { + "data_format": "", + "component_count":1, + "initialization_value":"0", + "numeric_type":nx.NumericType.int8, + "output_data_array":nx.DataPath("Geometry/Cell Data/data"), + "advanced_options": False, + "tuple_dimensions": [[10,20,30]] + } + pipeline[1].set_args(create_data_array_args) + # Execute the modified pipeline + result = pipeline.execute(data_structure) + nxutility.check_pipeline_result(result=result) + # Save the modified pipeline to a file. + pipeline.to_file( "Modified Pipeline", "Output/lesson_2b_modified_pipeline.d3dpipeline") + +.. py:class:: PipelineFilter + + This class represents a filter in a Pipeline object. It can be modified in place with a new + set of parameters and the pipeline run again. + + .. py:method:: get_args() + + Returns the dictionary of parameters for a filter + + :return: The parameter dictionary for the filter + :rtype: Dictionary + + + .. py:method:: set_args(parameter_dictionary) + + Sets the dictionary of parameters that a filter will use. + + :ivar parameter_dictionary: Dictionary: The dictionary of parameter arguments for the filter. + + .. py:method:: get_filter() + + Returns the nx.IFilter object + + .. code:: python + + """ + This shows how to loop on a pipeline making changes each loop. + Filter [0] is the ReadAngDataFilter which we will need to adjust the input file + Filter [5] is the image writing filter where we need to adjust the output file + """ + + for i in range(1, 6): + # Create the DataStructure instance + data_structure = nx.DataStructure() + # Read the pipeline file + pipeline = nx.Pipeline().from_file( 'Pipelines/lesson_2_ebsd.d3dpipeline') + # Get the parameter dictionary for the first filter and + # modify the input file. Then set the modified dictionary back into + # the pipeine at the same location + read_ang_parameters = pipeline[0].get_args() + read_ang_parameters["input_file"] = f"Data/Small_IN100/Slice_{i}.ang" + pipeline[0].set_args(read_ang_parameters) + + # Do the same modification here for the 5th filter in the pipeline + write_image_parameters = pipeline[5].get_args() + write_image_parameters["file_name"] = f"Output/Edax_IPF_Colors/Small_IN100_Slice_{i}.png" + pipeline[5].set_args(write_image_parameters) + + # Execute the modified pipeline + result = pipeline.execute(data_structure) + nxutility.check_pipeline_result(result=result) diff --git a/wrapping/python/docs/source/Overview.rst b/wrapping/python/docs/source/Overview.rst index dcfa6e18a1..22b8d51c02 100644 --- a/wrapping/python/docs/source/Overview.rst +++ b/wrapping/python/docs/source/Overview.rst @@ -163,6 +163,7 @@ scales, allowing for connections and correlations to be assessed. :target: path Mapping between AttributeMatrix using **Cell Data** "FeatureIds" to link to the feature data and **Cell Data** "Phases" to link to the Ensemble Data + -------------- .. figure:: Images/Elements_Features_Ensembles.png diff --git a/wrapping/python/examples/scripts/basic_arrays.py b/wrapping/python/examples/scripts/basic_arrays.py index 4b987b3479..19ab0a5713 100644 --- a/wrapping/python/examples/scripts/basic_arrays.py +++ b/wrapping/python/examples/scripts/basic_arrays.py @@ -110,6 +110,15 @@ nxtest.check_filter_result(nx.CreateDataGroup, result) +#------------------------------------------------------------------------------ +# Check if that object exists in the DataStructure +#------------------------------------------------------------------------------ +value = data_structure.exists("/Some/Path/To/Group") +print(f'The path "/Some/Path/To/Group" exists: {value} ') + +value = data_structure.exists("/Some/Path/To/NonExistantGroup") +print(f'The path "/Some/Path/To/NonExistantGroup" exists: {value} ') + #------------------------------------------------------------------------------ # Create 1D Array #------------------------------------------------------------------------------ diff --git a/wrapping/python/plugins/ExamplePlugin/ExampleFilter1.py b/wrapping/python/plugins/ExamplePlugin/ExampleFilter1.py index 799e96bde0..9c7f4c46f6 100644 --- a/wrapping/python/plugins/ExamplePlugin/ExampleFilter1.py +++ b/wrapping/python/plugins/ExamplePlugin/ExampleFilter1.py @@ -216,7 +216,7 @@ def preflight_impl(self, data_structure: nx.DataStructure, args: dict, message_h file_list: nx.GeneratedFileListParameter.ValueType = [ExampleFilter1.PARAM17_KEY].generate() message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Preflight: {input_dir_path}')) - return nx.IFilter.PreflightResult() + return nx.IFilter.PreflightResult(preflight_values=[nx.IFilter.PreflightValue('name', 'value')]) def execute_impl(self, data_structure: nx.DataStructure, args: dict, message_handler: nx.IFilter.MessageHandler, should_cancel: nx.AtomicBoolProxy) -> nx.IFilter.ExecuteResult: """ This method actually executes the filter algorithm and reports results. diff --git a/wrapping/python/plugins/ExamplePlugin/Plugin.py b/wrapping/python/plugins/ExamplePlugin/Plugin.py index e22f969ead..e68edda334 100644 --- a/wrapping/python/plugins/ExamplePlugin/Plugin.py +++ b/wrapping/python/plugins/ExamplePlugin/Plugin.py @@ -3,11 +3,33 @@ Insert documentation here. """ -from ExamplePlugin.ExampleFilter1 import ExampleFilter1 -from ExamplePlugin.ExampleFilter2 import ExampleFilter2 -from ExamplePlugin.CreateArray import CreateArrayFilter -from ExamplePlugin.InitializeData import InitializeDataPythonFilter -from ExamplePlugin.TemplateFilter import TemplateFilter +_filters = [] + +try: + from ExamplePlugin.ExampleFilter1 import ExampleFilter1 + _filters.append(ExampleFilter1) +except ImportError: + pass +try: + from ExamplePlugin.ExampleFilter2 import ExampleFilter2 + _filters.append(ExampleFilter2) +except ImportError: + pass +try: + from ExamplePlugin.CreateArray import CreateArrayFilter + _filters.append(CreateArrayFilter) +except ImportError: + pass +try: + from ExamplePlugin.InitializeData import InitializeDataPythonFilter + _filters.append(InitializeDataPythonFilter) +except ImportError: + pass +try: + from ExamplePlugin.TemplateFilter import TemplateFilter + _filters.append(TemplateFilter) +except ImportError: + pass # FILTER_INCLUDE_INSERT @@ -30,4 +52,4 @@ def vendor(self) -> str: return 'Description' def get_filters(self): - return [ExampleFilter1,ExampleFilter2,CreateArrayFilter,InitializeDataPythonFilter,TemplateFilter] # FILTER_NAME_INSERT + return _filters # FILTER_NAME_INSERT diff --git a/wrapping/python/plugins/ExamplePlugin/__init__.py b/wrapping/python/plugins/ExamplePlugin/__init__.py index 8117594a4e..b47890133c 100644 --- a/wrapping/python/plugins/ExamplePlugin/__init__.py +++ b/wrapping/python/plugins/ExamplePlugin/__init__.py @@ -1,15 +1,34 @@ from ExamplePlugin.Plugin import ExamplePlugin -from ExamplePlugin.ExampleFilter1 import ExampleFilter1 -from ExamplePlugin.ExampleFilter2 import ExampleFilter2 -from ExamplePlugin.CreateArray import CreateArrayFilter -from ExamplePlugin.InitializeData import InitializeDataPythonFilter -from ExamplePlugin.TemplateFilter import TemplateFilter +__all__ = ['ExamplePlugin', 'get_plugin'] + +try: + from ExamplePlugin.ExampleFilter1 import ExampleFilter1 + __all__.append('ExampleFilter1') +except ImportError: + pass +try: + from ExamplePlugin.ExampleFilter2 import ExampleFilter2 + __all__.append('ExampleFilter2') +except ImportError: + pass +try: + from ExamplePlugin.CreateArray import CreateArrayFilter + __all__.append('CreateArrayFilter') +except ImportError: + pass +try: + from ExamplePlugin.InitializeData import InitializeDataPythonFilter + __all__.append('InitializeDataPythonFilter') +except ImportError: + pass +try: + from ExamplePlugin.TemplateFilter import TemplateFilter + __all__.append('TemplateFilter') +except ImportError: + pass # FILTER_INCLUDE_INSERT def get_plugin(): return ExamplePlugin() - -__all__ = ['ExamplePlugin','ExampleFilter1', 'ExampleFilter2', 'CreateArrayFilter', 'InitializeDataPythonFilter', 'TemplateFilter', 'get_plugin'] # FILTER_NAME_INSERT -