PY: Add infrastructure to allow python plugins to be reloaded in a GU…

…I. (#910)

Signed-off-by: Jared Duffey <[email protected]>
Signed-off-by: Michael Jackson <[email protected]>
Co-authored-by: Michael Jackson <[email protected]>
Co-authored-by: Jessica Marquis <[email protected]>

CMakeLists.txt

@@ -461,9 +461,11 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Parameters/util/DynamicTableInfo.hpp
   ${SIMPLNX_SOURCE_DIR}/Parameters/util/ReadCSVData.hpp
 
+  ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineFilter.hpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineNode.hpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Pipeline.hpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/PipelineFilter.hpp
+  ${SIMPLNX_SOURCE_DIR}/Pipeline/PlaceholderFilter.hpp
 
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/AbstractPipelineMessage.hpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/FilterPreflightMessage.hpp
@@ -664,9 +666,11 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Parameters/util/ReadCSVData.cpp
   ${SIMPLNX_SOURCE_DIR}/Parameters/util/DynamicTableInfo.cpp
 
+  ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineFilter.cpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/AbstractPipelineNode.cpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Pipeline.cpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/PipelineFilter.cpp
+  ${SIMPLNX_SOURCE_DIR}/Pipeline/PlaceholderFilter.cpp
 
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/AbstractPipelineMessage.cpp
   ${SIMPLNX_SOURCE_DIR}/Pipeline/Messaging/FilterPreflightMessage.cpp

src/Plugins/SimplnxCore/CMakeLists.txt

@@ -368,18 +368,6 @@ string(HEX ${PYTHON_PLUGIN_TEMPLATE} PYTHON_PLUGIN_TEMPLATE)
 string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE ${PYTHON_PLUGIN_TEMPLATE})
 string(APPEND PYTHON_PLUGIN_TEMPLATE "0x00")
 
-if(WINDOWS)
-  file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginTemplate.bat" PYTHON_PLUGIN_TEMPLATE_BAT)
-  string(HEX ${PYTHON_PLUGIN_TEMPLATE_BAT} PYTHON_PLUGIN_TEMPLATE_BAT)
-  string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE_BAT ${PYTHON_PLUGIN_TEMPLATE_BAT})
-  string(APPEND PYTHON_PLUGIN_TEMPLATE_BAT "0x00")
-else()
-  file(READ "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginTemplate.sh" PYTHON_PLUGIN_TEMPLATE_BAT)
-  string(HEX ${PYTHON_PLUGIN_TEMPLATE_BAT} PYTHON_PLUGIN_TEMPLATE_BAT)
-  string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\0," PYTHON_PLUGIN_TEMPLATE_BAT ${PYTHON_PLUGIN_TEMPLATE_BAT})
-  string(APPEND PYTHON_PLUGIN_TEMPLATE_BAT "0x00")
-endif()
-
 cmpConfigureFileWithMD5Check(CONFIGURED_TEMPLATE_PATH "${${PLUGIN_NAME}_SOURCE_DIR}/src/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.in.hpp"
                                GENERATED_FILE_PATH "${${PLUGIN_NAME}_BINARY_DIR}/generated/${PLUGIN_NAME}/utils/PythonPluginSourceTemplate.hpp")
 

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.cpp

@@ -35,7 +35,6 @@ Result<> GeneratePythonSkeleton::operator()()
   }
   else
   {
-    return nx::core::WritePythonPluginFiles(m_InputValues->pluginOutputDir, m_InputValues->pluginName, m_InputValues->pluginName, "Description", m_InputValues->filterNames,
-                                            m_InputValues->createBatchShellScript, m_InputValues->anacondaEnvName);
+    return nx::core::WritePythonPluginFiles(m_InputValues->pluginOutputDir, m_InputValues->pluginName, m_InputValues->pluginName, "Description", m_InputValues->filterNames);
   }
 }

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/GeneratePythonSkeleton.hpp

@@ -20,8 +20,6 @@ struct SIMPLNXCORE_EXPORT GeneratePythonSkeletonInputValues
   std::string pluginName;
   std::string pluginHumanName;
   std::string filterNames;
-  bool createBatchShellScript;
-  std::string anacondaEnvName;
 };
 
 /**

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.cpp

@@ -66,15 +66,10 @@ Parameters GeneratePythonSkeletonFilter::parameters() const
   params.insert(
       std::make_unique<StringParameter>(k_PluginFilterNames, "Filter Names (comma-separated)", "The names of filters that will be created, separated by commas (,).", "FirstFilter,SecondFilter"));
 
-  params.insertLinkableParameter(
-      std::make_unique<BoolParameter>(k_CreateBatchFile_Key, "Create Anaconda Init Batch/Shell Script", "Generates a script file that can be used to export needed environment variables", false));
-  params.insert(std::make_unique<StringParameter>(k_AnacondaEnvName_Key, "Anaconda Environment Name", "The name of the Anaconda environment.", "nxpython"));
-
   params.linkParameters(k_UseExistingPlugin_Key, k_PluginName_Key, false);
   params.linkParameters(k_UseExistingPlugin_Key, k_PluginHumanName_Key, false);
   params.linkParameters(k_UseExistingPlugin_Key, k_PluginOutputDirectory_Key, false);
   params.linkParameters(k_UseExistingPlugin_Key, k_PluginInputDirectory_Key, true);
-  params.linkParameters(k_CreateBatchFile_Key, k_AnacondaEnvName_Key, true);
   return params;
 }
 
@@ -99,20 +94,41 @@ IFilter::PreflightResult GeneratePythonSkeletonFilter::preflightImpl(const DataS
 
   auto filterList = StringUtilities::split(filterNames, ',');
 
+  std::stringstream preflightUpdatedValue;
+
   std::string pluginPath = fmt::format("{}{}{}", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName);
   if(useExistingPlugin)
   {
     pluginPath = pluginInputDir.string();
   }
+  std::string fullPath = fmt::format("{}{}{}{}Plugin.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+  if(std::filesystem::exists({fullPath}))
+  {
+    fullPath = "[REPLACE]: " + fullPath;
+  }
+  else
+  {
+    fullPath = "[New]: " + fullPath;
+  }
+  preflightUpdatedValue << fullPath << '\n';
 
-  std::stringstream preflightUpdatedValue;
+  fullPath = fmt::format("{}{}{}{}__init__.py", pluginOutputDir.string(), std::string{fs::path::preferred_separator}, pluginName, std::string{fs::path::preferred_separator});
+  if(std::filesystem::exists({fullPath}))
+  {
+    fullPath = "[REPLACE]: " + fullPath;
+  }
+  else
+  {
+    fullPath = "[New]: " + fullPath;
+  }
+  preflightUpdatedValue << fullPath << '\n';
 
   for(const auto& filterName : filterList)
   {
-    std::string fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName);
+    fullPath = fmt::format("{}{}{}.py", pluginPath, std::string{fs::path::preferred_separator}, filterName);
     if(std::filesystem::exists({fullPath}))
     {
-      fullPath = "[EXISTS]: " + fullPath;
+      fullPath = "[REPLACE]: " + fullPath;
     }
     else
     {
@@ -139,8 +155,6 @@ Result<> GeneratePythonSkeletonFilter::executeImpl(DataStructure& dataStructure,
   inputValues.pluginName = filterArgs.value<StringParameter::ValueType>(k_PluginName_Key);
   inputValues.pluginHumanName = filterArgs.value<StringParameter::ValueType>(k_PluginHumanName_Key);
   inputValues.filterNames = filterArgs.value<StringParameter::ValueType>(k_PluginFilterNames);
-  inputValues.createBatchShellScript = filterArgs.value<BoolParameter::ValueType>(k_CreateBatchFile_Key);
-  inputValues.anacondaEnvName = filterArgs.value<StringParameter::ValueType>(k_AnacondaEnvName_Key);
 
   return GeneratePythonSkeleton(dataStructure, messageHandler, shouldCancel, &inputValues)();
 }

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/GeneratePythonSkeletonFilter.hpp

@@ -29,8 +29,6 @@ class SIMPLNXCORE_EXPORT GeneratePythonSkeletonFilter : public IFilter
   static inline constexpr StringLiteral k_PluginHumanName_Key = "plugin_human_name";
   static inline constexpr StringLiteral k_PluginInputDirectory_Key = "plugin_input_directory";
   static inline constexpr StringLiteral k_PluginOutputDirectory_Key = "plugin_output_directory";
-  static inline constexpr StringLiteral k_CreateBatchFile_Key = "create_batch_shell_script";
-  static inline constexpr StringLiteral k_AnacondaEnvName_Key = "anaconda_env_name";
   static inline constexpr StringLiteral k_PluginFilterNames = "filter_names";
 
   /**

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonFilterTemplate.py

@@ -2,11 +2,6 @@
 import simplnx as nx
 
 class #PYTHON_FILTER_NAME#:
-  """
-  This section should contain the 'keys' that store each parameter. The value of the key should be snake_case. The name
-  of the value should be ALL_CAPITOL_KEY
-  """
-  TEST_KEY = 'test'
 
 # -----------------------------------------------------------------------------
 # These methods should not be edited
@@ -59,35 +54,91 @@ def default_tags(self) -> List[str]:
     :rtype: list
     """
     return ['python', '#PYTHON_FILTER_HUMAN_NAME#']
-   
+
+
+  """
+  This section should contain the 'keys' that store each parameter. The value of the key should be snake_case. The name
+  of the value should be ALL_CAPITOL_KEY
+  """
+  ARRAY_PATH_KEY = 'output_array_path'
+  NUM_TUPLES_KEY = 'num_tuples'
+
   def parameters(self) -> nx.Parameters:
-    """This function defines the parameters that are needed by the filter. Parameters collect the values from the user
-       or through a pipeline file.
+    """This function defines the parameters that are needed by the filter. Parameters collect the values from the user interface
+    and pack them up into a dictionary for use in the preflight and execute methods.
     """
     params = nx.Parameters()
 
-    params.insert(nx.Float64Parameter(#PYTHON_FILTER_NAME#.TEST_KEY, 'Test', '', 0.0))
+    params.insert(nx.ArrayCreationParameter(#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY, 'Created Array', 'Array storing the data', nx.DataPath()))
+
+    params.insert(nx.UInt64Parameter(#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY, 'Num Tuples', 'The number of tuples the array will have', 0))
 
     return params
 
   def preflight_impl(self, data_structure: nx.DataStructure, args: dict, message_handler: nx.IFilter.MessageHandler, should_cancel: nx.AtomicBoolProxy) -> nx.IFilter.PreflightResult:
     """This method preflights the filter and should ensure that all inputs are sanity checked as best as possible. Array
-    sizes can be checked if the arrays are actually know at preflight time. Some filters will not be able to report output
-    array sizes during preflight (segmentation filters for example).
+    sizes can be checked if the array sizes are actually known at preflight time. Some filters will not be able to report output
+    array sizes during preflight (segmentation filters for example). If in doubt, set the tuple dimensions of an array to [1].
     :returns:
     :rtype: nx.IFilter.PreflightResult
     """
-    value: float = args[#PYTHON_FILTER_NAME#.TEST_KEY]
-    message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Preflight: {value}'))
-    return nx.IFilter.PreflightResult()
+
+    # Extract the values from the user interface from the 'args' 
+    data_array_path: nx.DataPath = args[#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY]
+    num_tuples: int = args[#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY]
+
+    # Create an OutputActions object to hold any DataStructure modifications that we are going to make
+    output_actions = nx.OutputActions()
+
+    # Create the Errors and Warnings Lists to commuicate back to the user if anything has gone wrong
+    # errors = []
+    # warnings = []
+    # preflight_values = []
+
+    # Validate that the number of tuples > 0, otherwise return immediately with an error message
+    if num_tuples == 0:
+      return nx.IFilter.PreflightResult(nx.OutputActions(), [nx.Error(-65020, "The number of tuples should be at least 1.")])
+
+    # Append a "CreateArrayAction" 
+    output_actions.append_action(nx.CreateArrayAction(nx.DataType.float32, [num_tuples], [1], data_array_path))
+
+    # Send back any messages that will appear in the "Output" widget in the UI. This is optional.
+    message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f"Creating array at: '{data_array_path.to_string('/')}' with {num_tuples} tuples"))
+
+    # Return the output_actions so the changes are reflected in the User Interface.
+    return nx.IFilter.PreflightResult(output_actions=output_actions, errors=None, warnings=None, preflight_values=None)
 
   def execute_impl(self, data_structure: nx.DataStructure, args: dict, message_handler: nx.IFilter.MessageHandler, should_cancel: nx.AtomicBoolProxy) -> nx.IFilter.ExecuteResult:
     """ This method actually executes the filter algorithm and reports results.
     :returns:
     :rtype: nx.IFilter.ExecuteResult
     """
+    # Extract the values from the user interface from the 'args'
+    # This is basically repeated from the preflight because the variables are scoped to the method()
+    data_array_path: nx.DataPath = args[#PYTHON_FILTER_NAME#.ARRAY_PATH_KEY]
+    num_tuples: int = args[#PYTHON_FILTER_NAME#.NUM_TUPLES_KEY]
+
+    # At this point the array has been allocated with the proper number of tuples and components. And we can access
+    # the data array through a numpy view.
+    data_array_view = data_structure[data_array_path].npview()
+    # Now you can go off and use numpy or anything else that can use a numpy view to modify the data
+    # or use the data in another calculation. Any operation that works on the numpy view in-place
+    # has an immediate effect within the DataStructure
+
+    # -----------------------------------------------------------------------------
+    # If you want to send back progress on your filter, you can use the message_handler
+    # -----------------------------------------------------------------------------
+    message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Information Message: Num_Tuples = {num_tuples}'))
+
+    # -----------------------------------------------------------------------------
+    # If you have a long running process, check the should_cancel to see if the user cancelled the filter
+    # -----------------------------------------------------------------------------
+    if not should_cancel:
+      return nx.Result()
+
 
-    value: float = args[#PYTHON_FILTER_NAME#.TEST_KEY]
-    message_handler(nx.IFilter.Message(nx.IFilter.Message.Type.Info, f'Execute: {value}'))
     return nx.Result()
 
+
+
+

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginInitTemplate.py

@@ -1,9 +1,16 @@
+
+"""
+Insert documentation here for #PLUGIN_NAME#
+"""
 from #PLUGIN_NAME#.Plugin import #PLUGIN_NAME#
 
-#PLUGIN_IMPORT_CODE## FILTER_INCLUDE_INSERT
+__all__ = ['#PLUGIN_NAME#', 'get_plugin']
 
-def get_plugin():
-  return #PLUGIN_NAME#()
+"""
+This section conditionally tries to import each filter
+"""
 
-__all__ = [#PLUGIN_FILTER_LIST#] # FILTER_NAME_INSERT
+#PLUGIN_IMPORT_CODE#
 
+def get_plugin():
+  return #PLUGIN_NAME#()

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginSourceTemplate.in.hpp

@@ -37,13 +37,4 @@ inline const std::string PluginInitPythonFile()
   return {k_PluginInitPythonFileCharArray};
 }
 
-// clang-format off
-static const char k_PluginBatchFileCharArray[] = {@PYTHON_PLUGIN_TEMPLATE_BAT@};
-// clang-format on
-
-inline const std::string PluginBatchFile()
-{
-  return {k_PluginBatchFileCharArray};
-}
-
 }; // namespace nx::core

src/Plugins/SimplnxCore/src/SimplnxCore/utils/PythonPluginTemplate.py

@@ -2,26 +2,51 @@
 Insert documentation here.
 """
 
-#PLUGIN_IMPORT_CODE## FILTER_INCLUDE_INSERT
+_filters = []
+
+"""
+This section conditionally tries to import each filter
+"""
+
+#PLUGIN_IMPORT_CODE#
 
 import simplnx as nx
 
 class #PLUGIN_NAME#:
+  """
+  This class defines the plugin's basic information. 
+  """
   def __init__(self) -> None:
     pass
 
   def id(self) -> nx.Uuid:
+    """This returns the UUID of the filter. Each Plugin has a unique UUID value. DO NOT change this.
+    :return: The Plugins's Uuid value
+    :rtype: string
+    """
     return nx.Uuid('#PLUGIN_UUID#')
 
   def name(self) -> str:
+    """The returns the name of plugin. DO NOT Change this
+    :return: The name of the plugin
+    :rtype: string
+    """    
     return '#PLUGIN_NAME#'
 
   def description(self) -> str:
+    """This returns the description of the plugin. Feel free to edit this.
+    :return: The plugin's descriptive text
+    :rtype: string
+    """    
     return '#PLUGIN_SHORT_NAME#'
 
   def vendor(self) -> str:
+    """This returns the name of the organization that is writing the plugin. Feel free to edit this.
+    :return: The plugin's organization
+    :rtype: string
+    """
     return '#PLUGIN_DESCRIPTION#'
 
   def get_filters(self):
-    return [#PLUGIN_FILTER_LIST#] # FILTER_NAME_INSERT
+    return _filters