perf: Add the summary folder to display all files

geochemistrypi/data_mining/cli_pipeline.py

@@ -44,7 +44,7 @@
 from .process.decompose import DecompositionModelSelection
 from .process.detect import AbnormalDetectionModelSelection
 from .process.regress import RegressionModelSelection
-from .utils.base import check_package, clear_output, create_geopi_output_dir, get_os, install_package, log, save_data, show_warning
+from .utils.base import check_package, clear_output, copy_files, create_geopi_output_dir, get_os, install_package, log, save_data, show_warning
 from .utils.mlflow_utils import retrieve_previous_experiment_id
 
 
@@ -681,4 +681,10 @@ def cli_pipeline(training_data_path: str, application_data_path: Optional[str] =
                     else:
                         model_inference(inference_data_fe_selected, is_inference, run, transformer_config, transform_pipeline)
                     clear_output()
+
+    # <--- Data Dumping --->
+    # In this section, convert the data in the output to the summary.
+    GEOPI_OUTPUT_PATH = os.getenv("GEOPI_OUTPUT_PATH")
+    GEOPI_SUMMARY_PATH = os.getenv("GEOPI_SUMMARY_PATH")
+    copy_files(GEOPI_OUTPUT_PATH, GEOPI_SUMMARY_PATH)
     mlflow.end_run()

geochemistrypi/data_mining/constants.py

@@ -15,6 +15,9 @@
 # the root directory where all the output stays
 OUTPUT_PATH = os.path.join(WORKING_PATH, "geopi_output")
 
+# the summary root directory where all the output stays
+SUMMARY_PATH = os.path.join(OUTPUT_PATH, "summary")
+
 # the directory where the artifact is saved within the MLflow run's artifact directory
 MLFLOW_ARTIFACT_DATA_PATH = "data"
 MLFLOW_ARTIFACT_IMAGE_STATISTIC_PATH = os.path.join("image", "statistic")

geochemistrypi/data_mining/utils/base.py

@@ -3,6 +3,7 @@
 import os
 import pickle
 import platform
+import shutil
 from typing import Optional
 
 import joblib
@@ -11,7 +12,7 @@
 from matplotlib import pyplot as plt
 from rich import print
 
-from ..constants import OUTPUT_PATH
+from ..constants import OUTPUT_PATH, SUMMARY_PATH
 
 
 def create_geopi_output_dir(experiment_name: str, run_name: str, sub_run_name: Optional[str] = None) -> None:
@@ -32,10 +33,14 @@ def create_geopi_output_dir(experiment_name: str, run_name: str, sub_run_name: O
     # timestamp = datetime.datetime.now().strftime("%m-%d-%H-%M")
     if sub_run_name:
         geopi_output_path = os.path.join(OUTPUT_PATH, experiment_name, f"{run_name}", sub_run_name)
+        geopi_summary_path = os.path.join(SUMMARY_PATH, experiment_name, f"{run_name}", sub_run_name)
     else:
         geopi_output_path = os.path.join(OUTPUT_PATH, experiment_name, f"{run_name}")
+        geopi_summary_path = os.path.join(SUMMARY_PATH, experiment_name, f"{run_name}")
     os.environ["GEOPI_OUTPUT_PATH"] = geopi_output_path
     os.makedirs(geopi_output_path, exist_ok=True)
+    os.environ["GEOPI_SUMMARY_PATH"] = geopi_summary_path
+    os.makedirs(geopi_summary_path, exist_ok=True)
 
     # Set the output artifacts path for the current run
     geopi_output_artifacts_path = os.path.join(geopi_output_path, "artifacts")
@@ -309,3 +314,20 @@ def show_warning(is_show: bool = True) -> None:
 
             os.environ["PYTHONWARNINGS"] = "ignore"
             # os.environ["PYTHONWARNINGS"] = "default"
+
+
+def copy_files(GEOPI_OUTPUT_PATH: str, GEOPI_SUMMARY_PATH: str) -> None:
+    """Copy all files from the source folder to the destination folder.
+
+    Parameters
+    ----------
+    GEOPI_OUTPUT_PATH: str
+        Source folder path.
+
+    GEOPI_SUMMARY_PATH: str
+        Destination folder path
+    """
+    for root, dirs, files in os.walk(GEOPI_OUTPUT_PATH):
+        for file in files:
+            source_file_path = os.path.join(root, file)
+            shutil.copy2(source_file_path, GEOPI_SUMMARY_PATH)