Merge pull request #10 from MiraGeoscience/GEOPY-1032

GEOPY-1032

las_geoh5/import_files/driver.py

@@ -10,9 +10,12 @@
 import logging
 import sys
 from multiprocessing import Pool
+from pathlib import Path
+from shutil import move
 from time import time
 
 import lasio
+from geoh5py import Workspace
 from geoh5py.shared.utils import fetch_active_workspace
 from geoh5py.ui_json import InputFile
 from tqdm import tqdm
@@ -49,7 +52,8 @@ def run(filepath: str):  # pylint: disable=too-many-locals
     ifile = InputFile.read_ui_json(filepath)
 
     logger.info(
-        "Importing las file data to workspace %s.", ifile.data["geoh5"].h5file.stem
+        "Importing las file data to workspace %s.geoh5.",
+        ifile.data["geoh5"].h5file.stem,
     )
 
     translator = LASTranslator(
@@ -59,7 +63,9 @@ def run(filepath: str):  # pylint: disable=too-many-locals
         collar_z=ifile.data["collar_z_name"],
     )
 
+    workspace = Workspace()
     begin_reading = time()
+
     with Pool() as pool:
         futures = []
         for file in tqdm(ifile.data["files"].split(";"), desc="Reading las files"):
@@ -68,37 +74,55 @@ def run(filepath: str):  # pylint: disable=too-many-locals
             )
 
         lasfiles = [future.get() for future in futures]
+
     end_reading = time()
     logger.info(
         elapsed_time_logger(begin_reading, end_reading, "Finished reading las files")
     )
 
-    with fetch_active_workspace(ifile.data["geoh5"], mode="a") as geoh5:
+    with fetch_active_workspace(ifile.data["geoh5"]) as geoh5:
         dh_group = geoh5.get_entity(ifile.data["drillhole_group"].uid)[0]
-        logger.info(
-            "Saving drillhole data into drillhole group %s under property group %s",
-            dh_group.name,
-            ifile.data["name"],
-        )
-        begin_saving = time()
-        _ = las_to_drillhole(
-            geoh5,
-            lasfiles,
-            dh_group,
-            ifile.data["name"],
-            translator=translator,
-            skip_empty_header=ifile.data["skip_empty_header"],
-        )
-        end_saving = time()
-        logger.info(
-            elapsed_time_logger(
-                begin_saving, end_saving, "Finished saving drillhole data"
-            )
-        )
+        dh_group = dh_group.copy(parent=workspace)
 
+    logger.info(
+        "Saving drillhole data into drillhole group %s under property group %s",
+        dh_group.name,
+        ifile.data["name"],
+    )
+    begin_saving = time()
+    las_to_drillhole(
+        workspace,
+        lasfiles,
+        dh_group,
+        ifile.data["name"],
+        translator=translator,
+        skip_empty_header=ifile.data["skip_empty_header"],
+    )
+    end_saving = time()
+    logger.info(
+        elapsed_time_logger(begin_saving, end_saving, "Finished saving drillhole data")
+    )
     end = time()
     logger.info(elapsed_time_logger(start, end, "All done."))
 
+    if ifile.data["monitoring_directory"]:
+        working_path = Path(ifile.data["monitoring_directory"]) / ".working"
+        working_path.mkdir(exist_ok=True)
+        temp_geoh5 = f"temp{time():.3f}.geoh5"
+        workspace.save_as(working_path / temp_geoh5)
+        workspace.close()
+        move(
+            working_path / temp_geoh5,
+            Path(ifile.data["monitoring_directory"]) / temp_geoh5,
+        )
+
+    else:
+        geoh5_path = geoh5.h5file
+        geoh5.h5file.unlink()
+        workspace.save_as(geoh5_path)
+
+    workspace.close()
+
 
 if __name__ == "__main__":
     run(sys.argv[1])

las_geoh5/import_las.py

@@ -13,7 +13,7 @@
 import lasio
 import numpy as np
 from geoh5py import Workspace
-from geoh5py.groups import DrillholeGroup, PropertyGroup
+from geoh5py.groups import DrillholeGroup
 from geoh5py.objects import Drillhole
 from geoh5py.shared import Entity
 from tqdm import tqdm
@@ -209,7 +209,7 @@ def add_survey(survey: str | Path, drillhole: Drillhole) -> Drillhole:
 def add_data(
     drillhole: Drillhole,
     lasfile: lasio.LASFile,
-    property_group: PropertyGroup,
+    group_name: str,
 ) -> Drillhole:
     """
     Add data from las file curves to drillhole.
@@ -222,45 +222,62 @@ def add_data(
     """
 
     depths = get_depths(lasfile)
-    kwargs: dict[str, Any] = {**depths}
-    for curve in [
-        k for k in lasfile.curves if k.mnemonic not in ["DEPT", "DEPTH", "TO"]
-    ]:
+    if "depth" in depths:
+        method_name = "validate_depth_data"
+        locations = depths["depth"]
+    else:
+        method_name = "validate_interval_data"
+        locations = depths["from-to"]
+
+    try:
+        property_group = getattr(drillhole, method_name)(
+            "noname",
+            locations,
+            np.zeros_like(locations),
+            property_group=group_name,
+            collocation_distance=1e-4,
+        )
+    except ValueError as err:
+        msg = (
+            f"validate_depth_data call failed with message:\n{err.args[0]}. "
+            f"Skipping import for drillhole {drillhole.name}."
+        )
+        warnings.warn(msg)
+
+        # TODO: Increment property group name if it already exists and the depth
+        # Sampling is different.  Could try removing the try/except block once
+        # done and see if error start to appear.
+
+        return drillhole
+
+    kwargs: dict[str, Any] = {}
+    for curve in tqdm(
+        [k for k in lasfile.curves if k.mnemonic not in ["DEPT", "DEPTH", "TO"]]
+    ):
         name = curve.mnemonic
         if drillhole.get_data(name):
             msg = f"Drillhole '{drillhole.name}' already contains '{name}' data"
             warnings.warn(msg)
             continue
 
-        kwargs["values"] = curve.data
+        kwargs[name] = {"values": curve.data, "association": "DEPTH"}
 
         is_referenced = any(name in k.mnemonic for k in lasfile.params)
         is_referenced &= any(k.descr == "REFERENCE" for k in lasfile.params)
         if is_referenced:
-            kwargs["values"] = kwargs["values"].astype(int)
+            kwargs[name]["values"] = kwargs[name]["values"].astype(int)
             value_map = {
                 k.mnemonic: k.value for k in lasfile.params if name in k.mnemonic
             }
             value_map = {int(k.split()[1][1:-1]): v for k, v in value_map.items()}
-            kwargs["value_map"] = value_map
-            kwargs["type"] = "referenced"
+            kwargs[name]["value_map"] = value_map
+            kwargs[name]["type"] = "referenced"
 
         existing_data = drillhole.workspace.get_entity(name)[0]
         if existing_data and isinstance(existing_data, Entity):
-            kwargs["entity_type"] = existing_data.entity_type
+            kwargs[name]["entity_type"] = existing_data.entity_type
 
-        try:
-            drillhole.add_data({name: kwargs}, property_group=property_group)
-        except ValueError as err:
-            msg = (
-                f"ValueError raised trying to add data '{name}' to "
-                f"drillhole '{drillhole.name}' with message:\n{err.args[0]}."
-            )
-            warnings.warn(msg)
-
-        # TODO: Increment property group name if it already exists and the depth
-        # Sampling is different.  Could try removing the try/except block once
-        # done and see if error start to appear.
+    drillhole.add_data(kwargs, property_group=property_group)
 
     return drillhole
 
@@ -269,7 +286,7 @@ def create_or_append_drillhole(
     workspace: Workspace,
     lasfile: lasio.LASFile,
     drillhole_group: DrillholeGroup,
-    group_name: str | None = None,
+    group_name: str,
     translator: LASTranslator | None = None,
 ) -> Drillhole:
     """
@@ -310,11 +327,7 @@ def create_or_append_drillhole(
             f"Drillhole {name} exists in workspace but is not a Drillhole object."
         )
 
-    pg_type = "Interval table" if "TO" in lasfile.curves else "Depth table"
-    property_group = drillhole.find_or_create_property_group(
-        name=group_name, property_group_type=pg_type, association="DEPTH"
-    )
-    drillhole = add_data(drillhole, lasfile, property_group)
+    drillhole = add_data(drillhole, lasfile, group_name)
 
     return drillhole
 
@@ -323,7 +336,7 @@ def las_to_drillhole(  # pylint: disable=too-many-arguments
     workspace: Workspace,
     data: lasio.LASFile | list[lasio.LASFile],
     drillhole_group: DrillholeGroup,
-    property_group: str | None = None,
+    property_group: str,
     survey: Path | list[Path] | None = None,
     translator: LASTranslator | None = None,
     skip_empty_header: bool = False,
@@ -349,7 +362,6 @@ def las_to_drillhole(  # pylint: disable=too-many-arguments
     if translator is None:
         translator = LASTranslator()
 
-    drillhole = None
     for datum in tqdm(data):
         collar = get_collar(datum, translator)
         if all(k == 0 for k in collar) and skip_empty_header:
@@ -361,6 +373,4 @@ def las_to_drillhole(  # pylint: disable=too-many-arguments
         ind = [drillhole.name == s.name.rstrip(".las") for s in survey]
         if any(ind):
             survey_path = survey[np.where(ind)[0][0]]
-            drillhole = add_survey(survey_path, drillhole)
-
-    return drillhole
+            _ = add_survey(survey_path, drillhole)