Merge pull request #27 from AgPipeline/develop

Merge develop to main branch

.github/workflows/pylint_check.yaml → .github/workflows/testing_checks.yaml

@@ -10,6 +10,9 @@ on:
       - develop
     tags:
       - v*
+  schedule:
+    # Every 01:00 Sunday re-run the test on the main branch
+    - cron: '0 1 * * 0'
   workflow_dispatch:
 
 jobs:
@@ -19,11 +22,13 @@ jobs:
     steps:
       - name: Current python version
         run: python3 --version || echo python3 not installed
-      - name: Install Python 3.7
-        run: sudo apt-get install -y --no-install-recommends python3.7 python3-pip && sudo ln -sfn /usr/bin/python3.7 /usr/bin/python3
-        id: install_python_3_7
+      - name: Install Python 3.8
+        run: sudo apt-get install -y --no-install-recommends python3.8 python3-pip && sudo ln -sfn /usr/bin/python3.8 /usr/bin/python3
+        id: install_python_3_8
       - name: Updated python version
         run: python3 --version
+      - name: Install PDAL
+        run: sudo apt-get install -y --no-install-recommends pdal
       - name: Update pip
         run: python3 -m pip install --upgrade --no-cache-dir pip
         id: pip-install
@@ -45,24 +50,31 @@ jobs:
         id: find-python-files
       - name: Install system requirements
         shell: bash
-        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.7-dev'
+        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.8-dev'
         id: install-gdal-other-reqs
       - name: Install Python numpy
         shell: bash
         run: 'python3 -m pip install --upgrade --no-cache-dir numpy wheel terrautils'
         id: install-python-numpy
       - name: Install Python pygdal
         shell: bash
-        run: 'sudo python3 -m pip install --no-cache-dir pygdal==2.2.3.5'
+        run: 'sudo python3 -m pip install --no-cache-dir pygdal==2.2.3.*'
         id: install-python-pygdal
       - name: Install system requirements from source
         shell: bash
-        run: '[ -s "packages.txt" ] && (cat packages.txt | xargs apt-get install -y --no-install-recommends) || (echo "No addtional packages to install")'
+        run: '[ -s "packages.txt" ] && (cat packages.txt | sudo xargs apt-get install -y --no-install-recommends) || (echo "No additional packages to install")'
         id: install-system-reqs
       - name: Install Python requirements from source
         shell: bash
         run: '[ -s "requirements.txt" ] && (python3 -m pip install --no-cache-dir -r requirements.txt) || (echo "No Python packages to install")'
         id: install-python-reqs
+      - name: check
+        shell: bash
+        run: ls
+      - name: Install agpypeline for tests
+        shell: bash
+        run: python3 -m pip install -e .
+        id: agpypeline_install
       - name: Run action pylint script
         shell: bash
         run: '[ -s "action_pylint.sh" ] && (chmod +x "action_pylint.sh" && ./action_pylint.sh) || (echo "Not running special action script - empty or not found")'
@@ -76,3 +88,5 @@ jobs:
         run: cat action_pylint_files.txt
       - name: Running pylint
         run: cat action_pylint_files.txt | xargs python3 -m pylint --rcfile ./pylint.rc
+      - name: Running pytest
+        run: python3 -m pytest

.gitignore

@@ -81,6 +81,9 @@ target/
 profile_default/
 ipython_config.py
 
+# LAS files
+.las
+
 # pyenv
 .python-version
 

Dockerfile_testing

@@ -0,0 +1,95 @@
+FROM ubuntu:20.04
+LABEL maintainer="Chris Schnaufer <[email protected]>"
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Add user
+RUN useradd -u 49044 extractor \
+    && mkdir /home/extractor
+RUN chown -R extractor /home/extractor \
+    && chgrp -R extractor /home/extractor
+
+# Install the Python version we want
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        python3.8 \
+        python3-pip \
+        pdal && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python3 && \
+    ln -sfn /usr/bin/python3.8m /usr/bin/python3m && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Perform some upgrades
+RUN python3 -m pip install --upgrade --no-cache-dir pip
+RUN python3 -m pip install --upgrade --no-cache-dir setuptools
+
+# Install applications we need
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        python3-gdal \
+        gdal-bin   \
+        libgdal-dev  \
+        gcc \
+        g++ \
+        python3.8-dev && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        wheel && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        numpy && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        pygdal==3.0.4.* && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        pylint && \
+    python3 -m pip install --upgrade --no-cache-dir \
+        pdal==2.3.6 && \
+    apt-get remove -y \
+        libgdal-dev \
+        gcc \
+        g++ \
+        python3-dev && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt packages.txt /home/extractor/
+
+USER root
+
+RUN [ -s /home/extractor/packages.txt ] && \
+    (echo 'Installing packages' && \
+        apt-get update && \
+        cat /home/extractor/packages.txt | xargs apt-get install -y --no-install-recommends && \
+        rm /home/extractor/packages.txt && \
+        apt-get autoremove -y && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*) || \
+    (echo 'No packages to install' && \
+        rm /home/extractor/packages.txt)
+
+RUN [ -s /home/extractor/requirements.txt ] && \
+    (echo "Install python modules" && \
+    python -m pip install -U --no-cache-dir pip && \
+    python -m pip install --no-cache-dir setuptools && \
+    python -m pip install --no-cache-dir -r /home/extractor/requirements.txt && \
+    rm /home/extractor/requirements.txt) || \
+    (echo "No python modules to install" && \
+    rm /home/extractor/requirements.txt)
+
+# Install the library
+COPY agpypeline /home/extractor/agpypeline/agpypeline
+COPY setup.py README.md /home/extractor/agpypeline/
+COPY tests /home/extractor/agpypeline/tests
+COPY data /home/extractor/agpypeline/data
+COPY images /home/extractor/agpypeline/images
+COPY integration_tests /home/extractor/agpypeline/integration_tests
+
+RUN python3 -m pip install --upgrade /home/extractor/agpypeline
+
+USER root
+RUN chmod -R 777 /home/extractor/agpypeline
+
+USER extractor
+ENTRYPOINT ["/home/extractor/agpypeline/integration_tests/basic_algorithm.py"]

README.md

@@ -1,3 +1,8 @@
+[![license](https://img.shields.io/badge/license-BSD%203-green?logo=Open-Source-Initiative)](https://github.com/AgPipeline/agpypeline/blob/add_badges/LICENSE)
+
+[![PyPI - version](https://img.shields.io/pypi/v/agpypeline?label=current&color=green)](https://pypi.org/project/agpypeline/)
+[![PyPI - downloads](https://img.shields.io/pypi/dm/agpypeline)](https://pypi.org/project/agpypeline/)
+
 # AgPypeline
 Python library of common classes and functions
 

agpypeline/algorithm.py

@@ -1,6 +1,6 @@
 """Template class for a Transformer Algorithm
 """
-from agpypeline.environment import Environment
+from agpypeline.environment import Environment, CheckMD
 
 
 class Algorithm:
@@ -11,7 +11,7 @@ def __init__(self):
         """Initializes a class instance
         """
 
-    def perform_process(self, environment: Environment, check_md: dict, transformer_md: dict,
+    def perform_process(self, environment: Environment, check_md: CheckMD, transformer_md: dict,
                         full_md: list) -> dict:
         """Perform the processing of data
         Arguments:

agpypeline/checkmd.py

@@ -0,0 +1,22 @@
+"""CheckMD Class"""
+
+from typing import List, NamedTuple, Optional, TextIO
+
+
+# pylint: disable=invalid-name
+class CheckMD(NamedTuple):
+    """This is the CheckMD class based off of NamedTuple which can be used in
+    order to store data passed into argparse."""
+    timestamp: str
+    season: str
+    experiment: str
+    working_folder: str
+    list_files: List[TextIO]
+    container_name: Optional[str] = None
+    target_container_name: Optional[str] = None
+    trigger_name: Optional[str] = None
+    context_md: Optional[str] = None
+
+    def get_list_files(self):
+        """Returns list_files"""
+        return self.list_files

agpypeline/entrypoint.py

@@ -163,8 +163,12 @@ def load_metadata_files(metadata_files: list) -> dict:
         metadata = []
         result = {'metadata': metadata}
         for metadata_file in metadata_files:
-            if not os.path.exists(metadata_file):
-                result = __internal__.handle_error(-2, "Unable to access metadata file '%s'" % metadata_file)
+            if hasattr(metadata_file, 'name'):
+                if not metadata_file.closed:
+                    metadata_file.close()
+                metadata_file = metadata_file.name
+            if not os.path.exists(str(metadata_file)):
+                result = __internal__.handle_error(-2, "Unable to access metadata file '%s'" % metadata_file.name)
                 break
             logging.info("Loading metadata from file: '%s'", metadata_file)
             md_loaded = __internal__.load_metadata(metadata_file)
@@ -343,20 +347,21 @@ def add_parameters(parser: argparse.ArgumentParser, algorithm_instance: Algorith
         algorithm_instance: working Transformer instance
         environment_instance: the current Environment
     """
-    parser.add_argument('--debug', '-d', action='store_const',
+    parser.add_argument('-d', '--debug', action='store_const',
                         default=logging.WARN, const=logging.DEBUG,
                         help='enable debug logging (default=WARN)')
 
-    parser.add_argument('--info', '-i', action='store_const',
+    parser.add_argument('-i', '--info', action='store_const',
                         default=logging.WARN, const=logging.INFO,
                         help='enable info logging (default=WARN)')
 
     parser.add_argument('--result', nargs='?', default='all',
                         help='Direct the result of a run to one or more of (all is default): "all,file,print"')
 
-    parser.add_argument('--metadata', type=str, action='append', help='The path to the source metadata')
+    parser.add_argument('-m', '--metadata', type=argparse.FileType('rt'), action='append',
+                        help='The path to the source metadata')
 
-    parser.add_argument('--working_space', type=str,
+    parser.add_argument('-w', '--working_space', type=str, default='output',
                         help='the folder to use use as a workspace and for storing results')
 
     # Let the transformer class add parameters
@@ -368,7 +373,9 @@ def add_parameters(parser: argparse.ArgumentParser, algorithm_instance: Algorith
         algorithm_instance.add_parameters(parser)
 
     # Assume the rest of the arguments are the files
-    parser.add_argument('file_list', nargs=argparse.REMAINDER, help='additional files for transformer')
+    parser.add_argument('file_list', nargs='*', type=argparse.FileType('r'),
+                        help='additional files, folders, and other information'
+                             ' for the transformer')
 
 
 def do_work(parser: argparse.ArgumentParser, configuration_info: Configuration,
@@ -387,10 +394,12 @@ def do_work(parser: argparse.ArgumentParser, configuration_info: Configuration,
     if not transformer_instance:
         result = __internal__.handle_error(-100, "Unable to create transformer class instance for processing")
         return __internal__.handle_result(result, None, None)
-
     add_parameters(parser, algorithm_instance, transformer_instance)
     args = parser.parse_args()
 
+    if not os.path.isdir(args.working_space):
+        os.makedirs(args.working_space)
+
     # start logging system
     logging.getLogger().setLevel(args.debug if args.debug == logging.DEBUG else args.info)
 
@@ -422,5 +431,14 @@ def entrypoint(configuration_info: Configuration, algorithm_instance: Algorithm)
         configuration_info: an instance of Configuration class
         algorithm_instance: an instance of class for preparing work
     """
+    begin_process(configuration_info, algorithm_instance)
+
+
+def begin_process(configuration_info: Configuration, algorithm_instance: Algorithm):
+    """entrypoint() functionality moved here to allow users to transition over time
+    Arguments:
+        configuration_info: an instance of the Configuration class
+        algorithm_instance: an instance of class for preparing work
+    """
     parser = argparse.ArgumentParser(description=configuration_info.transformer_description)
     do_work(parser, configuration_info, algorithm_instance)