Provenance tracking

ceci/__init__.py

@@ -1,12 +1,14 @@
 """Ceci n'est pas une pipeline"""
 
+from .provenance import Provenance
 from .stage import PipelineStage
 from .pipeline import Pipeline, MiniPipeline, ParslPipeline, DryRunPipeline
+from .handle import  BaseIOHandle
 from pkg_resources import DistributionNotFound
 from pkg_resources import get_distribution
 
 try:
     __version__ = get_distribution(__name__).version
 except DistributionNotFound:  # pragma: no cover
     # package is not installed
-    pass
+    __version__ = "unknown"

ceci/handle.py

@@ -0,0 +1,20 @@
+import copy
+
+class BaseIOHandle:
+    """
+    Base class for input and output objects. For now this is pretty empty.
+    """
+    suffix = ""
+
+    def __init__(self, provenance):
+        # If provenance is provided then pull that out
+        self.provenance =  provenance
+
+    @property
+    def provenance(self):
+        return self._provenance
+
+    @provenance.setter
+    def provenance(self, provenance):
+        # always copy the provenance
+        self._provenance = copy.deepcopy(provenance)

ceci/provenance/__init__.py

@@ -0,0 +1,2 @@
+from .provenance import Provenance
+from .errors import *

ceci/provenance/errors.py

@@ -0,0 +1,22 @@
+class ProvenanceError(Exception):
+    pass
+
+
+class ProvenanceFileTypeUnknown(ProvenanceError):
+    pass
+
+
+class ProvenanceFileSchemeUnsupported(ProvenanceError):
+    pass
+
+
+class ProvenanceMissingFile(ProvenanceError):
+    pass
+
+
+class ProvenanceMissingSection(ProvenanceError):
+    pass
+
+
+class ProvenanceMissingItem(ProvenanceError):
+    pass

ceci/provenance/git.py

@@ -0,0 +1,81 @@
+from .utils import get_caller_directory
+import subprocess
+
+
+def diff(dirname=None, parent_frames=1):
+    """
+    Run git diff in the caller's directory (default) or another specified directory,
+    and return stdout+stderr
+    """
+    if dirname is None:
+        dirname = get_caller_directory(parent_frames + 1)
+
+    if dirname is None:
+        return "ERROR_GIT_NO_DIRECTORY"
+    # We use git diff head because it shows all differences,
+    # including any that have been staged but not committed.
+    try:
+        diff = subprocess.run(
+            "git diff HEAD".split(),
+            cwd=dirname,
+            universal_newlines=True,
+            timeout=5,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+
+    # There are lots of different ways this can go wrong.
+    # Here are some - any others it is probably worth knowing
+    # about
+    except subprocess.TimeoutExpired:
+        return "ERROR_GIT_TIMEOUT"
+    except UnicodeDecodeError:
+        return "ERROR_GIT_DECODING"
+    except subprocess.SubprocessError:
+        return "ERROR_GIT_OTHER"
+    except FileNotFoundError:
+        return "ERROR_GIT_NOT_RUNNABLE"
+    except OSError:
+        return "ERROR_GIT_OTHER_OSERROR"
+    # If for some reason we are running outside the main repo
+    # this will return an error too
+    if diff.returncode:
+        return "ERROR_GIT_FAIL"
+
+    return diff.stdout
+
+
+def current_revision(dirname=None, parent_frames=1):
+    """Return the git revision ID in the caller's directory (default) or another
+    specified directory.
+    """
+    if dirname is None:
+        dirname = get_caller_directory(parent_frames + 1)
+
+    if dirname is None:
+        return "ERROR_GIT_NO_DIRECTORY"
+    try:
+        rev = subprocess.run(
+            "git rev-parse HEAD".split(),
+            cwd=dirname,
+            universal_newlines=True,
+            timeout=5,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+    # Same as git diff above.
+    except subprocess.TimeoutExpired:
+        return "ERROR_GIT_TIMEOUT"
+    except UnicodeDecodeError:
+        return "ERROR_GIT_DECODING"
+    except subprocess.SubprocessError:
+        return "ERROR_GIT_OTHER"
+    except FileNotFoundError:
+        return "ERROR_GIT_NOT_RUNNABLE"
+    except OSError:
+        return "ERROR_GIT_OTHER_OSERROR"
+    # If for some reason we are running outside the main repo
+    # this will return an error too
+    if rev.returncode:
+        return "ERROR_GIT_FAIL"
+    return rev.stdout