Merge pull request #226 from credo-ai/release/1.1.0

Release/1.1.0

credoai/__init__.py

@@ -2,4 +2,4 @@
 Primary interface for Credo AI Lens package
 """
 
-__version__ = "1.0.1"
+__version__ = "1.1.0"

credoai/artifacts/model/base_model.py

@@ -1,6 +1,6 @@
 """Abstract class for model artifacts used by `Lens`"""
 from abc import ABC
-from typing import List
+from typing import List, Optional
 
 from credoai.utils import ValidationError
 from credoai.utils.model_utils import get_model_info
@@ -31,16 +31,27 @@ def __init__(
         necessary_functions: List[str],
         name: str,
         model_like,
+        tags: Optional[dict] = None,
     ):
-
         self.type = type
         self.name = name
         self.model_like = model_like
+        self.tags = tags
         self.model_info = get_model_info(model_like)
         self._validate(necessary_functions)
         self._build(possible_functions)
         self._update_functionality()
 
+    @property
+    def tags(self):
+        return self._tags
+
+    @tags.setter
+    def tags(self, value):
+        if not isinstance(value, dict) and value is not None:
+            raise ValidationError("Tags must be of type dictionary")
+        self._tags = value
+
     def _build(self, function_names: List[str]):
         """
         Makes the necessary methods available in the class

credoai/artifacts/model/classification_model.py

@@ -20,13 +20,14 @@ class ClassificationModel(Model):
             the class labels probabilities for each sample.
     """
 
-    def __init__(self, name: str, model_like=None):
+    def __init__(self, name: str, model_like=None, tags=None):
         super().__init__(
             "Classification",
             ["predict", "predict_proba"],
             ["predict"],
             name,
             model_like,
+            tags,
         )
 
     def _update_functionality(self):

credoai/artifacts/model/regression_model.py

@@ -20,14 +20,8 @@ class RegressionModel(Model):
             the class labels probabilities for each sample.
     """
 
-    def __init__(self, name: str, model_like=None):
-        super().__init__(
-            "Regression",
-            ["predict"],
-            ["predict"],
-            name,
-            model_like,
-        )
+    def __init__(self, name: str, model_like=None, tags=None):
+        super().__init__("Regression", ["predict"], ["predict"], name, model_like, tags)
 
 
 class DummyRegression:

credoai/evaluators/evaluator.py

@@ -115,9 +115,9 @@ def _get_artifacts(self):
         artifacts = {}
         save_keys = {
             "model": "model_name",
-            "data": "data_name",
-            "assessment_data": "assessment_data_name",
-            "training_data": "training_data_name",
+            "data": "dataset_name",
+            "assessment_data": "assessment_dataset_name",
+            "training_data": "training_dataset_name",
         }
         for k in self.artifact_keys:
             save_key = save_keys.get(k, k)

credoai/evaluators/utils/utils.py

@@ -3,9 +3,15 @@
 
 def name2evaluator(evaluator_name):
     """Converts evaluator name to evaluator class"""
-    for eval in Evaluator.__subclasses__():
+    for eval in all_subclasses(Evaluator):
         if evaluator_name == eval.__name__:
             return eval
     raise Exception(
         f"<{evaluator_name}> not found in list of Evaluators. Please confirm specified evaluator name is identical to Evaluator class definition."
     )
+
+
+def all_subclasses(cls):
+    return set(cls.__subclasses__()).union(
+        [s for c in cls.__subclasses__() for s in all_subclasses(c)]
+    )

credoai/evidence/evidence_requirement.py

@@ -10,6 +10,7 @@ def __init__(
         self._evidence_type: str = data.get("evidence_type")
         self._label: dict = data.get("label")
         self._sensitive_features: List[str] = data.get("sensitive_features", [])
+        self._tags: dict = data.get("tags")
 
     def __str__(self) -> str:
         return f"{self.evidence_type}-EvidenceRequirement.label-{self.label}"
@@ -21,3 +22,7 @@ def evidence_type(self):
     @property
     def label(self):
         return self._label
+
+    @property
+    def tags(self):
+        return self._tags

credoai/governance/credo_api.py

@@ -105,20 +105,16 @@ def get_assessment_plan(self, url: str):
 
         return self._client.get(url)
 
-    def create_assessment(
-        self, use_case_id: str, policy_pack_id: str, evidences: List[dict]
-    ):
+    def create_assessment(self, use_case_id: str, data: dict):
         """
         Upload evidences to API server.
 
         Parameters
         ----------
         use_case_id : str
             use case id
-        policy_pack_id : str
-            policy pack id, ie: FAIR+1
-        evidences: list[dict]
-            list of evidences
+        data : dict
+            assessment data generated by Governance
 
         Raises
         ------
@@ -127,10 +123,4 @@ def create_assessment(
         """
 
         path = f"use_cases/{use_case_id}/assessments"
-
-        data = {
-            "policy_pack_id": policy_pack_id,
-            "evidences": evidences,
-            "$type": "assessments",
-        }
         return self._client.post(path, data)

credoai/governance/governance.py

@@ -71,7 +71,9 @@ def __init__(
         self._policy_pack_id: Optional[str] = None
         self._evidence_requirements: List[EvidenceRequirement] = []
         self._evidences: List[Evidence] = []
+        self._model = None
         self._plan: Optional[dict] = None
+        self._unique_tags: List[dict] = []
 
         if credo_api_client:
             client = credo_api_client
@@ -157,10 +159,21 @@ def register(
                 )
             )
 
+            # Extract unique tags
+            for x in self._evidence_requirements:
+                if x.tags not in self._unique_tags:
+                    self._unique_tags.append(x.tags)
+            self._unique_tags = [x for x in self._unique_tags if x]
+
             global_logger.info(
                 f"Successfully registered with {len(self._evidence_requirements)} evidence requirements"
             )
 
+            if self._unique_tags:
+                global_logger.info(
+                    f"The following tags have being found in the evidence requirements: {self._unique_tags}"
+                )
+
             self.clear_evidence()
 
     def __parse_json_api(self, json_str):
@@ -170,43 +183,14 @@ def __parse_json_api(self, json_str):
     def registered(self):
         return bool(self._plan)
 
-    def get_evidence_requirements(self):
-        """
-        Returns evidence requirements
-
-
-        Returns
-        -------
-        List[EvidenceRequirement]
-        """
-        return self._evidence_requirements
-
-    def clear_evidence(self):
-        self.set_evidence([])
-
-    def set_evidence(self, evidences: List[Evidence]):
-        """
-        Update evidences
-        """
-        self._evidences = evidences
-
     def add_evidence(self, evidences: Union[Evidence, List[Evidence]]):
         """
         Add evidences
         """
         self._evidences += wrap_list(evidences)
 
-    def match_requirements(self):
-        missing = []
-        required_labels = [e.label for e in self.get_evidence_requirements()]
-        for label in required_labels:
-            matching_evidence = self._check_inclusion(label, self._evidences)
-            if not matching_evidence:
-                missing.append(label)
-                global_logger.info(f"Missing required evidence with label ({label}).")
-            else:
-                matching_evidence[0].label = label
-        return not bool(missing)
+    def clear_evidence(self):
+        self.set_evidence([])
 
     def export(self, filename=None):
         """
@@ -221,14 +205,12 @@ def export(self, filename=None):
         """
         if not self._validate_export():
             return False
-        to_return = self.match_requirements()
-
-        evidences = self._prepare_evidences()
+        to_return = self._match_requirements()
 
         if filename is None:
-            self._api_export(evidences)
+            self._api_export()
         else:
-            self._file_export(evidences, filename)
+            self._file_export(filename)
 
         if to_return:
             export_status = "All requirements were matched."
@@ -238,11 +220,82 @@ def export(self, filename=None):
         global_logger.info(export_status)
         return to_return
 
-    def _api_export(self, evidences):
+    def get_evidence_requirements(self, tags: dict = None):
+        """
+        Returns evidence requirements. Each evidence requirement can have optional tags
+        (a dictionary). Only returns requirements that have tags that match the model
+        (if provided), which are tags with the same tags as the model, or no tags.
+
+        Parameters
+        ----------
+        tags : dict, optional
+            Tags to subset evidence requirements. If a model has been set, will default
+            to the model's tags. Evidence requirements will be returned that have no
+            tags or have the same tag as provided.
+
+        Returns
+        -------
+        List[EvidenceRequirement]
+        """
+        if tags is None:
+            tags = self.get_model_tags()
+
+        reqs = [
+            e for e in self._evidence_requirements if (not e.tags or e.tags == tags)
+        ]
+        return reqs
+
+    def get_evidence_tags(self):
+        """Return the unique tags used for all evidence requirements"""
+        return self._unique_tags
+
+    def get_model_tags(self):
+        """Get the tags for the associated model"""
+        if self._model:
+            return self._model["tags"]
+        else:
+            return None
+
+    def set_artifacts(self, model, training_dataset=None, assessment_dataset=None):
+        """Sets up internal knowledge of model and datasets to send to Credo AI Platform"""
         global_logger.info(
-            f"Uploading {len(evidences)} evidences.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id}"
+            f"Adding model ({model.name}) to governance. Model has tags: {model.tags}"
         )
-        self._api.create_assessment(self._use_case_id, self._policy_pack_id, evidences)
+        prepared_model = {"name": model.name, "tags": model.tags}
+        if training_dataset:
+            prepared_model["training_dataset_name"] = training_dataset.name
+        if assessment_dataset:
+            prepared_model["assessment_dataset_name"] = assessment_dataset.name
+        self._model = prepared_model
+
+    def set_evidence(self, evidences: List[Evidence]):
+        """
+        Update evidences
+        """
+        self._evidences = evidences
+
+    def tag_model(self, model):
+        """Interactive utility to tag a model tags from assessment plan"""
+        tags = self.get_evidence_tags()
+        print(f"Select tag from assessment plan to associated with model:")
+        print("0: No tags")
+        for number, tag in enumerate(tags):
+            print(f"{number+1}: {tag}")
+        selection = int(input("Number of tag to associate: "))
+        if selection == 0:
+            selected_tag = None
+        else:
+            selected_tag = tags[selection - 1]
+        print(f"Selected tag = {selected_tag}. Applying to model...")
+        model.tags = selected_tag
+        if self._model:
+            self._model["tags"] = selected_tag
+
+    def _api_export(self):
+        global_logger.info(
+            f"Uploading {len(self._evidences)} evidences.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id}"
+        )
+        self._api.create_assessment(self._use_case_id, self._prepare_export_data())
 
     def _check_inclusion(self, label, evidence):
         matching_evidence = []
@@ -258,19 +311,37 @@ def _check_inclusion(self, label, evidence):
             return False
         return matching_evidence
 
-    def _file_export(self, evidences, filename):
+    def _file_export(self, filename):
         global_logger.info(
-            f"Saving {len(evidences)} evidences to {filename}.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id} "
+            f"Saving {len(self._evidences)} evidences to {filename}.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id} "
         )
+        data = self._prepare_export_data()
+        meta = {"client": "Credo AI Lens", "version": __version__}
+        data = json_dumps(serialize(data=data, meta=meta))
+        with open(filename, "w") as f:
+            f.write(data)
+
+    def _match_requirements(self):
+        missing = []
+        required_labels = [e.label for e in self.get_evidence_requirements()]
+        for label in required_labels:
+            matching_evidence = self._check_inclusion(label, self._evidences)
+            if not matching_evidence:
+                missing.append(label)
+                global_logger.info(f"Missing required evidence with label ({label}).")
+            else:
+                matching_evidence[0].label = label
+        return not bool(missing)
+
+    def _prepare_export_data(self):
+        evidences = self._prepare_evidences()
         data = {
             "policy_pack_id": self._policy_pack_id,
+            "models": [self._model] if self._model else None,
             "evidences": evidences,
             "$type": "assessments",
         }
-        meta = {"client": "Credo AI Lens", "version": __version__}
-        data = json_dumps(serialize(data=data, meta=meta))
-        with open(filename, "w") as f:
-            f.write(data)
+        return data
 
     def _prepare_evidences(self):
         evidences = list(map(lambda e: e.struct(), self._evidences))