merge upstream

.github/workflows/publish_to_pypi.yaml

@@ -40,4 +40,4 @@ jobs:
           python3 -m twine upload dist/*
         env:
           TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+          TWINE_PASSWORD: ${{ my_secrets.PYPI_API_TOKEN }}

.gitignore

@@ -9,4 +9,5 @@ atlas_client/test/
 atlas_client/__pycache__/
 .idea/
 log
+my_secrets
 

atlas_client/__init__.py

@@ -1,5 +1,6 @@
 __version__ = '0.2.2'
 
+
 # Set default logging handler to avoid "No handler found" warnings.
 import logging
 logging.getLogger('atlaspyapi').addHandler(logging.NullHandler())

atlas_client/base.py

@@ -16,13 +16,13 @@
 
 import ast
 import json
-import logging
 import time
 from datetime import datetime, timedelta
 import six
 from atlas_client import events, exceptions, utils
+from atlas_client.log_manager import LogManager
 
-LOG = logging.getLogger('pyatlasclient')
+LOG = LogManager(__name__).get_logger()
 
 OLDEST_SUPPORTED_VERSION = (1, 7, 0)
 

atlas_client/client.py

@@ -14,14 +14,15 @@
 import functools
 import io
 import json
-import logging
 import tarfile
 
 import requests
+
 from atlas_client import models, utils, base, exceptions
 from atlas_client.exceptions import handle_response, BadHttpAuthArg
+from atlas_client.log_manager import LogManager
 
-LOG = logging.getLogger('pyatlasclient')
+LOG = LogManager(__name__).get_logger()
 
 # this defines where the Atlas client delegates to for actual logic
 ENTRY_POINTS = {'entity_guid': models.EntityGuid,
@@ -65,7 +66,7 @@ class Atlas(object):
     def __init__(self, host: str, port: int = None, username: str = None, password: str = None, oidc_token: str = None,
                  identifier: str = None, protocol: str = None, validate_ssl: bool = True,
                  timeout=10, max_retries=5, auth=None):
-
+        self.oidc_token = oidc_token
         self.base_url = utils.generate_base_url(host, port=port, protocol=protocol)
         if identifier is None:
             identifier = 'python-atlasclient'
@@ -101,6 +102,37 @@ def __getattr__(self, attr):
 
         raise AttributeError(attr)
 
+    def purge_entity_by_guid(self, guid: str):
+        headers = {'Authorization': self.client.auth_header,
+                   "Content-Type": "application/json",
+                   "Accept": "application/json"}
+        purge_url = f"{self.base_url}/api/atlas/v2/admin/purge/"
+        response = requests.put(purge_url, data=guid, headers=headers)
+        return response
+
+    def get_guid_by_qualified_name(self, entity_type_name: str, entity_qualified_name, **kwargs):
+        if 'limit' in kwargs and isinstance(kwargs['limit'], int):
+            input_limit = kwargs['limit']
+        else:
+            input_limit = 10
+        if 'offset' in kwargs and isinstance(kwargs['offset'], int):
+            input_offset = kwargs['offset']
+        else:
+            input_offset = 0
+        headers = {'Authorization': self.client.auth_header,
+                   "Content-Type": "application/json",
+                   "Accept": "application/json"}
+        params = {"attrName": "qualifiedName",
+                  "attrValuePrefix": f"{entity_qualified_name}",
+                  "limit": f"{input_limit}",
+                  "offset": f"{input_offset}",
+                  "typeName": f"{entity_type_name}"}
+        search_url = f"{self.base_url}/api/atlas/v2/search/attribute"
+        response = requests.get(f"{search_url}", params=params, headers=headers)
+        # convert response json text to python dict
+        response_dict = json.loads(response.text)
+        return response_dict["entities"][0]["guid"]
+
 
 class HttpClient(object):
     """Our HTTP based REST client.
@@ -117,15 +149,15 @@ class HttpClient(object):
     def __init__(self, host, identifier, username=None, password=None, oidc_token=None, validate_ssl=True,
                  timeout=10, max_retries=5, auth=None):
         if oidc_token:
-            auth_header = f'Bearer {oidc_token}'
+            self.auth_header = f'Bearer {oidc_token}'
         elif username and password:
             basic_token = utils.generate_http_basic_token(username=username, password=password)
-            auth_header = f'Basic {basic_token}'
+            self.auth_header = f'Basic {basic_token}'
         else:
             raise BadHttpAuthArg
         self.request_params = {
             'headers': {'X-Requested-By': identifier,
-                        'Authorization': auth_header},
+                        'Authorization': self.auth_header},
             'verify': validate_ssl,
             'timeout': timeout,
         }

atlas_client/entity_management/EntityManager.py

@@ -12,6 +12,8 @@
 
 from abc import ABC, abstractmethod
 from typing import KeysView
+from requests.models import Response
+
 from atlas_client.client import Atlas
 
 
@@ -41,11 +43,14 @@ def show_entity_attributes(entity: dict) -> None:
     def get_s3_attributes_key_list(entity: dict) -> KeysView:
         return EntityManager.get_entity_attributes(entity).keys()
 
-    def update_entity(self, guid: str, attribute_name: str, attribute_value: str) -> None:
-        s3_bucket = self.client.entity_guid(guid)
-        s3_bucket.entity['attributes'][attribute_name] = attribute_value
-        s3_bucket.update(attribute=attribute_name)
+    def update_entity(self, guid: str, attribute_name: str, attribute_value: str) -> Response:
+        current_entity_obj = self.client.entity_guid(guid)
+        current_entity_obj.entity['attributes'][attribute_name] = attribute_value
+        return current_entity_obj.update(attribute=attribute_name)
+
+    def delete_entity(self, guid: str) -> Response:
+        current_entity_obj = self.client.entity_guid(guid)
+        return current_entity_obj.delete()
 
-    def delete_entity(self, guid: str) -> None:
-        s3_bucket = self.client.entity_guid(guid)
-        s3_bucket.delete()
+    def purge_entity(self, guid: str) -> Response:
+        return self.client.purge_entity_by_guid(guid)

atlas_client/entity_management/hive/HiveColumnManager.py

@@ -15,10 +15,10 @@
 from atlas_client.client import Atlas
 from atlas_client.entity_management.EntityManager import EntityManager
 from atlas_client.entity_source_generation.HiveColumnEntityGenerator import HiveColumnEntityGenerator
-from atlas_client.log_manager import get_logger
+from atlas_client.log_manager import LogManager
 
-my_logger = get_logger(__name__)
-my_logger.debug("a debug message")
+my_logger = LogManager(__name__).get_logger()
+my_logger.debug("Init hive column manager")
 
 
 class HiveColumnManager(EntityManager):

atlas_client/entity_management/hive/HiveDBManager.py

@@ -15,10 +15,10 @@
 from atlas_client.client import Atlas
 from atlas_client.entity_management.EntityManager import EntityManager
 from atlas_client.entity_source_generation.HiveDBEntityGenerator import HiveDBEntityGenerator
-from atlas_client.log_manager import get_logger
+from atlas_client.log_manager import LogManager
 
-my_logger = get_logger(__name__)
-my_logger.debug("a debug message")
+my_logger = LogManager(__name__).get_logger()
+my_logger.debug("Init hive db manager")
 
 
 class HiveDBManager(EntityManager):

atlas_client/entity_management/hive/HiveTableManager.py

@@ -15,10 +15,10 @@
 from atlas_client.client import Atlas
 from atlas_client.entity_management.EntityManager import EntityManager
 from atlas_client.entity_source_generation.HiveTableEntityGenerator import HiveTableEntityGenerator
-from atlas_client.log_manager import get_logger
+from atlas_client.log_manager import LogManager
 
-my_logger = get_logger(__name__)
-my_logger.debug("a debug message")
+my_logger = LogManager(__name__).get_logger()
+my_logger.debug("Init hive table manager")
 
 
 class HiveTableManager(EntityManager):

atlas_client/entity_search/EntityFinder.py

@@ -1,20 +1,29 @@
 from atlas_client.client import Atlas
 
+from atlas_client.log_manager import LogManager
+
+LOG = LogManager(__name__).get_logger()
+
 
 class EntityFinder:
     def __init__(self, atlas_client: Atlas):
         self.client = atlas_client
 
     def search_by_attribute(self, type_name: str, attribute_name: str, attribute_value: str) -> dict:
-        params = {'typeName': type_name, 'attrName': attribute_name, 'attrValue': attribute_value, 'offset': '1',
+        params = {'typeName': type_name, 'attrName': attribute_name, 'attrValue': attribute_value, 'offset': '0',
                   'limit': '10'}
+        LOG.debug(f"Search query params {params}")
         return self.client.search_attribute(**params)
 
     def search_full_text(self, type_name: str, attribute_value: str) -> dict:
-        params = {'typeName': type_name, 'attrValue': attribute_value, 'offset': '1',
+        params = {'typeName': type_name, 'attrValue': attribute_value, 'offset': '0',
                   'limit': '10'}
+        LOG.debug(f"Search query params {params}")
         return self.client.search_basic(**params)
 
+    def get_guid_by_qualified_name(self, type_name: str, qualified_name: str):
+        return self.client.get_guid_by_qualified_name(type_name, qualified_name)
+
     @staticmethod
     def show_search_results(search_results: dict) -> None:
         for s in search_results:

atlas_client/events.py

@@ -11,10 +11,11 @@
 #    under the License.
 
 import inspect
-import logging
 from collections import namedtuple
 
-LOG = logging.getLogger('pyatlasclient')
+from atlas_client.log_manager import LogManager
+
+LOG = LogManager(__name__).get_logger()
 
 EVENT_HANDLERS = {}
 state_list = ['ANY', 'STARTED', 'FAILED', 'FINISHED', 'PROGRESS']

atlas_client/exceptions.py

@@ -17,9 +17,10 @@
 
 https://github.com/openstack/python-novaclient/blob/master/novaclient/exceptions.py
 """
-import logging
 
-LOG = logging.getLogger('pyatlasclient')
+from atlas_client.log_manager import LogManager
+
+LOG = LogManager(__name__).get_logger()
 
 
 class ClientError(Exception):

atlas_client/log_manager.py

@@ -1,28 +1,37 @@
 import logging
 import sys
 from logging.handlers import TimedRotatingFileHandler
+from pathlib import Path
 
-FORMATTER = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
-LOG_FILE = "../log/atlas_client.log"
 
+class LogManager:
 
-def get_console_handler():
-    console_handler = logging.StreamHandler(sys.stdout)
-    console_handler.setFormatter(FORMATTER)
-    return console_handler
+    def __init__(self, logger_name: str, enable_file_handler=False, log_propagate=False,
+                 log_format="%(asctime)s — %(name)s — %(levelname)s — %(message)s",
+                 log_file_path="~/atlas_client_log"):
+        self.logger_name = logger_name
+        self.enable_file_handler = enable_file_handler
+        self.log_propagate = log_propagate
+        self.log_file_path = log_file_path
+        self.formatter = logging.Formatter(log_format)
 
+    def get_console_handler(self):
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(self.formatter)
+        return console_handler
 
-def get_file_handler():
-    file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight')
-    file_handler.setFormatter(FORMATTER)
-    return file_handler
+    def get_file_handler(self):
+        Path(self.log_file_path).mkdir(parents=True, exist_ok=True)
+        file_handler = TimedRotatingFileHandler(f"{self.log_file_path}/app.log", when='midnight')
+        file_handler.setFormatter(self.formatter)
+        return file_handler
 
-
-def get_logger(logger_name):
-    logger = logging.getLogger(logger_name)
-    logger.setLevel(logging.DEBUG)  # better to have too much log than not enough
-    logger.addHandler(get_console_handler())
-    logger.addHandler(get_file_handler())
-    # with this pattern, it's rarely necessary to propagate the error up to parent
-    logger.propagate = False
-    return logger
+    def get_logger(self):
+        logger = logging.getLogger(self.logger_name)
+        logger.setLevel(logging.DEBUG)  # better to have too much log than not enough
+        logger.addHandler(self.get_console_handler())
+        if self.enable_file_handler:
+            logger.addHandler(self.get_file_handler())
+        # with this pattern, it's rarely necessary to propagate the error up to parent
+        logger.propagate = self.log_propagate
+        return logger

atlas_client/models.py

@@ -14,12 +14,13 @@
 Defines all the model classes for the various parts of the API.
 """
 import json
-import logging
 import itertools
 import six
 from atlas_client import base, exceptions, events
 
-LOG = logging.getLogger('atlaspyapi')
+from atlas_client.log_manager import LogManager
+
+LOG = LogManager(__name__).get_logger()
 
 
 class EntityCollection(base.DependentModelCollection):
@@ -47,9 +48,10 @@ def __call__(self, *args):
 
 class Entity(base.DependentModel):
     collection_class = EntityCollection
-    fields = ('guid', 'status', 'displayText', 'classificationNames', 'typeName', 'attributes', 'createdBy',
-              'updatedBy', 'createTime', 'updateTime', 'version', 'relationshipAttributes',)
-
+    fields = ('guid', 'status', 'displayText', 'classificationNames', 'classifications',
+              'typeName', 'attributes', 'createdBy', 'updatedBy', 'createTime',
+              'updateTime', 'version', 'relationshipAttributes', 'businessAttributes', 'customAttributes',
+              'homeId', 'isIncomplete', 'labels', 'meanings', 'provenanceType', 'proxy', 'status',)
 
 class EntityPostCollection(base.QueryableModelCollection):
     def __call__(self, *args, **kwargs):

pyproject.toml

@@ -1,15 +1,27 @@
 [tool.poetry]
-name = "atlas_client"
-version = "0.1.0"
+name = "atlaspyapi"
+version = "0.2.0"
 description = ""
 authors = ["pengfei <[email protected]>"]
+keywords = ["keyword", "another_keyword"]
+readme = "README.md"
+license = "Apache License 2.0"
+homepage = "https://github.com/pengfei99/AtlasPyApi"
+repository = "https://github.com/pengfei99/AtlasPyApi"
+include = [
+    "LICENSE",
+]
+
 
 [tool.poetry.dependencies]
 python = "^3.8"
 Jinja2 = "^3.0.3"
 six = "^1.16.0"
 click = "^8.0.4"
 requests = "^2.27.1"
+twine = "^3.8.0"
+apache-atlas = "^0.0.11"
+
 
 
 [tool.poetry.dev-dependencies]

setup.py

@@ -28,7 +28,7 @@
     license='Apache License 2.0',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url="https://git.lab.sspcloud.fr/pengfei/atlaspyapi_forpypi",
+    url="https://github.com/pengfei99/AtlasPyApi",
     # we need to indicate excitement which package will be published, otherwise import will raise module name not found
     packages=setuptools.find_packages(include=['atlas_client', 'atlas_client.*']),
     classifiers=[