diff --git a/.github/workflows/publish_to_pypi.yaml b/.github/workflows/publish_to_pypi.yaml index bd46a7a..5b8618e 100644 --- a/.github/workflows/publish_to_pypi.yaml +++ b/.github/workflows/publish_to_pypi.yaml @@ -40,4 +40,4 @@ jobs: python3 -m twine upload dist/* env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + TWINE_PASSWORD: ${{ my_secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9bda1e9..2fcdbdb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ atlas_client/test/ atlas_client/__pycache__/ .idea/ log +my_secrets diff --git a/atlas_client/__init__.py b/atlas_client/__init__.py index 28f11d5..fb61699 100644 --- a/atlas_client/__init__.py +++ b/atlas_client/__init__.py @@ -1,5 +1,6 @@ __version__ = '0.2.2' + # Set default logging handler to avoid "No handler found" warnings. import logging logging.getLogger('atlaspyapi').addHandler(logging.NullHandler()) diff --git a/atlas_client/base.py b/atlas_client/base.py index 8b95467..bfd9cdc 100644 --- a/atlas_client/base.py +++ b/atlas_client/base.py @@ -16,13 +16,13 @@ import ast import json -import logging import time from datetime import datetime, timedelta import six from atlas_client import events, exceptions, utils +from atlas_client.log_manager import LogManager -LOG = logging.getLogger('pyatlasclient') +LOG = LogManager(__name__).get_logger() OLDEST_SUPPORTED_VERSION = (1, 7, 0) diff --git a/atlas_client/client.py b/atlas_client/client.py index a63bd56..0443c67 100644 --- a/atlas_client/client.py +++ b/atlas_client/client.py @@ -14,14 +14,15 @@ import functools import io import json -import logging import tarfile import requests + from atlas_client import models, utils, base, exceptions from atlas_client.exceptions import handle_response, BadHttpAuthArg +from atlas_client.log_manager import LogManager -LOG = logging.getLogger('pyatlasclient') +LOG = LogManager(__name__).get_logger() # this defines where the Atlas client delegates to for actual logic ENTRY_POINTS = {'entity_guid': models.EntityGuid, @@ -65,7 +66,7 @@ class Atlas(object): def __init__(self, host: str, port: int = None, username: str = None, password: str = None, oidc_token: str = None, identifier: str = None, protocol: str = None, validate_ssl: bool = True, timeout=10, max_retries=5, auth=None): - + self.oidc_token = oidc_token self.base_url = utils.generate_base_url(host, port=port, protocol=protocol) if identifier is None: identifier = 'python-atlasclient' @@ -101,6 +102,37 @@ def __getattr__(self, attr): raise AttributeError(attr) + def purge_entity_by_guid(self, guid: str): + headers = {'Authorization': self.client.auth_header, + "Content-Type": "application/json", + "Accept": "application/json"} + purge_url = f"{self.base_url}/api/atlas/v2/admin/purge/" + response = requests.put(purge_url, data=guid, headers=headers) + return response + + def get_guid_by_qualified_name(self, entity_type_name: str, entity_qualified_name, **kwargs): + if 'limit' in kwargs and isinstance(kwargs['limit'], int): + input_limit = kwargs['limit'] + else: + input_limit = 10 + if 'offset' in kwargs and isinstance(kwargs['offset'], int): + input_offset = kwargs['offset'] + else: + input_offset = 0 + headers = {'Authorization': self.client.auth_header, + "Content-Type": "application/json", + "Accept": "application/json"} + params = {"attrName": "qualifiedName", + "attrValuePrefix": f"{entity_qualified_name}", + "limit": f"{input_limit}", + "offset": f"{input_offset}", + "typeName": f"{entity_type_name}"} + search_url = f"{self.base_url}/api/atlas/v2/search/attribute" + response = requests.get(f"{search_url}", params=params, headers=headers) + # convert response json text to python dict + response_dict = json.loads(response.text) + return response_dict["entities"][0]["guid"] + class HttpClient(object): """Our HTTP based REST client. @@ -117,15 +149,15 @@ class HttpClient(object): def __init__(self, host, identifier, username=None, password=None, oidc_token=None, validate_ssl=True, timeout=10, max_retries=5, auth=None): if oidc_token: - auth_header = f'Bearer {oidc_token}' + self.auth_header = f'Bearer {oidc_token}' elif username and password: basic_token = utils.generate_http_basic_token(username=username, password=password) - auth_header = f'Basic {basic_token}' + self.auth_header = f'Basic {basic_token}' else: raise BadHttpAuthArg self.request_params = { 'headers': {'X-Requested-By': identifier, - 'Authorization': auth_header}, + 'Authorization': self.auth_header}, 'verify': validate_ssl, 'timeout': timeout, } diff --git a/atlas_client/entity_management/EntityManager.py b/atlas_client/entity_management/EntityManager.py index 512d127..2e4215a 100644 --- a/atlas_client/entity_management/EntityManager.py +++ b/atlas_client/entity_management/EntityManager.py @@ -12,6 +12,8 @@ from abc import ABC, abstractmethod from typing import KeysView +from requests.models import Response + from atlas_client.client import Atlas @@ -41,11 +43,14 @@ def show_entity_attributes(entity: dict) -> None: def get_s3_attributes_key_list(entity: dict) -> KeysView: return EntityManager.get_entity_attributes(entity).keys() - def update_entity(self, guid: str, attribute_name: str, attribute_value: str) -> None: - s3_bucket = self.client.entity_guid(guid) - s3_bucket.entity['attributes'][attribute_name] = attribute_value - s3_bucket.update(attribute=attribute_name) + def update_entity(self, guid: str, attribute_name: str, attribute_value: str) -> Response: + current_entity_obj = self.client.entity_guid(guid) + current_entity_obj.entity['attributes'][attribute_name] = attribute_value + return current_entity_obj.update(attribute=attribute_name) + + def delete_entity(self, guid: str) -> Response: + current_entity_obj = self.client.entity_guid(guid) + return current_entity_obj.delete() - def delete_entity(self, guid: str) -> None: - s3_bucket = self.client.entity_guid(guid) - s3_bucket.delete() + def purge_entity(self, guid: str) -> Response: + return self.client.purge_entity_by_guid(guid) diff --git a/atlas_client/entity_management/hive/HiveColumnManager.py b/atlas_client/entity_management/hive/HiveColumnManager.py index 3147480..fb252da 100644 --- a/atlas_client/entity_management/hive/HiveColumnManager.py +++ b/atlas_client/entity_management/hive/HiveColumnManager.py @@ -15,10 +15,10 @@ from atlas_client.client import Atlas from atlas_client.entity_management.EntityManager import EntityManager from atlas_client.entity_source_generation.HiveColumnEntityGenerator import HiveColumnEntityGenerator -from atlas_client.log_manager import get_logger +from atlas_client.log_manager import LogManager -my_logger = get_logger(__name__) -my_logger.debug("a debug message") +my_logger = LogManager(__name__).get_logger() +my_logger.debug("Init hive column manager") class HiveColumnManager(EntityManager): diff --git a/atlas_client/entity_management/hive/HiveDBManager.py b/atlas_client/entity_management/hive/HiveDBManager.py index 9595367..7655d43 100644 --- a/atlas_client/entity_management/hive/HiveDBManager.py +++ b/atlas_client/entity_management/hive/HiveDBManager.py @@ -15,10 +15,10 @@ from atlas_client.client import Atlas from atlas_client.entity_management.EntityManager import EntityManager from atlas_client.entity_source_generation.HiveDBEntityGenerator import HiveDBEntityGenerator -from atlas_client.log_manager import get_logger +from atlas_client.log_manager import LogManager -my_logger = get_logger(__name__) -my_logger.debug("a debug message") +my_logger = LogManager(__name__).get_logger() +my_logger.debug("Init hive db manager") class HiveDBManager(EntityManager): diff --git a/atlas_client/entity_management/hive/HiveTableManager.py b/atlas_client/entity_management/hive/HiveTableManager.py index d3e9bc2..b292031 100644 --- a/atlas_client/entity_management/hive/HiveTableManager.py +++ b/atlas_client/entity_management/hive/HiveTableManager.py @@ -15,10 +15,10 @@ from atlas_client.client import Atlas from atlas_client.entity_management.EntityManager import EntityManager from atlas_client.entity_source_generation.HiveTableEntityGenerator import HiveTableEntityGenerator -from atlas_client.log_manager import get_logger +from atlas_client.log_manager import LogManager -my_logger = get_logger(__name__) -my_logger.debug("a debug message") +my_logger = LogManager(__name__).get_logger() +my_logger.debug("Init hive table manager") class HiveTableManager(EntityManager): diff --git a/atlas_client/entity_search/EntityFinder.py b/atlas_client/entity_search/EntityFinder.py index 9b12213..5c03993 100644 --- a/atlas_client/entity_search/EntityFinder.py +++ b/atlas_client/entity_search/EntityFinder.py @@ -1,20 +1,29 @@ from atlas_client.client import Atlas +from atlas_client.log_manager import LogManager + +LOG = LogManager(__name__).get_logger() + class EntityFinder: def __init__(self, atlas_client: Atlas): self.client = atlas_client def search_by_attribute(self, type_name: str, attribute_name: str, attribute_value: str) -> dict: - params = {'typeName': type_name, 'attrName': attribute_name, 'attrValue': attribute_value, 'offset': '1', + params = {'typeName': type_name, 'attrName': attribute_name, 'attrValue': attribute_value, 'offset': '0', 'limit': '10'} + LOG.debug(f"Search query params {params}") return self.client.search_attribute(**params) def search_full_text(self, type_name: str, attribute_value: str) -> dict: - params = {'typeName': type_name, 'attrValue': attribute_value, 'offset': '1', + params = {'typeName': type_name, 'attrValue': attribute_value, 'offset': '0', 'limit': '10'} + LOG.debug(f"Search query params {params}") return self.client.search_basic(**params) + def get_guid_by_qualified_name(self, type_name: str, qualified_name: str): + return self.client.get_guid_by_qualified_name(type_name, qualified_name) + @staticmethod def show_search_results(search_results: dict) -> None: for s in search_results: diff --git a/atlas_client/events.py b/atlas_client/events.py index 230ea93..080b5b3 100644 --- a/atlas_client/events.py +++ b/atlas_client/events.py @@ -11,10 +11,11 @@ # under the License. import inspect -import logging from collections import namedtuple -LOG = logging.getLogger('pyatlasclient') +from atlas_client.log_manager import LogManager + +LOG = LogManager(__name__).get_logger() EVENT_HANDLERS = {} state_list = ['ANY', 'STARTED', 'FAILED', 'FINISHED', 'PROGRESS'] diff --git a/atlas_client/exceptions.py b/atlas_client/exceptions.py index 2326abe..1285b89 100644 --- a/atlas_client/exceptions.py +++ b/atlas_client/exceptions.py @@ -17,9 +17,10 @@ https://github.com/openstack/python-novaclient/blob/master/novaclient/exceptions.py """ -import logging -LOG = logging.getLogger('pyatlasclient') +from atlas_client.log_manager import LogManager + +LOG = LogManager(__name__).get_logger() class ClientError(Exception): diff --git a/atlas_client/log_manager.py b/atlas_client/log_manager.py index 554af47..310c5e9 100644 --- a/atlas_client/log_manager.py +++ b/atlas_client/log_manager.py @@ -1,28 +1,37 @@ import logging import sys from logging.handlers import TimedRotatingFileHandler +from pathlib import Path -FORMATTER = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s") -LOG_FILE = "../log/atlas_client.log" +class LogManager: -def get_console_handler(): - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setFormatter(FORMATTER) - return console_handler + def __init__(self, logger_name: str, enable_file_handler=False, log_propagate=False, + log_format="%(asctime)s — %(name)s — %(levelname)s — %(message)s", + log_file_path="~/atlas_client_log"): + self.logger_name = logger_name + self.enable_file_handler = enable_file_handler + self.log_propagate = log_propagate + self.log_file_path = log_file_path + self.formatter = logging.Formatter(log_format) + def get_console_handler(self): + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(self.formatter) + return console_handler -def get_file_handler(): - file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight') - file_handler.setFormatter(FORMATTER) - return file_handler + def get_file_handler(self): + Path(self.log_file_path).mkdir(parents=True, exist_ok=True) + file_handler = TimedRotatingFileHandler(f"{self.log_file_path}/app.log", when='midnight') + file_handler.setFormatter(self.formatter) + return file_handler - -def get_logger(logger_name): - logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) # better to have too much log than not enough - logger.addHandler(get_console_handler()) - logger.addHandler(get_file_handler()) - # with this pattern, it's rarely necessary to propagate the error up to parent - logger.propagate = False - return logger + def get_logger(self): + logger = logging.getLogger(self.logger_name) + logger.setLevel(logging.DEBUG) # better to have too much log than not enough + logger.addHandler(self.get_console_handler()) + if self.enable_file_handler: + logger.addHandler(self.get_file_handler()) + # with this pattern, it's rarely necessary to propagate the error up to parent + logger.propagate = self.log_propagate + return logger \ No newline at end of file diff --git a/atlas_client/models.py b/atlas_client/models.py index 6dc79ef..6bcb696 100644 --- a/atlas_client/models.py +++ b/atlas_client/models.py @@ -14,12 +14,13 @@ Defines all the model classes for the various parts of the API. """ import json -import logging import itertools import six from atlas_client import base, exceptions, events -LOG = logging.getLogger('atlaspyapi') +from atlas_client.log_manager import LogManager + +LOG = LogManager(__name__).get_logger() class EntityCollection(base.DependentModelCollection): @@ -47,9 +48,10 @@ def __call__(self, *args): class Entity(base.DependentModel): collection_class = EntityCollection - fields = ('guid', 'status', 'displayText', 'classificationNames', 'typeName', 'attributes', 'createdBy', - 'updatedBy', 'createTime', 'updateTime', 'version', 'relationshipAttributes',) - + fields = ('guid', 'status', 'displayText', 'classificationNames', 'classifications', + 'typeName', 'attributes', 'createdBy', 'updatedBy', 'createTime', + 'updateTime', 'version', 'relationshipAttributes', 'businessAttributes', 'customAttributes', + 'homeId', 'isIncomplete', 'labels', 'meanings', 'provenanceType', 'proxy', 'status',) class EntityPostCollection(base.QueryableModelCollection): def __call__(self, *args, **kwargs): diff --git a/pyproject.toml b/pyproject.toml index 332cf80..43eda4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,17 @@ [tool.poetry] -name = "atlas_client" -version = "0.1.0" +name = "atlaspyapi" +version = "0.2.0" description = "" authors = ["pengfei "] +keywords = ["keyword", "another_keyword"] +readme = "README.md" +license = "Apache License 2.0" +homepage = "https://github.com/pengfei99/AtlasPyApi" +repository = "https://github.com/pengfei99/AtlasPyApi" +include = [ + "LICENSE", +] + [tool.poetry.dependencies] python = "^3.8" @@ -10,6 +19,9 @@ Jinja2 = "^3.0.3" six = "^1.16.0" click = "^8.0.4" requests = "^2.27.1" +twine = "^3.8.0" +apache-atlas = "^0.0.11" + [tool.poetry.dev-dependencies] diff --git a/setup.py b/setup.py index fa3a995..9f1720a 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ license='Apache License 2.0', long_description=long_description, long_description_content_type="text/markdown", - url="https://git.lab.sspcloud.fr/pengfei/atlaspyapi_forpypi", + url="https://github.com/pengfei99/AtlasPyApi", # we need to indicate excitement which package will be published, otherwise import will raise module name not found packages=setuptools.find_packages(include=['atlas_client', 'atlas_client.*']), classifiers=[ diff --git a/tests/EntityFinderTest.py b/tests/EntityFinderTest.py new file mode 100644 index 0000000..371cad2 --- /dev/null +++ b/tests/EntityFinderTest.py @@ -0,0 +1,29 @@ +from atlas_client.client import Atlas +from atlas_client.entity_search.EntityFinder import EntityFinder +from my_secrets import secret + + +def main(): + local = False + # config for atlas client + + if local: + atlas_local_hostname = "http://localhost" + login = "admin" + pwd = "admin" + atlas_client = Atlas(atlas_local_hostname, port=21000, username=login, password=pwd) + else: + # create an instance of the atlas Client with oidc token + atlas_prod_hostname = "https://atlas.lab.sspcloud.fr" + atlas_prod_port = 443 + oidc_token = secret.oidc_token + atlas_client = Atlas(atlas_prod_hostname, atlas_prod_port, oidc_token=oidc_token) + entity_finder = EntityFinder(atlas_client) + t_guid = entity_finder.get_guid_by_qualified_name("hive_table", "default.students") + print(t_guid) + d_guid = entity_finder.get_guid_by_qualified_name("hive_db", "default") + print(d_guid) + + +if __name__ == "__main__": + main() diff --git a/tests/IntegrationTest.py b/tests/HiveEntityCreationIntegrationTest.py similarity index 51% rename from tests/IntegrationTest.py rename to tests/HiveEntityCreationIntegrationTest.py index 79f08f9..39e3cc8 100644 --- a/tests/IntegrationTest.py +++ b/tests/HiveEntityCreationIntegrationTest.py @@ -2,23 +2,22 @@ from atlas_client.entity_management.hive.HiveColumnManager import HiveColumnManager from atlas_client.entity_management.hive.HiveDBManager import HiveDBManager from atlas_client.entity_management.hive.HiveTableManager import HiveTableManager -from atlas_client.entity_search.EntityFinder import EntityFinder +from my_secrets import secret def main(): local = False # config for atlas client - atlas_prod_hostname = "https://atlas.lab.sspcloud.fr" - atlas_prod_port = 443 - oidc_token = "" - atlas_local_hostname = "http://localhost" - login = "admin" - pwd = "admin" - if local: + atlas_local_hostname = "http://localhost" + login = "admin" + pwd = "admin" atlas_client = Atlas(atlas_local_hostname, port=21000, username=login, password=pwd) else: # create an instance of the atlas Client with oidc token + atlas_prod_hostname = "https://atlas.lab.sspcloud.fr" + atlas_prod_port = 443 + oidc_token = secret.oidc_token atlas_client = Atlas(atlas_prod_hostname, atlas_prod_port, oidc_token=oidc_token) # finder = EntityFinder(atlas_client) # res = finder.search_full_text("hive_table", "pengfei") @@ -28,10 +27,14 @@ def main(): hive_column = HiveColumnManager(atlas_client) # insert hive tables - hive_db.create_entity("pengfei-stock", "pengfei.org", "database for my stock market",owner="pliu",location="pengfei.org") - hive_table.create_entity("favorite", "pengfei.org@pengfei-stock", "favorite stock") - hive_column.create_entity("stock_id", "int", "pengfei.org@pengfei-stock.favorite", "id of the stock") - hive_column.create_entity("stock_name", "string", "pengfei.org@pengfei-stock.favorite", "name of the stock") + # hive_db.create_entity("pengfei-stock", "pengfei.org", "database for my stock market",owner="pliu",location="pengfei.org") + # hive_table.create_entity("favorite", "pengfei.org@pengfei-stock", "favorite stock") + # hive_column.create_entity("stock_id", "int", "pengfei.org@pengfei-stock.favorite", "id of the stock") + # hive_column.create_entity("stock_name", "string", "pengfei.org@pengfei-stock.favorite", "name of the stock") + hive_table.delete_entity("e57ce313-ef6c-4ada-b40d-697b693893f6") + db_guid = "a5201924-87f7-4c3c-aab4-b8086917b108" + hive_db.delete_entity(db_guid) + hive_db.purge_entity(db_guid) if __name__ == "__main__": diff --git a/tests/SearchByAttributeIntegrationTest.py b/tests/SearchByAttributeIntegrationTest.py new file mode 100644 index 0000000..d5ad408 --- /dev/null +++ b/tests/SearchByAttributeIntegrationTest.py @@ -0,0 +1,31 @@ +from atlas_client.client import Atlas +from atlas_client.entity_search.EntityFinder import EntityFinder +from my_secrets import secret + + +def main(): + local = False + # config for atlas client + + + + if local: + atlas_local_hostname = "http://localhost" + login = "admin" + pwd = "admin" + atlas_client = Atlas(atlas_local_hostname, port=21000, username=login, password=pwd) + else: + # create an instance of the atlas Client with oidc token + atlas_prod_hostname = "https://atlas.lab.sspcloud.fr" + atlas_prod_port = 443 + oidc_token = secret.oidc_token + atlas_client = Atlas(atlas_prod_hostname, atlas_prod_port, oidc_token=oidc_token) + entity_finder = EntityFinder(atlas_client) + res = entity_finder.search_full_text("hive_table", "user-pengfei@movies.Character") + EntityFinder.show_search_results(res) + guids = EntityFinder.get_result_entity_guid_list(res) + print(guids) + + +if __name__ == "__main__": + main()