feat(ingestion): [WIP] Adding SSAS as a New Source for Data Ingestion

metadata-ingestion/setup.py

@@ -263,6 +263,14 @@
 
 slack = {"slack-sdk==3.18.1"}
 
+ssas = {
+    "xmltodict==0.13.0",
+    "requests==2.28.1",
+    "beautifulsoup4==4.11.1",
+    "lxml==4.9.1",
+    "requests-kerberos==0.14.0"
+}
+
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
     "databricks-sdk>=0.9.0",
@@ -393,6 +401,7 @@
     "sqlalchemy": sql_common,
     "sql-queries": usage_common | sqlglot_lib,
     "slack": slack,
+    "ssas": ssas,
     "superset": {
         "requests",
         "sqlalchemy",
@@ -538,6 +547,7 @@
             "s3",
             "snowflake",
             "slack",
+            "ssas",
             "tableau",
             "teradata",
             "trino",
@@ -580,6 +590,7 @@
             "ldap",
             "mongodb",
             "slack",
+            #"ssas",
             "mssql",
             "mysql",
             "mariadb",
@@ -637,6 +648,8 @@
         "redash = datahub.ingestion.source.redash:RedashSource",
         "redshift = datahub.ingestion.source.redshift.redshift:RedshiftSource",
         "slack = datahub.ingestion.source.slack.slack:SlackSource",
+        "ssas_multidimension = datahub.ingestion.source.ssas.ssas_multidimension.ssas_multidimension:SsasMultidimensionSource",
+        "ssas_tabular = datahub.ingestion.source.ssas.ssas_tabular.ssas_tabular:SsasTabularSource",
         "snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source",
         "superset = datahub.ingestion.source.superset:SupersetSource",
         "tableau = datahub.ingestion.source.tableau:TableauSource",

metadata-ingestion/src/datahub/ingestion/source/ssas/api.py

@@ -0,0 +1,54 @@
+"""
+Module xmla communicate classes.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Union
+
+import xmltodict
+from requests.auth import HTTPBasicAuth
+from requests_kerberos import HTTPKerberosAuth
+
+from .config import SsasServerHTTPConfig
+from .tools import MsXmlaTemplates
+from .xmlaclient import XmlaClient
+
+
+class ISsasAPI(ABC):
+    @abstractmethod
+    def get_server(self):
+        pass
+
+    @property
+    @abstractmethod
+    def auth_credentials(self):
+        pass
+
+
+class SsasXmlaAPI:
+    """
+    Class for parse ssas xmla server response
+    """
+
+    def __init__(self, config: SsasServerHTTPConfig, auth: Union[HTTPKerberosAuth, HTTPBasicAuth]):
+        self.__config = config
+        self.__auth = auth
+        self.__client = XmlaClient(config=config, auth=self.__auth)
+
+    def get_server_info(self) -> Dict[str, Any]:
+        """
+        Extract server metadata info from response
+        """
+
+        server_data_xml = xmltodict.parse(self.get_server_metadata())
+
+        return server_data_xml["soap:Envelope"]["soap:Body"]["DiscoverResponse"][
+            "return"
+        ]["root"]["row"]["xars:METADATA"]["Server"]
+
+    def get_server_metadata(self) -> str:
+        """
+        Get ssas server metadata
+        """
+
+        return str(self.__client.discover(query=MsXmlaTemplates.QUERY_METADATA))

metadata-ingestion/src/datahub/ingestion/source/ssas/config.py

@@ -0,0 +1,68 @@
+from typing import List, Dict
+
+import datahub.emitter.mce_builder as builder
+import pydantic
+from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.source_common import EnvConfigMixin
+
+
+class SsasServerHTTPConfig(EnvConfigMixin):
+    """
+    Class represent config object.
+    Contains parameters for connect to ssas over http (HTTP Access to Analysis Services)
+    https://learn.microsoft.com/en-us/analysis-services/instances/configure-http-access-to-analysis-services-on-iis-8-0?view=asallproducts-allversions
+
+    username - Active Directory user login
+    username - Active Directory user password
+    host_port - XMLA gateway url (format - url:port)
+    server_alias - XMLA gateway server alias
+    virtual_directory_name -
+    instance -  not used ???
+    use_https - set true if use XMLA gateway over https
+    dns_suffixes - list dns zone if use ssas servers in different domains.
+        Used to search for the main domain for the ssas server if it is not specified in the cube properties
+
+    """
+    username: str = pydantic.Field(description="Windows account username")
+    password: str = pydantic.Field(description="Windows account password")
+    instance: str
+    host_port: str = pydantic.Field(
+        default="localhost:81", description="XMLA gateway url"
+    )
+    server_alias: str = pydantic.Field(default="localhost")
+
+    virtual_directory_name: str = pydantic.Field(
+        default="ssas", description="Report Virtual Directory URL name"
+    )
+    ssas_instance: str
+    use_https: bool = pydantic.Field(default=True)
+    ssas_instance_auth_type: str = pydantic.Field(default="HTTPKerberosAuth", description="SSAS instance auth type")
+
+    dns_suffixes: List = pydantic.Field(default_factory=list)
+    default_ssas_instances_by_server: Dict = pydantic.Field(default_factory=dict)
+
+    @pydantic.validator('ssas_instance_auth_type')
+    def check_ssas_instance_auth_type(cls, v):
+        if v not in ["HTTPBasicAuth", "HTTPKerberosAuth"]:
+            raise ValueError("Support only HTTPBasicAuth or HTTPKerberosAuth auth type")
+        return v
+
+    @property
+    def use_dns_resolver(self) -> bool:
+        return bool(self.dns_suffixes)
+
+    @property
+    def base_api_url(self) -> str:
+        protocol = "https" if self.use_https else "http"
+        return f"{protocol}://{self.host_port}/{self.virtual_directory_name}/{self.ssas_instance}/msmdpump.dll"
+
+    @property
+    def host(self) -> str:
+        return self.server_alias or self.host_port.split(":")[0]
+
+
+class SsasServerHTTPSourceConfig(SsasServerHTTPConfig):
+    platform_name: str = "ssas"
+    platform_urn: str = builder.make_data_platform_urn(platform=platform_name)
+    report_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()
+    chart_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()