diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afe081d9c..202e0be9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,6 @@ pip install -r dev-requirements.txt cd tests py.test -vvv --target ``` - ## Pre-commit hooks We use several pre-commit hooks in order to ensure code quality. These will also @@ -104,4 +103,5 @@ installing the dev-requirements as shown in the previous section): ```bash # Run in the root directory of the repository pre-commit install -``` + + diff --git a/Dockerfile b/Dockerfile index 2cfa5a61a..7f64fe71c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,3 +22,9 @@ RUN pip install --upgrade pip RUN pip install --no-cache-dir "/app[all]" ENTRYPOINT ["edr"] +Those features will allow Elementary to get all required info for computing the data lineage graph. + +### Connecting Looker to Elementary + +Navigate to the **Account settings > Environments** and choose the environment to which you would like to connect Elementary. +Choose the Power BI connection and provide the following details to validate and complete the integration. \ No newline at end of file diff --git a/README.md b/README.md index 99a2b80c7..a887521fa 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,29 @@ For additional information and help, you can use one of these channels: + Installation + +1. Clone the repository: +```bash +git clone https://github.com/user/repo.git +Install the dependencies: +pip install -r requirements.txt +Usage + +python script.py --option argument +For detailed instructions on usage, refer to the documentation. +Q: How do I install X? +A: Run the following command: +pip install X +Q: I'm getting an error when I run the script. +A: Make sure you've installed all the required dependencies by running: +pip install -r requirements.txt +Feel free to replace placeholder text like `"https://github.com/user/repo.git"` and `"link-t + +Images & Interactive Links: Ensure the links and images (such as GitHub stars, Slack badges, and demo GIFs) are properly formatted and hosted at accessible URLs. +Centering Elements: You have centered the main logo and content, which is great for a clean, focused layout. Just make sure the styling matches the platform you're using (like GitHub or a website). +Badges and Descriptions: You might want to double-check the badge link for downloads as it contains an extra "&left_text=Downloads" that might not work as intended.s \ No newline at end of file diff --git a/elementary/monitor/data_monitoring/data_monitoring.py b/elementary/monitor/data_monitoring/data_monitoring.py index 74888eea8..ad8b4bf4c 100644 --- a/elementary/monitor/data_monitoring/data_monitoring.py +++ b/elementary/monitor/data_monitoring/data_monitoring.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, Optional, Union, cast from packaging import version @@ -58,7 +58,7 @@ def __init__( self.selector_filter = selector_filter def _init_internal_dbt_runner(self): - internal_dbt_runner = create_dbt_runner( + return create_dbt_runner( dbt_project_utils.CLI_DBT_PROJECT_PATH, self.config.profiles_dir, self.config.profile_target, @@ -66,13 +66,9 @@ def _init_internal_dbt_runner(self): run_deps_if_needed=self.config.run_dbt_deps_if_needed, force_dbt_deps=self.force_update_dbt_package, ) - return internal_dbt_runner def properties(self): - data_monitoring_properties = { - "data_monitoring_properties": self.execution_properties - } - return data_monitoring_properties + return {"data_monitoring_properties": self.execution_properties} def get_elementary_database_and_schema(self): try: @@ -82,7 +78,7 @@ def get_elementary_database_and_schema(self): logger.info(f"Elementary's database and schema: '{relation}'") return relation except Exception as ex: - logger.error("Failed to parse Elementary's database and schema.") + logger.error(f"Failed to parse Elementary's database and schema: {str(ex)}") if self.tracking: self.tracking.record_internal_exception(ex) return "." @@ -94,7 +90,7 @@ def get_latest_invocation(self) -> Dict[str, Any]: )[0] return json.loads(latest_invocation)[0] if latest_invocation else {} except Exception as err: - logger.error(f"Unable to get the latest invocation: {err}") + logger.error(f"Unable to get the latest invocation: {str(err)}") if self.tracking: self.tracking.record_internal_exception(err) return {} @@ -106,33 +102,16 @@ def _check_dbt_package_compatibility(dbt_pkg_ver_str: str) -> None: logger.warning("Could not get package version!") return - dbt_pkg_ver = cast(version.Version, version.parse(dbt_pkg_ver_str)) - py_pkg_ver = cast(version.Version, version.parse(py_pkg_ver_str)) - if dbt_pkg_ver.major > py_pkg_ver.major or ( - dbt_pkg_ver.major == py_pkg_ver.major - and dbt_pkg_ver.minor > py_pkg_ver.minor - ): - logger.warning( - f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n " - "To fix please run:\n" - "pip install --upgrade elementary-data\n", - ) - return + dbt_pkg_ver, py_pkg_ver = map(version.parse, [dbt_pkg_ver_str, py_pkg_ver_str]) - if dbt_pkg_ver.major < py_pkg_ver.major or ( - dbt_pkg_ver.major == py_pkg_ver.major - and dbt_pkg_ver.minor < py_pkg_ver.minor - ): + if dbt_pkg_ver != py_pkg_ver: + action = "upgrade" if dbt_pkg_ver > py_pkg_ver else "update" logger.warning( - f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n " - "To fix please update your packages.yml, and run:\n" - "dbt deps && dbt run --select elementary\n", + f"Incompatible versions: edr ({py_pkg_ver}) vs dbt package ({dbt_pkg_ver}). " + f"Please {action} the package." ) - return - - logger.info( - f"edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}) are compatible." - ) + else: + logger.info(f"Versions are compatible: {py_pkg_ver}") def _get_warehouse_info(self, hash_id: bool = False) -> Optional[WarehouseInfo]: try: @@ -145,6 +124,6 @@ def _get_warehouse_info(self, hash_id: bool = False) -> Optional[WarehouseInfo]: id=warehouse_unique_id if not hash_id else hash(warehouse_unique_id), type=warehouse_type, ) - except Exception: - logger.debug("Could not get warehouse info.", exc_info=True) + except Exception as ex: + logger.debug(f"Could not get warehouse info: {str(ex)}", exc_info=True) return None