From b1bae8d370b20b1def75bc41a31733e43cb02f15 Mon Sep 17 00:00:00 2001 From: Minura Punchihewa <49385643+MinuraPunchihewa@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:47:50 +0530 Subject: [PATCH] feat: Separated the Lint Dependencies for the Plugins (#901) * separated the lint requirements for kedro-datasets Signed-off-by: Minura Punchihewa * added the lint requirements to all Signed-off-by: Minura Punchihewa * added the make cmd for installing lint dependencies Signed-off-by: Minura Punchihewa * separated the lint requirements for the other plugins Signed-off-by: Minura Punchihewa * updated the lint workflow template to install correct dependencies Signed-off-by: Minura Punchihewa * Trying to fix lint Signed-off-by: Ankita Katiyar --------- Signed-off-by: Minura Punchihewa Signed-off-by: Ankita Katiyar Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Co-authored-by: Ankita Katiyar --- .github/workflows/lint.yml | 12 +++++----- Makefile | 3 +++ kedro-airflow/pyproject.toml | 13 ++++++----- .../polars/lazy_polars_dataset.py | 4 +++- kedro-datasets/pyproject.toml | 22 +++++++++++-------- kedro-docker/pyproject.toml | 15 ++++++++----- kedro-telemetry/pyproject.toml | 13 ++++++----- 7 files changed, 50 insertions(+), 32 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1594b493d..b8e471d8a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,14 +34,14 @@ jobs: python -m pip install "uv==0.2.21" - name: Install dependencies run: | - cd ${{ inputs.plugin }} - uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main" - uv pip install --system "${{inputs.plugin}}[test] @ ." - uv pip freeze --system + cd ${{ inputs.plugin }} + uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main" + uv pip install --system "${{inputs.plugin}}[lint] @ ." + uv pip freeze --system - name: Install pre-commit hooks run: | - pre-commit install --install-hooks - pre-commit install --hook-type pre-push + pre-commit install --install-hooks + pre-commit install --hook-type pre-push - name: Run linter run: make plugin=${{ inputs.plugin }} lint - name: Run mypy diff --git a/Makefile b/Makefile index 324332972..c7946d605 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,9 @@ secret-scan: install-test-requirements: cd $(plugin) && uv pip install ".[test]" +install-lint-requirements: + cd $(plugin) && uv pip install ".[lint]" + install-pre-commit: pre-commit install --install-hooks diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index 5f5a9be65..ec7563cdd 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -25,20 +25,23 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ "apache-airflow<3.0", - "bandit", "behave", - "black~=22.0", "coverage>=7.2.0", "kedro-datasets", - "mypy~=1.0", - "pre-commit>=2.9.2", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist", + "wheel", +] + +lint = [ + "bandit", + "black~=22.0", + "mypy~=1.0", + "pre-commit>=2.9.2", "trufflehog>=2.1.0, <3.0", "ruff~=0.0.290", - "wheel", # mypy requirements "types-PyYAML", "types-cachetools", diff --git a/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py b/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py index d1c59e049..c41c3fce9 100644 --- a/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py +++ b/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py @@ -27,7 +27,9 @@ logger = logging.getLogger(__name__) -class LazyPolarsDataset(AbstractVersionedDataset[pl.LazyFrame, PolarsFrame]): +class LazyPolarsDataset( + AbstractVersionedDataset[pl.LazyFrame, pl.LazyFrame | pl.DataFrame] +): """``LazyPolarsDataset`` loads/saves data from/to a data file using an underlying filesystem (e.g.: local, S3, GCS). It uses Polars to handle the type of read/write target. It uses lazy loading with Polars Lazy API, but it can diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml index c33584112..005e6cdf8 100644 --- a/kedro-datasets/pyproject.toml +++ b/kedro-datasets/pyproject.toml @@ -203,11 +203,8 @@ docs = [ test = [ "accelerate<0.32", # Temporary pin "adlfs~=2023.1", - "bandit>=1.6.2, <2.0", "behave==1.2.6", "biopython~=1.73", - "blacken-docs==1.9.2", - "black~=22.0", "cloudpickle<=2.0.0", "compress-pickle[lz4]~=2.1.0", "coverage>=7.2.0", @@ -222,7 +219,6 @@ test = [ "hdfs>=2.5.8, <3.0", "holoviews>=1.13.0", "ibis-framework[duckdb,examples]", - "import-linter[toml]==1.2.6", "ipython>=7.31.1, <8.0", "Jinja2<3.2.0", "joblib>=0.14", @@ -232,7 +228,6 @@ test = [ "matplotlib>=3.5, <3.6", "memory_profiler>=0.50.0, <1.0", "moto==5.0.0", - "mypy~=1.0", "networkx~=2.4", "opencv-python~=4.5.5.64", "openpyxl>=3.0.3, <4.0", @@ -241,7 +236,6 @@ test = [ "Pillow~=10.0", "plotly>=4.8.0, <6.0", "polars[xlsx2csv, deltalake]~=0.18.0", - "pre-commit>=2.9.2", "pyarrow>=1.0; python_version < '3.11'", "pyarrow>=7.0; python_version >= '3.11'", # Adding to avoid numpy build errors "pyodbc~=5.0", @@ -254,7 +248,6 @@ test = [ "redis~=4.1", "requests-mock~=1.6", "requests~=2.20", - "ruff~=0.0.290", "s3fs>=2021.04", "snowflake-snowpark-python~=1.0; python_version < '3.11'", "scikit-learn>=1.0.2,<2", @@ -265,13 +258,24 @@ test = [ "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'", "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'", "triad>=0.6.7, <1.0", - "trufflehog~=2.1", "xarray>=2023.1.0", "xlsxwriter~=1.0", # huggingface "datasets", "huggingface_hub", "transformers[torch]", +] + +# Lint requirements +lint = [ + "bandit>=1.6.2, <2.0", + "blacken-docs==1.9.2", + "black~=22.0", + "import-linter[toml]==1.2.6", + "mypy~=1.0", + "pre-commit>=2.9.2", + "ruff~=0.0.290", + "trufflehog~=2.1", # mypy related dependencies "types-cachetools", "types-PyYAML", @@ -297,7 +301,7 @@ experimental = [ ] # All requirements -all = ["kedro-datasets[test,docs]"] +all = ["kedro-datasets[test,docs,lint]"] [project.urls] Source = "https://github.com/kedro-org/kedro-plugins/tree/main/kedro-datasets" diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml index c607ef9a2..15c8d04fc 100644 --- a/kedro-docker/pyproject.toml +++ b/kedro-docker/pyproject.toml @@ -24,22 +24,25 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ - "bandit", "behave", - "black~=22.0", "coverage>=7.2.0", "docker", - "mypy~=1.0", - "pre-commit>=2.9.2", "psutil", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist[psutil]~=2.2.1", "PyYAML>=5.1, <7.0", - "trufflehog>=2.0.99, <3.0", + "wheel==0.32.2", +] + +lint = [ + "bandit", + "black~=22.0", + "mypy~=1.0", + "pre-commit>=2.9.2", + "trufflehog>=2.1.0, <3.0", "ruff~=0.0.290", - "wheel==0.32.2" ] [project.entry-points."kedro.project_commands"] diff --git a/kedro-telemetry/pyproject.toml b/kedro-telemetry/pyproject.toml index fcb56fcde..45f9d995d 100644 --- a/kedro-telemetry/pyproject.toml +++ b/kedro-telemetry/pyproject.toml @@ -24,18 +24,21 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ - "bandit>=1.6.2, <2.0", - "black~=22.0", - "mypy~=1.0", - "pre-commit>=2.9.2", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist[psutil]~=2.2.1", "PyYAML==5.3.1", # Temporary fix, to be removed + "wheel", +] + +lint = [ + "bandit>=1.6.2, <2.0", + "black~=22.0", + "mypy~=1.0", + "pre-commit>=2.9.2", "trufflehog>=2.1.0, <3.0", "ruff~=0.0.290", - "wheel", # mypy requirements "types-requests", "types-PyYAML",