feat: Separated the Lint Dependencies for the Plugins (#901)

* separated the lint requirements for kedro-datasets Signed-off-by: Minura Punchihewa <[email protected]> * added the lint requirements to all Signed-off-by: Minura Punchihewa <[email protected]> * added the make cmd for installing lint dependencies Signed-off-by: Minura Punchihewa <[email protected]> * separated the lint requirements for the other plugins Signed-off-by: Minura Punchihewa <[email protected]> * updated the lint workflow template to install correct dependencies Signed-off-by: Minura Punchihewa <[email protected]> * Trying to fix lint Signed-off-by: Ankita Katiyar <[email protected]> --------- Signed-off-by: Minura Punchihewa <[email protected]> Signed-off-by: Ankita Katiyar <[email protected]> Co-authored-by: Ankita Katiyar <[email protected]> Co-authored-by: Ankita Katiyar <[email protected]>
kedro-org · Oct 22, 2024 · b1bae8d · b1bae8d
1 parent 92952cf
commit b1bae8d
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 32 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -34,14 +34,14 @@ jobs:
           python -m pip install "uv==0.2.21"
       - name: Install dependencies
         run: |
-            cd ${{ inputs.plugin }}
-            uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main"
-            uv pip install --system "${{inputs.plugin}}[test] @ ."
-            uv pip freeze --system
+          cd ${{ inputs.plugin }}
+          uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main"
+          uv pip install --system "${{inputs.plugin}}[lint] @ ."
+          uv pip freeze --system
       - name: Install pre-commit hooks
         run: |
-            pre-commit install --install-hooks
-            pre-commit install --hook-type pre-push
+          pre-commit install --install-hooks
+          pre-commit install --hook-type pre-push
       - name: Run linter
         run: make plugin=${{ inputs.plugin }} lint
       - name: Run mypy

diff --git a/Makefile b/Makefile
@@ -27,6 +27,9 @@ secret-scan:
 install-test-requirements:
 	cd $(plugin) && uv pip install ".[test]"
 
+install-lint-requirements:
+	cd $(plugin) && uv pip install ".[lint]"
+
 install-pre-commit:
 	pre-commit install --install-hooks
 

diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml
@@ -25,20 +25,23 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues"
 [project.optional-dependencies]
 test = [
     "apache-airflow<3.0",
-    "bandit",
     "behave",
-    "black~=22.0",
     "coverage>=7.2.0",
     "kedro-datasets",
-    "mypy~=1.0",
-    "pre-commit>=2.9.2",
     "pytest",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist",
+    "wheel",
+]
+
+lint = [
+    "bandit",
+    "black~=22.0",
+    "mypy~=1.0",
+    "pre-commit>=2.9.2",
     "trufflehog>=2.1.0, <3.0",
     "ruff~=0.0.290",
-    "wheel",
     # mypy requirements
     "types-PyYAML",
     "types-cachetools",

diff --git a/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py b/kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py
@@ -27,7 +27,9 @@
 logger = logging.getLogger(__name__)
 
 
-class LazyPolarsDataset(AbstractVersionedDataset[pl.LazyFrame, PolarsFrame]):
+class LazyPolarsDataset(
+    AbstractVersionedDataset[pl.LazyFrame, pl.LazyFrame | pl.DataFrame]
+):
     """``LazyPolarsDataset`` loads/saves data from/to a data file using an
     underlying filesystem (e.g.: local, S3, GCS). It uses Polars to handle
     the type of read/write target. It uses lazy loading with Polars Lazy API, but it can

diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml
@@ -203,11 +203,8 @@ docs = [
 test = [
     "accelerate<0.32", # Temporary pin
     "adlfs~=2023.1",
-    "bandit>=1.6.2, <2.0",
     "behave==1.2.6",
     "biopython~=1.73",
-    "blacken-docs==1.9.2",
-    "black~=22.0",
     "cloudpickle<=2.0.0",
     "compress-pickle[lz4]~=2.1.0",
     "coverage>=7.2.0",
@@ -222,7 +219,6 @@ test = [
     "hdfs>=2.5.8, <3.0",
     "holoviews>=1.13.0",
     "ibis-framework[duckdb,examples]",
-    "import-linter[toml]==1.2.6",
     "ipython>=7.31.1, <8.0",
     "Jinja2<3.2.0",
     "joblib>=0.14",
@@ -232,7 +228,6 @@ test = [
     "matplotlib>=3.5, <3.6",
     "memory_profiler>=0.50.0, <1.0",
     "moto==5.0.0",
-    "mypy~=1.0",
     "networkx~=2.4",
     "opencv-python~=4.5.5.64",
     "openpyxl>=3.0.3, <4.0",
@@ -241,7 +236,6 @@ test = [
     "Pillow~=10.0",
     "plotly>=4.8.0, <6.0",
     "polars[xlsx2csv, deltalake]~=0.18.0",
-    "pre-commit>=2.9.2",
     "pyarrow>=1.0; python_version < '3.11'",
     "pyarrow>=7.0; python_version >= '3.11'",  # Adding to avoid numpy build errors
     "pyodbc~=5.0",
@@ -254,7 +248,6 @@ test = [
     "redis~=4.1",
     "requests-mock~=1.6",
     "requests~=2.20",
-    "ruff~=0.0.290",
     "s3fs>=2021.04",
     "snowflake-snowpark-python~=1.0; python_version < '3.11'",
     "scikit-learn>=1.0.2,<2",
@@ -265,13 +258,24 @@ test = [
     "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
     "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
     "triad>=0.6.7, <1.0",
-    "trufflehog~=2.1",
     "xarray>=2023.1.0",
     "xlsxwriter~=1.0",
     # huggingface
     "datasets",
     "huggingface_hub",
     "transformers[torch]",
+]
+
+# Lint requirements
+lint = [
+    "bandit>=1.6.2, <2.0",
+    "blacken-docs==1.9.2",
+    "black~=22.0",
+    "import-linter[toml]==1.2.6",
+    "mypy~=1.0",
+    "pre-commit>=2.9.2",
+    "ruff~=0.0.290",
+    "trufflehog~=2.1",
     # mypy related dependencies
     "types-cachetools",
     "types-PyYAML",
@@ -297,7 +301,7 @@ experimental = [
 ]
 
 # All requirements
-all = ["kedro-datasets[test,docs]"]
+all = ["kedro-datasets[test,docs,lint]"]
 
 [project.urls]
 Source = "https://github.com/kedro-org/kedro-plugins/tree/main/kedro-datasets"

diff --git a/kedro-docker/pyproject.toml b/kedro-docker/pyproject.toml
@@ -24,22 +24,25 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues"
 
 [project.optional-dependencies]
 test = [
-    "bandit",
     "behave",
-    "black~=22.0",
     "coverage>=7.2.0",
     "docker",
-    "mypy~=1.0",
-    "pre-commit>=2.9.2",
     "psutil",
     "pytest",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist[psutil]~=2.2.1",
     "PyYAML>=5.1, <7.0",
-    "trufflehog>=2.0.99, <3.0",
+    "wheel==0.32.2",
+]
+
+lint = [
+    "bandit",
+    "black~=22.0",
+    "mypy~=1.0",
+    "pre-commit>=2.9.2",
+    "trufflehog>=2.1.0, <3.0",
     "ruff~=0.0.290",
-    "wheel==0.32.2"
 ]
 
 [project.entry-points."kedro.project_commands"]

diff --git a/kedro-telemetry/pyproject.toml b/kedro-telemetry/pyproject.toml
@@ -24,18 +24,21 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues"
 
 [project.optional-dependencies]
 test = [
-    "bandit>=1.6.2, <2.0",
-    "black~=22.0",
-    "mypy~=1.0",
-    "pre-commit>=2.9.2",
     "pytest",
     "pytest-cov",
     "pytest-mock",
     "pytest-xdist[psutil]~=2.2.1",
     "PyYAML==5.3.1", # Temporary fix, to be removed
+    "wheel",
+]
+
+lint = [
+    "bandit>=1.6.2, <2.0",
+    "black~=22.0",
+    "mypy~=1.0",
+    "pre-commit>=2.9.2",
     "trufflehog>=2.1.0, <3.0",
     "ruff~=0.0.290",
-    "wheel",
     # mypy requirements
     "types-requests",
     "types-PyYAML",