Merge branch 'main' into fix-snowpark

.github/workflows/kedro-airflow.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-airflow
@@ -34,13 +34,13 @@ jobs:
     with:
       plugin: kedro-airflow
       os: ubuntu-latest
-      python-version: "3.8"
+      python-version: "3.11"
 
   e2e-tests:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       plugin: kedro-airflow

.github/workflows/kedro-datasets.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-datasets
@@ -53,6 +53,7 @@ jobs:
           restore-keys: kedro-datasets
       - name: Install dependencies
         run: |
+          python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
           cd kedro-datasets
           pip install ".[docs]"
           pip install ".[test]"

.github/workflows/kedro-docker.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-docker
@@ -34,13 +34,13 @@ jobs:
     with:
       plugin: kedro-docker
       os: ubuntu-latest
-      python-version: "3.8"
+      python-version: "3.11"
 
   e2e-tests:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       plugin: kedro-docker

.github/workflows/kedro-telemetry.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-telemetry
@@ -34,13 +34,13 @@ jobs:
     with:
       plugin: kedro-telemetry
       os: ubuntu-latest
-      python-version: "3.8"
+      python-version: "3.11"
 
   e2e-tests:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       plugin: kedro-telemetry

.github/workflows/lint.yml

@@ -32,6 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
             cd ${{ inputs.plugin }}
+            python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
             pip install git+https://github.com/kedro-org/kedro@main
             pip install ".[test]"
             pip freeze

.github/workflows/unit-tests.yml

@@ -46,12 +46,16 @@ jobs:
       - name: Install dependencies
         run: |
           cd ${{ inputs.plugin }}
+          python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
           pip install ".[test]"
       - name: pip freeze
         run: pip freeze
-      - name: Run unit tests for Linux / all plugins
-        if: inputs.os != 'windows-latest'
+      - name: Run unit tests for Linux / kedro-airflow, kedro-docker, kedro-telemetry
+        if: inputs.os != 'windows-latest' && inputs.plugin != 'kedro-datasets'
         run: make plugin=${{ inputs.plugin }} test
+      - name: Run unit tests for Linux / kedro-datasets
+        if: inputs.os != 'windows-latest' && inputs.plugin == 'kedro-datasets'
+        run: make dataset-tests
       - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry
         if: inputs.os == 'windows-latest' && inputs.plugin != 'kedro-datasets'
         run: |

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_stages: [commit, manual]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v3.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -14,178 +14,53 @@ repos:
       - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems
       - id: check-merge-conflict # Check for files that contain merge conflict strings.
       - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source.
-      - id: flake8
-        files: ^(kedro-datasets/kedro_datasets/|kedro-airflow/kedro_airflow/|kedro-docker/kedro_docker/|kedro-telemetry/kedro_telemetry/)
-        args:
-          - "--max-line-length=88"
-          - "--max-complexity=18"
-          - "--select=B,C,E,F,W,T4,B9"
-          - "--ignore=E203,E266,E501,W503"
-        exclude: "^kedro_airflow/dag_template.py|^template.py"
+
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+    - id: blacken-docs
+      args:
+        - "--rst-literal-blocks"
+      additional_dependencies:
+        - black==22.12.0
 
   - repo: local
     hooks:
-      # pylint quick checks
-      - id: pylint-quick-kedro-datasets
-        name: "Quick PyLint on kedro_datasets/*"
+      - id: ruff-kedro-datasets
+        name: "Ruff on kedro_datasets/*"
         language: system
-        types: [file, python]
         files: ^kedro-datasets/kedro_datasets/
         exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$
-        entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass
-        stages: [commit]
-
-      - id: pylint-quick-kedro-airflow
-        name: "Quick PyLint on kedro_airflow/*"
-        language: system
-        types: [file, python]
-        files: ^kedro-airflow/kedro_airflow/
-        exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$
-        entry: pylint --disable=unnecessary-pass
-        stages: [commit]
-
-      - id: pylint-quick-kedro-docker
-        name: "Quick PyLint on kedro_docker/*"
-        language: system
-        types: [file, python]
-        files: ^kedro-docker/kedro_docker/
-        exclude: ^(?!kedro-docker/kedro_docker/).*\.py$
-        entry: pylint --disable=unnecessary-pass
-        stages: [commit]
-
-      - id: pylint-quick-kedro-telemetry
-        name: "Quick PyLint on kedro_telemetry/*"
-        language: system
-        types: [file, python]
-        files: ^kedro-telemetry/kedro_telemetry/
-        exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
-        entry: pylint --disable=unnecessary-pass
-        stages: [commit]
-
-      # pylint full checks
-      - id: pylint-kedro-datasets
-        name: "PyLint on kedro_datasets/*"
-        language: system
-        files: ^kedro-datasets/kedro_datasets/.*\.py$
-        exclude: ^(?!kedro-datasets/kedro_datasets/).*\.py$
         pass_filenames: false
-        stages: [manual]
-        entry: pylint --rcfile kedro-datasets/pyproject.toml --disable=unnecessary-pass,E0401 kedro-datasets/kedro_datasets
-
-      - id: pylint-kedro-datasets-features
-        name: "PyLint on kedro-datasets features/*"
-        language: system
-        files: ^kedro-datasets/features/.*\.py$
-        exclude: ^(?!kedro-datasets/features/).*\.py$
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --rcfile kedro-datasets/pyproject.toml  --disable=missing-docstring,no-name-in-module,E0401 kedro-datasets/features
-
-      - id: pylint-kedro-datasets-tests
-        name: "PyLint on kedro-datasets tests/*"
-        language: system
-        files: ^kedro-datasets/tests/.*\.py$
-        exclude: ^(?!kedro-datasets/tests/).*\.py$
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --rcfile kedro-datasets/pyproject.toml  --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,E0401 kedro-datasets/tests
+        stages: [ manual ]
+        entry: ruff kedro-datasets --fix --exit-non-zero-on-fix
 
-      - id: pylint-kedro-airflow
-        name: "PyLint on kedro_airflow/*"
+      - id: ruff-kedro-airflow
+        name: "Ruff on kedro_airflow/*"
         language: system
-        files: ^kedro-airflow/kedro_airflow/.*\.py$
+        files: ^kedro-airflow/kedro_airflow/
         exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$
         pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=unnecessary-pass,E0401 kedro-airflow/kedro_airflow
-
-      - id: pylint-kedro-airflow-features
-        name: "PyLint on kedro-airflow features/*"
-        language: system
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=missing-docstring,no-name-in-module kedro-airflow/features
-
-      - id: pylint-kedro-airflow-tests
-        name: "PyLint on kedro-airflow tests/*"
-        language: system
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-airflow/tests
+        stages: [ manual ]
+        entry: ruff kedro-airflow --fix --exit-non-zero-on-fix
 
-      - id: pylint-kedro-docker
-        name: "PyLint on kedro_docker/*"
+      - id: ruff-kedro-docker
+        name: "Ruff on kedro_docker/*"
         language: system
-        files: ^kedro-docker/kedro_docker/.*\.py$
+        files: ^kedro-docker/kedro_docker/
         exclude: ^(?!kedro-docker/kedro_docker/).*\.py$
         pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=unnecessary-pass,E0401 kedro-docker/kedro_docker
-
-      - id: pylint-kedro-docker-features
-        name: "PyLint on kedro-docker features/*"
-        language: system
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=missing-docstring,no-name-in-module kedro-docker/features
-
-      - id: pylint-kedro-docker-tests
-        name: "PyLint on kedro-docker tests/*"
-        language: system
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=missing-docstring,redefined-outer-name,invalid-name,protected-access,too-many-arguments kedro-docker/tests
-
-      - id: pylint-kedro-telemetry
-        name: "PyLint on kedro_telemetry/*"
-        language: system
-        files: ^kedro-telemetry/kedro_telemetry/.*\.py$
-        exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
-        pass_filenames: false
-        stages: [manual]
-        entry: pylint --disable=unnecessary-pass,E0401 kedro-telemetry/kedro_telemetry
-
-      - id: pylint-kedro-telemetry-features
-        name: "PyLint on kedro-docker features/*"
-        language: system
         stages: [ manual ]
-        entry: echo 'Not needed to run for this directory'
-        files: .*
+        entry: ruff kedro-docker --fix --exit-non-zero-on-fix
 
-      - id: pylint-kedro-telemetry-tests
-        name: "PyLint on kedro-telemetry tests/*"
+      - id: ruff-kedro-telemetry
+        name: "Ruff on kedro_telemetry/*"
         language: system
+        files: ^kedro-telemetry/kedro_telemetry/
+        exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
         pass_filenames: false
         stages: [manual]
-        entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments kedro-telemetry/tests
-
-      - id: isort-kedro-datasets
-        name: "Sort imports"
-        language: system
-        types: [ file, python ]
-        files: ^kedro-datasets/
-        entry: isort
-
-      - id: isort-kedro-docker
-        name: "Sort imports"
-        language: system
-        types: [ file, python ]
-        files: ^kedro-docker/
-        entry: isort
-
-      - id: isort-kedro-airflow
-        name: "Sort imports"
-        language: system
-        types: [ file, python ]
-        files: ^kedro-airflow/
-        entry: isort
-
-      - id: isort-kedro-telemetry
-        name: "Sort imports"
-        language: system
-        types: [ file, python ]
-        files: ^kedro-telemetry/
-        entry: isort
+        entry: ruff kedro-telemetry --fix --exit-non-zero-on-fix
 
       - id: black-kedro-datasets
         name: "Black"

CONTRIBUTING.md

@@ -15,20 +15,19 @@ We also curate a [GitHub repo that lists content created by the Kedro community]
 ## Contribute to the project
 
 There are quite a few ways to contribute to Kedro, such as answering questions about Kedro to help others, fixing a typo on the documentation, reporting a bug, reviewing pull requests or adding a feature.
-ls
 Take a look at some of our [contribution suggestions on the Kedro GitHub Wiki](https://github.com/kedro-org/kedro/wiki/Contribute-to-Kedro)!
 
 ## Which plugin contributions are likely to be accepted?
 
 Dataset contributions to the [Kedro-Datasets](https://github.com/kedro-org/kedro-plugins/tree/main/kedro-datasets) plugin are the most frequently accepted, since they do not require any changes to the framework itself.
 
-However, we accept contributions to any of the other [Kedro-Plugins](https://github.com/kedro-org/kedro-plugins) or the framework or `Kedro-Viz`. As a guide, contributions based on existing issues from the Kedro team, or issues that the team has deemed useful, are most likely to be accepted. Any contributions that affect fundamental changes to the Kedro Framework would require discussion first. In this case, we recommend opening an issue instead of a pull request.
+However, we accept contributions to any of the other [Kedro-Plugins](https://github.com/kedro-org/kedro-plugins) or the framework or [`Kedro-Viz`](https://github.com/kedro-org/kedro-viz). As a guide, contributions based on existing issues from the Kedro team, or issues that the team has deemed useful, are most likely to be accepted. Any contributions that affect fundamental changes to the Kedro Framework would require discussion first. In this case, we recommend opening an issue instead of a pull request.
 
-Make sure to check out the contributing guides for [Kedro-Docker](https://github.com/kedro-org/kedro-plugins/blob/main/kedro-docker/CONTRIBUTING.md) and [Kedro-Airflow](https://github.com/kedro-org/kedro-plugins/blob/main/kedro-airflow/CONTRIBUTING.md) if you intend to contribute to those specific plugins.
+Make sure to check out the contributing guides for [Kedro-Docker](https://github.com/kedro-org/kedro-plugins/blob/main/kedro-docker/CONTRIBUTING.md), [Kedro-Datasets](https://github.com/kedro-org/kedro-plugins/blob/main/kedro-datasets/CONTRIBUTING.md) and [Kedro-Airflow](https://github.com/kedro-org/kedro-plugins/blob/main/kedro-airflow/CONTRIBUTING.md) if you intend to contribute to those specific plugins.
 
 ## Join the Technical Steering Committee
 Kedro is an incubating project in [LF AI & Data](https://lfaidata.foundation/), a sub-organisation within the Linux Foundation that focuses on open innovation within the data and AI space.
 
-The project is governed by a group of maintainers, known as the Technical Steering Committee (TSC); read more about the structure of our TSC in our [Technical Charter](./kedro_technical_charter.pdf).
+The project is governed by a group of maintainers, known as the Technical Steering Committee (TSC); read more about the structure of our TSC in our [Technical Charter](https://github.com/kedro-org/kedro/blob/main/kedro_technical_charter.pdf).
 
 We regularly invite community members to join the TSC and help define the future of the Kedro project. Read the [guidance on becoming a Kedro maintainer](https://docs.kedro.org/en/stable/contribution/technical_steering_committee.html) to understand the process of joining the TSC.

Makefile

@@ -14,11 +14,16 @@ install-pip-setuptools:
 	python -m pip install -U pip setuptools wheel
 
 lint:
-	pre-commit run trailing-whitespace --all-files && pre-commit run end-of-file-fixer --all-files && pre-commit run check-yaml --all-files && pre-commit run check-added-large-files --all-files && pre-commit run check-case-conflict --all-files && pre-commit run check-merge-conflict --all-files && pre-commit run debug-statements --all-files && pre-commit run flake8 --all-files && pre-commit run isort-$(plugin) --all-files --hook-stage manual && pre-commit run black-$(plugin) --all-files --hook-stage manual && pre-commit run secret_scan --all-files --hook-stage manual && pre-commit run bandit --all-files --hook-stage manual && pre-commit run pylint-$(plugin) --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-features --all-files --hook-stage manual && pre-commit run pylint-$(plugin)-tests --all-files --hook-stage manual
+	pre-commit run -a --hook-stage manual ruff-$(plugin) && pre-commit run trailing-whitespace --all-files && pre-commit run end-of-file-fixer --all-files && pre-commit run check-yaml --all-files && pre-commit run check-added-large-files --all-files && pre-commit run check-case-conflict --all-files && pre-commit run check-merge-conflict --all-files && pre-commit run debug-statements --all-files && pre-commit run black-$(plugin) --all-files --hook-stage manual && pre-commit run secret_scan --all-files --hook-stage manual && pre-commit run bandit --all-files --hook-stage manual
 
 test:
 	cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile
 
+# Run test_tensorflow_model_dataset separately, because these tests are flaky when run as part of the full test-suite
+dataset-tests:
+	cd kedro-datasets && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile --ignore tests/tensorflow --ignore tests/databricks
+	cd kedro-datasets && pytest tests/tensorflow/test_tensorflow_model_dataset.py  --no-cov
+
 test-sequential:
 	cd $(plugin) && pytest tests --cov-config pyproject.toml
 
@@ -37,7 +42,7 @@ clean:
 install-test-requirements:
 	cd $(plugin) && pip install ".[test]"
 
-install-pre-commit: install-test-requirements
+install-pre-commit:
 	pre-commit install --install-hooks
 
 uninstall-pre-commit: