Merge pull request #62 from neptune-ai/aw/fix_neptune_requirement

update requirement for neptune
neptune-ai · Mar 10, 2023 · 273f6db · 273f6db
2 parents 1fa0346 + 3a1a4ab
commit 273f6db
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 96 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,10 +6,10 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        os: [macos-latest, ubuntu-latest]
+        os: [ubuntu-latest]
         python-version: [3.7, 3.8, 3.9, "3.10"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - uses: actions/setup-python@v2
         with:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,11 @@
-## [UNRELEASED] 0.1.5
+##  0.1.5
 
 ### Fixes
 - Fixed some of latest `neptune-client` warning messages ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58))
 - Fixed failing run on latest MacOS 12 ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58))
 
 ### Changes
+- Removed `neptune` and `neptune-client` from base requirements for backward compatibility ([#62](https://github.com/neptune-ai/kedro-neptune/pull/62))
 - Simplified example project and removed unrelated files ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58))
 - Better life-time of the internally created Run in `NeptuneRunDataSet` ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58))
 - Updated the integration for compatibility with `neptune-client` `1.0.0`. ([#59](https://github.com/neptune-ai/kedro-neptune/pull/59))

diff --git a/pyproject.toml b/pyproject.toml
@@ -15,22 +15,21 @@ python = "^3.7"
 importlib-metadata = { version = "*", python = "<3.8" }
 
 # Base requirements
-neptune-client = ">=0.16.17"
 kedro = ">=0.18.0"
 "ruamel.yaml" = "^0.17.0"
 
 # dev
 pre-commit = { version = "*", optional = true }
 pytest = { version = ">=5.0", optional = true }
 pytest-cov = { version = "2.10.1", optional = true }
-backoff = { version = "*", optional = true }
+neptune = { version = ">=1.0.0", optional = true }
 
 [tool.poetry.extras]
 dev = [
     "pre-commit",
     "pytest",
     "pytest-cov",
-    "backoff",
+    "neptune",
 ]
 
 [tool.poetry]

diff --git a/src/kedro_neptune/__init__.py b/src/kedro_neptune/__init__.py
@@ -51,19 +51,19 @@
 from kedro_neptune.version import __version__
 
 try:
-    # neptune-client=0.9.0+ package structure
-    import neptune.new as neptune
-    from neptune.new.handler import Handler
-    from neptune.new.integrations.utils import join_paths
-    from neptune.new.types import File
-    from neptune.new.utils import stringify_unsupported
-except ImportError:
     # neptune-client>=1.0.0 package structure
     import neptune
     from neptune.handler import Handler
     from neptune.integrations.utils import join_paths
     from neptune.types import File
     from neptune.utils import stringify_unsupported
+except ImportError:
+    # neptune-client=0.9.0+ package structure
+    import neptune.new as neptune
+    from neptune.new.handler import Handler
+    from neptune.new.integrations.utils import join_paths
+    from neptune.new.types import File
+    from neptune.new.utils import stringify_unsupported
 
 INTEGRATION_VERSION_KEY = "source_code/integrations/kedro-neptune"
 

diff --git a/src/kedro_neptune/version.py b/src/kedro_neptune/version.py
@@ -1,6 +1,7 @@
 __all__ = ["__version__"]
 
 import sys
+from importlib.util import find_spec
 
 if sys.version_info >= (3, 8):
     from importlib.metadata import (
@@ -13,6 +14,16 @@
         version,
     )
 
+if not (find_spec("neptune") or find_spec("neptune-client")):
+    msg = """
+            The Neptune client library was not found.
+
+            Install the neptune package with
+                `pip install neptune`
+
+            Need help? -> https://docs.neptune.ai/setup/installation/"""
+    raise PackageNotFoundError(msg)
+
 try:
     __version__ = version("kedro-neptune")
 except PackageNotFoundError:

diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py
@@ -17,99 +17,108 @@
 
 import hashlib
 import os
+import time
 from ast import literal_eval
 from typing import (
     List,
     Optional,
 )
 
-import backoff
-from neptune.new import init_run
-from neptune.new.metadata_containers import Run
+try:
+    from neptune import (
+        Run,
+        init_run,
+    )
+except ImportError:
+    from neptune.new import init_run
+    from neptune.new.metadata_containers import Run
 
 
 # It may take some time to refresh cache
-@backoff.on_exception(backoff.expo, AssertionError, max_value=10)
+# @backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60)
 def assert_structure(travel_speed: int = 10000):
-    run = restore_run()
-
-    # Base run information
-    assert run.exists("kedro")
-    assert run.exists("kedro/catalog")
-    assert run.exists("kedro/nodes")
-    assert run.exists("kedro/kedro_command")
-    assert run.exists("kedro/run_params")
-    assert run.exists("kedro/structure")
-
-    # Data catalog
-    assert run.exists("kedro/catalog/datasets")
-    assert run.exists("kedro/catalog/files")
-    assert run.exists("kedro/catalog/parameters")
-
-    assert run.exists("kedro/catalog/datasets/planets")
-    assert run["kedro/catalog/datasets/planets"].fetch() == {
-        "filepath": f"{os.getcwd()}/data/planets/planets.csv",
-        "name": "planets",
-        "protocol": "file",
-        "save_args": {"index": False},
-        "type": "CSVDataSet",
-        "version": "None",
-    }
-
-    assert run.exists("kedro/catalog/datasets/planets@neptune")
-    assert run["kedro/catalog/datasets/planets@neptune"].fetch() == {
-        "extension": "csv",
-        "filepath": f"{os.getcwd()}/data/planets/planets.csv",
-        "name": "planets@neptune",
-        "protocol": "file",
-        "type": "NeptuneFileDataSet",
-        "version": "None",
-    }
-    assert run.exists("kedro/catalog/files/planets@neptune")
-    run["kedro/catalog/files/planets@neptune"].download("/tmp/file")
-    with open("/tmp/file", "rb") as handler:
-        file_content = handler.read()
-    assert hashlib.md5(file_content).hexdigest() == "af37712c8c80afc9690e4b70b7a590c5"
-
-    assert run.exists("kedro/catalog/files/logo")
-    run["kedro/catalog/files/logo"].download("/tmp/file")
-    with open("/tmp/file", "rb") as handler:
-        file_content = handler.read()
-    assert hashlib.md5(file_content).hexdigest() == "85d289d3ed3f321557b6c428b7b35a67"
-
-    assert run.exists("kedro/catalog/parameters/travel_speed")
-    assert run["kedro/catalog/parameters/travel_speed"].fetch() == travel_speed
-
-    # Nodes data
-    check_node_metadata(
-        run=run, node_namespace="kedro/nodes/distances", inputs=["planets"], outputs=["distances_to_planets"]
-    )
-    check_node_metadata(
-        run=run,
-        node_namespace="kedro/nodes/furthest",
-        inputs=["distances_to_planets"],
-        outputs=["furthest_planet_distance", "furthest_planet_name"],
-    )
-    check_node_metadata(run=run, node_namespace="kedro/nodes/judge_model", inputs=["neptune_run", "dataset"])
-    check_node_metadata(run=run, node_namespace="kedro/nodes/prepare_dataset", inputs=["planets"], outputs=["dataset"])
-    check_node_metadata(
-        run=run,
-        node_namespace="kedro/nodes/travel_time",
-        inputs=["furthest_planet_distance", "furthest_planet_name", "params:travel_speed"],
-        outputs=["travel_hours"],
-    )
-    assert run.exists("kedro/nodes/travel_time/parameters")
-    assert run.exists("kedro/nodes/travel_time/parameters/travel_speed")
-    assert run["kedro/nodes/travel_time/parameters/travel_speed"].fetch() == travel_speed
-
-    # User defined data
-    assert run.exists("furthest_planet")
-    assert run.exists("furthest_planet/name")
-    assert run.exists("furthest_planet/travel_days")
-    assert run.exists("furthest_planet/travel_hours")
-    assert run.exists("furthest_planet/travel_months")
-    assert run.exists("furthest_planet/travel_years")
-    assert run["furthest_planet/name"].fetch() == "NEPTUNE"
+    time.sleep(30)
+    with restore_run() as run:
+        run.sync(wait=True)
+        # Base run information
+        assert run.exists("kedro")
+        assert run.exists("kedro/catalog")
+        assert run.exists("kedro/nodes")
+        assert run.exists("kedro/kedro_command")
+        assert run.exists("kedro/run_params")
+        assert run.exists("kedro/structure")
+
+        # Data catalog
+        assert run.exists("kedro/catalog/datasets")
+        assert run.exists("kedro/catalog/files")
+        assert run.exists("kedro/catalog/parameters")
+
+        assert run.exists("kedro/catalog/datasets/planets")
+
+        assert run["kedro/catalog/datasets/planets"].fetch() == {
+            "filepath": f"{os.getcwd()}/data/planets/planets.csv",
+            "name": "planets",
+            "save_args": {"index": False},
+            "type": "CSVDataSet",
+            "version": "None",
+        }
+
+        assert run.exists("kedro/catalog/datasets/planets@neptune")
+        assert run["kedro/catalog/datasets/planets@neptune"].fetch() == {
+            "extension": "csv",
+            "filepath": f"{os.getcwd()}/data/planets/planets.csv",
+            "name": "planets@neptune",
+            "protocol": "file",
+            "type": "NeptuneFileDataSet",
+            "version": "None",
+        }
+        assert run.exists("kedro/catalog/files/planets@neptune")
+        run["kedro/catalog/files/planets@neptune"].download("/tmp/file")
+        with open("/tmp/file", "rb") as handler:
+            file_content = handler.read()
+        assert hashlib.md5(file_content).hexdigest() == "af37712c8c80afc9690e4b70b7a590c5"
+
+        assert run.exists("kedro/catalog/files/logo")
+        run["kedro/catalog/files/logo"].download("/tmp/file")
+        with open("/tmp/file", "rb") as handler:
+            file_content = handler.read()
+        assert hashlib.md5(file_content).hexdigest() == "85d289d3ed3f321557b6c428b7b35a67"
+
+        assert run.exists("kedro/catalog/parameters/travel_speed")
+        assert run["kedro/catalog/parameters/travel_speed"].fetch() == travel_speed
+
+        # Nodes data
+        check_node_metadata(
+            run=run, node_namespace="kedro/nodes/distances", inputs=["planets"], outputs=["distances_to_planets"]
+        )
+        check_node_metadata(
+            run=run,
+            node_namespace="kedro/nodes/furthest",
+            inputs=["distances_to_planets"],
+            outputs=["furthest_planet_distance", "furthest_planet_name"],
+        )
+        check_node_metadata(run=run, node_namespace="kedro/nodes/judge_model", inputs=["neptune_run", "dataset"])
+        check_node_metadata(
+            run=run, node_namespace="kedro/nodes/prepare_dataset", inputs=["planets"], outputs=["dataset"]
+        )
+        check_node_metadata(
+            run=run,
+            node_namespace="kedro/nodes/travel_time",
+            inputs=["furthest_planet_distance", "furthest_planet_name", "params:travel_speed"],
+            outputs=["travel_hours"],
+        )
+        assert run.exists("kedro/nodes/travel_time/parameters")
+        assert run.exists("kedro/nodes/travel_time/parameters/travel_speed")
+        assert run["kedro/nodes/travel_time/parameters/travel_speed"].fetch() == travel_speed
+
+        # User defined data
+        assert run.exists("furthest_planet")
+        assert run.exists("furthest_planet/name")
+        assert run.exists("furthest_planet/travel_days")
+        assert run.exists("furthest_planet/travel_hours")
+        assert run.exists("furthest_planet/travel_months")
+        assert run.exists("furthest_planet/travel_years")
+        assert run["furthest_planet/name"].fetch() == "NEPTUNE"
 
 
 def restore_run():