From ff50a868440cfd19fb555717c6486ac102ed6bfc Mon Sep 17 00:00:00 2001 From: Manos Vlassis <57320708+mvlassis@users.noreply.github.com> Date: Wed, 24 Jul 2024 17:42:10 +0300 Subject: [PATCH] Update scipy image and requirements for Charmed Kubeflow 1.9 (#92) * Update image, requirements.in, and requirements.txt * Update version of pydantic in kserve notebook * Update comments in requirements.in * Update image to v1.9.0 --- driver/test_kubeflow_workloads.py | 2 +- tests/notebooks/e2e-wine/requirements.in | 4 +- tests/notebooks/e2e-wine/requirements.txt | 78 ++----------------- tests/notebooks/katib/requirements.in | 4 +- tests/notebooks/katib/requirements.txt | 4 +- tests/notebooks/kserve/requirements.in | 6 +- tests/notebooks/kserve/requirements.txt | 64 +++++++-------- tests/notebooks/mlflow-kserve/requirements.in | 4 +- .../notebooks/mlflow-kserve/requirements.txt | 53 ++++--------- 9 files changed, 60 insertions(+), 159 deletions(-) diff --git a/driver/test_kubeflow_workloads.py b/driver/test_kubeflow_workloads.py index d203c9b..203a995 100644 --- a/driver/test_kubeflow_workloads.py +++ b/driver/test_kubeflow_workloads.py @@ -20,7 +20,7 @@ TESTS_LOCAL_RUN = eval(os.environ.get("LOCAL")) TESTS_LOCAL_DIR = os.path.abspath(Path("tests")) -TESTS_IMAGE = "kubeflownotebookswg/jupyter-scipy:v1.8.0" +TESTS_IMAGE = "kubeflownotebookswg/jupyter-scipy:v1.9.0" NAMESPACE = "test-kubeflow" PROFILE_RESOURCE = create_global_resource( diff --git a/tests/notebooks/e2e-wine/requirements.in b/tests/notebooks/e2e-wine/requirements.in index 7ec99c2..0ffea69 100644 --- a/tests/notebooks/e2e-wine/requirements.in +++ b/tests/notebooks/e2e-wine/requirements.in @@ -1,7 +1,5 @@ boto3 -# pin to the latest <2.0 version to ensure compatibility -# with the KFP API server version deployed in CKF 1.7 -kfp==1.8.22 +kfp==2.2.0 # pin to <7.0 to avoid breaking changes in sdk minio<7.0 # pin the client to match the version of the deployed MLflow server diff --git a/tests/notebooks/e2e-wine/requirements.txt b/tests/notebooks/e2e-wine/requirements.txt index c834b94..b2656dd 100644 --- a/tests/notebooks/e2e-wine/requirements.txt +++ b/tests/notebooks/e2e-wine/requirements.txt @@ -4,14 +4,8 @@ # # pip-compile requirements.in # -absl-py==1.4.0 - # via kfp alembic==1.13.2 # via mlflow -attrs==23.2.0 - # via - # jsonschema - # referencing blinker==1.8.2 # via flask boto3==1.34.143 @@ -36,10 +30,8 @@ click==8.1.7 # flask # kfp # mlflow - # typer cloudpickle==2.2.1 # via - # kfp # mlflow # shap configparser==7.0.0 @@ -50,16 +42,12 @@ cycler==0.12.1 # via matplotlib databricks-cli==0.18.0 # via mlflow -deprecated==1.2.14 - # via kfp docker==6.1.3 # via mlflow docstring-parser==0.16 # via kfp entrypoints==0.4 # via mlflow -fire==0.6.0 - # via kfp flask==2.3.3 # via mlflow fonttools==4.53.1 @@ -70,23 +58,16 @@ gitpython==3.1.43 # via mlflow google-api-core==2.19.1 # via - # google-api-python-client # google-cloud-core # google-cloud-storage # kfp -google-api-python-client==1.12.11 - # via kfp google-auth==2.32.0 # via # google-api-core - # google-api-python-client - # google-auth-httplib2 # google-cloud-core # google-cloud-storage # kfp # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client google-cloud-core==2.4.1 # via google-cloud-storage google-cloud-storage==2.17.0 @@ -103,10 +84,6 @@ greenlet==3.0.3 # via sqlalchemy gunicorn==20.1.0 # via mlflow -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 idna==3.7 # via requests importlib-metadata==5.2.0 @@ -123,15 +100,11 @@ jmespath==1.0.1 # botocore joblib==1.4.2 # via scikit-learn -jsonschema==4.23.0 - # via kfp -jsonschema-specifications==2023.12.1 - # via jsonschema -kfp==1.8.22 +kfp==2.2.0 # via -r requirements.in -kfp-pipeline-spec==0.1.16 +kfp-pipeline-spec==0.2.2 # via kfp -kfp-server-api==1.8.5 +kfp-server-api==2.0.5 # via kfp kiwisolver==1.4.5 # via matplotlib @@ -143,8 +116,6 @@ mako==1.3.5 # via alembic markdown==3.6 # via mlflow -markdown-it-py==3.0.0 - # via rich markupsafe==2.1.5 # via # jinja2 @@ -152,8 +123,6 @@ markupsafe==2.1.5 # werkzeug matplotlib==3.9.1 # via mlflow -mdurl==0.1.2 - # via markdown-it-py minio==6.0.2 # via -r requirements.in mlflow==2.1.1 @@ -208,16 +177,10 @@ pyasn1==0.6.0 # rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==1.10.17 - # via kfp -pygments==2.18.0 - # via rich pyjwt==2.8.0 # via databricks-cli pyparsing==3.1.2 - # via - # httplib2 - # matplotlib + # via matplotlib python-dateutil==2.9.0.post0 # via # botocore @@ -238,10 +201,6 @@ pyyaml==6.0.1 # mlflow querystring-parser==1.2.4 # via mlflow -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications requests==2.32.3 # via # -r requirements.in @@ -257,12 +216,6 @@ requests-oauthlib==2.0.0 # via kubernetes requests-toolbelt==0.10.1 # via kfp -rich==13.7.1 - # via typer -rpds-py==0.19.0 - # via - # jsonschema - # referencing rsa==4.9 # via google-auth s3transfer==0.10.2 @@ -279,13 +232,9 @@ scipy==1.14.0 # shap shap==0.46.0 # via mlflow -shellingham==1.5.4 - # via typer six==1.16.0 # via # databricks-cli - # fire - # google-api-python-client # kfp-server-api # kubernetes # python-dateutil @@ -300,31 +249,18 @@ sqlalchemy==1.4.52 # mlflow sqlparse==0.5.0 # via mlflow -strip-hints==0.1.10 - # via kfp tabulate==0.9.0 # via # databricks-cli # kfp tenacity==8.5.0 # via -r requirements.in -termcolor==2.4.0 - # via fire threadpoolctl==3.5.0 # via scikit-learn tqdm==4.66.4 # via shap -typer==0.12.3 - # via kfp typing-extensions==4.12.2 - # via - # alembic - # pydantic - # typer -uritemplate==3.0.1 - # via - # google-api-python-client - # kfp + # via alembic urllib3==1.26.19 # via # botocore @@ -341,10 +277,6 @@ websocket-client==1.8.0 # kubernetes werkzeug==3.0.3 # via flask -wheel==0.43.0 - # via strip-hints -wrapt==1.16.0 - # via deprecated zipp==3.19.2 # via importlib-metadata diff --git a/tests/notebooks/katib/requirements.in b/tests/notebooks/katib/requirements.in index 68c21dc..a11b5ab 100644 --- a/tests/notebooks/katib/requirements.in +++ b/tests/notebooks/katib/requirements.in @@ -1,3 +1,3 @@ -# Pinning for katib sdk in 1.8/stable -kubeflow-katib==0.16.0 +# Pinning for katib sdk in 1.9/stable +kubeflow-katib==0.17.0 tenacity diff --git a/tests/notebooks/katib/requirements.txt b/tests/notebooks/katib/requirements.txt index 0dce725..548d455 100644 --- a/tests/notebooks/katib/requirements.txt +++ b/tests/notebooks/katib/requirements.txt @@ -19,7 +19,7 @@ grpcio==1.64.1 # via kubeflow-katib idna==3.7 # via requests -kubeflow-katib==0.16.0 +kubeflow-katib==0.17.0 # via -r requirements.in kubernetes==30.1.0 # via kubeflow-katib @@ -27,7 +27,7 @@ oauthlib==3.2.2 # via # kubernetes # requests-oauthlib -protobuf==3.20.3 +protobuf==4.25.3 # via kubeflow-katib pyasn1==0.6.0 # via diff --git a/tests/notebooks/kserve/requirements.in b/tests/notebooks/kserve/requirements.in index 4157ff5..f20ce52 100644 --- a/tests/notebooks/kserve/requirements.in +++ b/tests/notebooks/kserve/requirements.in @@ -1,5 +1,7 @@ -# Pinning for kserve sdk in 1.8/stable -kserve==0.11.2 +# Pinning for kserve sdk in 1.9 +kserve==0.13.0 +# Pining because kserve 0.13 needs to import RootModel which only exists in >= 2.0 +pydantic>=2.0 kubernetes requests tenacity diff --git a/tests/notebooks/kserve/requirements.txt b/tests/notebooks/kserve/requirements.txt index 9b94f92..0eaf7ff 100644 --- a/tests/notebooks/kserve/requirements.txt +++ b/tests/notebooks/kserve/requirements.txt @@ -10,25 +10,22 @@ aiohttp==3.9.5 # ray aiohttp-cors==0.7.0 # via ray -aiorwlock==1.4.0 - # via ray aiosignal==1.3.1 # via # aiohttp # ray +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via - # httpcore + # httpx # starlette # watchfiles attrs==23.2.0 # via # aiohttp # jsonschema - # ray # referencing -blessed==1.20.0 - # via gpustat cachetools==5.3.3 # via google-auth certifi==2024.7.4 @@ -51,7 +48,7 @@ deprecation==2.1.0 # via cloudevents distlib==0.3.8 # via virtualenv -fastapi==0.95.2 +fastapi==0.109.2 # via # kserve # ray @@ -72,8 +69,6 @@ google-auth==2.32.0 # kubernetes googleapis-common-protos==1.63.2 # via google-api-core -gpustat==1.1.1 - # via ray grpcio==1.51.3 # via # kserve @@ -82,23 +77,23 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.16.3 +httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.23.3 +httpx==0.26.0 # via kserve idna==3.7 # via # anyio + # httpx # requests - # rfc3986 # yarl jsonschema==4.23.0 # via ray jsonschema-specifications==2023.12.1 # via jsonschema -kserve==0.11.2 +kserve==0.13.0 # via -r requirements.in kubernetes==30.1.0 # via @@ -114,10 +109,6 @@ numpy==1.26.4 # via # kserve # pandas - # ray - # tritonclient -nvidia-ml-py==12.555.43 - # via gpustat oauthlib==3.2.2 # via # kubernetes @@ -136,7 +127,7 @@ pandas==2.2.2 # via kserve platformdirs==3.11.0 # via virtualenv -prometheus-client==0.13.1 +prometheus-client==0.20.0 # via # kserve # ray @@ -150,9 +141,7 @@ protobuf==3.20.3 # proto-plus # ray psutil==5.9.8 - # via - # gpustat - # kserve + # via kserve py-spy==0.3.14 # via ray pyasn1==0.6.0 @@ -161,10 +150,14 @@ pyasn1==0.6.0 # rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==1.10.17 +pydantic==2.8.2 # via + # -r requirements.in # fastapi + # kserve # ray +pydantic-core==2.20.1 + # via pydantic python-dateutil==2.9.0.post0 # via # kserve @@ -172,16 +165,15 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via uvicorn -python-rapidjson==1.18 - # via tritonclient pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # kserve # kubernetes # ray # uvicorn -ray[serve]==2.4.0 +ray[serve]==2.10.0 # via kserve referencing==0.35.1 # via @@ -196,8 +188,6 @@ requests==2.32.3 # requests-oauthlib requests-oauthlib==2.0.0 # via kubernetes -rfc3986[idna2008]==1.5.0 - # via httpx rpds-py==0.19.0 # via # jsonschema @@ -206,7 +196,6 @@ rsa==4.9 # via google-auth six==1.16.0 # via - # blessed # kserve # kubernetes # opencensus @@ -216,9 +205,8 @@ smart-open==7.0.4 sniffio==1.3.1 # via # anyio - # httpcore # httpx -starlette==0.27.0 +starlette==0.36.3 # via # fastapi # ray @@ -228,18 +216,18 @@ tenacity==8.5.0 # via -r requirements.in timing-asgi==0.3.1 # via kserve -tritonclient==2.47.0 - # via kserve typing-extensions==4.12.2 - # via pydantic + # via + # fastapi + # pydantic + # pydantic-core tzdata==2024.1 # via pandas urllib3==2.2.2 # via # kubernetes # requests - # tritonclient -uvicorn[standard]==0.19.0 +uvicorn[standard]==0.21.1 # via # kserve # ray @@ -248,9 +236,9 @@ uvloop==0.19.0 virtualenv==20.21.0 # via ray watchfiles==0.22.0 - # via uvicorn -wcwidth==0.2.13 - # via blessed + # via + # ray + # uvicorn websocket-client==1.8.0 # via kubernetes websockets==12.0 diff --git a/tests/notebooks/mlflow-kserve/requirements.in b/tests/notebooks/mlflow-kserve/requirements.in index c5714ee..2dadf84 100644 --- a/tests/notebooks/mlflow-kserve/requirements.in +++ b/tests/notebooks/mlflow-kserve/requirements.in @@ -9,6 +9,6 @@ requests # pin to ensure compatibility with the installed mlflow client scikit-learn<1.2 tenacity -# Pinning for kserve sdk in 1.8/stable -kserve==0.11.2 +# Pinning for kserve sdk in 1.9 +kserve==0.13.0 kubernetes diff --git a/tests/notebooks/mlflow-kserve/requirements.txt b/tests/notebooks/mlflow-kserve/requirements.txt index ee75226..a1a8928 100644 --- a/tests/notebooks/mlflow-kserve/requirements.txt +++ b/tests/notebooks/mlflow-kserve/requirements.txt @@ -10,8 +10,6 @@ aiohttp==3.9.5 # ray aiohttp-cors==0.7.0 # via ray -aiorwlock==1.4.0 - # via ray aiosignal==1.3.1 # via # aiohttp @@ -20,17 +18,14 @@ alembic==1.13.2 # via mlflow anyio==4.4.0 # via - # httpcore + # httpx # starlette # watchfiles attrs==23.2.0 # via # aiohttp # jsonschema - # ray # referencing -blessed==1.20.0 - # via gpustat blinker==1.8.2 # via flask boto3==1.34.143 @@ -81,7 +76,7 @@ docker==6.1.3 # via mlflow entrypoints==0.4 # via mlflow -fastapi==0.95.2 +fastapi==0.109.2 # via # kserve # ray @@ -110,8 +105,6 @@ google-auth==2.32.0 # kubernetes googleapis-common-protos==1.63.2 # via google-api-core -gpustat==1.1.1 - # via ray greenlet==3.0.3 # via sqlalchemy grpcio==1.51.3 @@ -124,17 +117,17 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.16.3 +httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.23.3 +httpx==0.26.0 # via kserve idna==3.7 # via # anyio + # httpx # requests - # rfc3986 # yarl importlib-metadata==5.2.0 # via mlflow @@ -156,7 +149,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema kiwisolver==1.4.5 # via matplotlib -kserve==0.11.2 +kserve==0.13.0 # via -r requirements.in kubernetes==30.1.0 # via @@ -197,13 +190,9 @@ numpy==1.26.4 # numba # pandas # pyarrow - # ray # scikit-learn # scipy # shap - # tritonclient -nvidia-ml-py==12.555.43 - # via gpustat oauthlib==3.2.2 # via # databricks-cli @@ -232,7 +221,7 @@ pillow==10.4.0 # via matplotlib platformdirs==3.11.0 # via virtualenv -prometheus-client==0.13.1 +prometheus-client==0.20.0 # via # kserve # ray @@ -247,9 +236,7 @@ protobuf==3.20.3 # proto-plus # ray psutil==5.9.8 - # via - # gpustat - # kserve + # via kserve py-spy==0.3.14 # via ray pyarrow==10.0.1 @@ -265,6 +252,7 @@ pyasn1-modules==0.4.0 pydantic==1.10.17 # via # fastapi + # kserve # ray pyjwt==2.8.0 # via databricks-cli @@ -280,8 +268,6 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via uvicorn -python-rapidjson==1.18 - # via tritonclient pytz==2022.7.1 # via # minio @@ -289,13 +275,14 @@ pytz==2022.7.1 # pandas pyyaml==6.0.1 # via + # kserve # kubernetes # mlflow # ray # uvicorn querystring-parser==1.2.4 # via mlflow -ray[serve]==2.4.0 +ray[serve]==2.10.0 # via kserve referencing==0.35.1 # via @@ -313,8 +300,6 @@ requests==2.32.3 # requests-oauthlib requests-oauthlib==2.0.0 # via kubernetes -rfc3986[idna2008]==1.5.0 - # via httpx rpds-py==0.19.0 # via # jsonschema @@ -337,7 +322,6 @@ shap==0.46.0 # via mlflow six==1.16.0 # via - # blessed # databricks-cli # kserve # kubernetes @@ -353,7 +337,6 @@ smmap==5.0.1 sniffio==1.3.1 # via # anyio - # httpcore # httpx sqlalchemy==1.4.52 # via @@ -361,7 +344,7 @@ sqlalchemy==1.4.52 # mlflow sqlparse==0.5.0 # via mlflow -starlette==0.27.0 +starlette==0.36.3 # via # fastapi # ray @@ -377,11 +360,10 @@ timing-asgi==0.3.1 # via kserve tqdm==4.66.4 # via shap -tritonclient==2.47.0 - # via kserve typing-extensions==4.12.2 # via # alembic + # fastapi # pydantic urllib3==2.2.2 # via @@ -391,8 +373,7 @@ urllib3==2.2.2 # kubernetes # minio # requests - # tritonclient -uvicorn[standard]==0.19.0 +uvicorn[standard]==0.21.1 # via # kserve # ray @@ -401,9 +382,9 @@ uvloop==0.19.0 virtualenv==20.21.0 # via ray watchfiles==0.22.0 - # via uvicorn -wcwidth==0.2.13 - # via blessed + # via + # ray + # uvicorn websocket-client==1.8.0 # via # docker