From f83bba0c10dcec49cf83bec658b2ce60e080f59f Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Tue, 18 Jul 2023 20:02:04 +0200 Subject: [PATCH] ci, build: use action to free space from GH runners and update dependencies (#261) * ci, build: use action to free space from GH runners and update dependencies The action will remove unused software from the GH runners to free some storage space in /dev/root. This will allow the heavy workload that are functional tests to run properly without storage limitations. At the same time, this commit is updating pyyaml (and other dependencies) as a workaround for https://github.com/juju/python-libjuju/issues/913 Fixes #250 --- .github/workflows/integrate.yaml | 17 ++++++++ charms/kfp-api/requirements.in | 3 ++ charms/kfp-api/requirements.txt | 40 ++++++++++--------- charms/kfp-persistence/requirements.in | 5 ++- charms/kfp-persistence/requirements.txt | 24 ++++++----- charms/kfp-profile-controller/requirements.in | 7 ++-- .../kfp-profile-controller/requirements.txt | 24 ++++++----- charms/kfp-schedwf/requirements.in | 3 ++ charms/kfp-schedwf/requirements.txt | 24 ++++++++--- charms/kfp-ui/requirements.in | 3 ++ charms/kfp-ui/requirements.txt | 24 ++++++----- charms/kfp-viewer/requirements.in | 3 ++ charms/kfp-viewer/requirements.txt | 24 ++++++++--- charms/kfp-viz/requirements.in | 3 ++ charms/kfp-viz/requirements.txt | 24 ++++++----- requirements-integration.in | 7 +++- requirements-integration.txt | 34 ++++++++-------- 17 files changed, 175 insertions(+), 94 deletions(-) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 7af00f5c..9e56cc57 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -103,8 +103,24 @@ jobs: runs-on: ubuntu-20.04 steps: + # This is a workaround for https://github.com/canonical/kfp-operators/issues/250 + # Ideally we'd use self-hosted runners, but this effort is still not stable + # This action will remove unused software (dotnet, haskell, android libs, codeql, + # and docker images) from the GH runner, which will liberate around 60 GB of storage + # distributed in 40GB for root and around 20 for a mnt point. + - name: Maximise GH runner space + uses: easimon/maximize-build-space@v7 + with: + root-reserve-mb: 40960 + remove-dotnet: 'true' + remove-haskell: 'true' + remove-android: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - name: Check out code uses: actions/checkout@v3 + - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: @@ -150,6 +166,7 @@ jobs: mkdir ~/kfp-operators-debug-logs juju debug-log --replay | tee ~/kfp-operators-debug-logs/juju-debug-log.log juju status | tee ~/kfp-operators-debug-logs/juju-status.log + bash -c "cp ~/.local/state/charmcraft/log/* ~/kfp-operators-debug-logs/" sg microk8s -c "microk8s.kubectl describe deployments -A | tee ~/kfp-operators-debug-logs/deployments.log" sg microk8s -c "microk8s.kubectl describe pods -A | tee ~/kfp-operators-debug-logs/pods.log" sg microk8s -c "microk8s.kubectl describe workloads -A | tee ~/kfp-operators-debug-logs/workloads.log" diff --git a/charms/kfp-api/requirements.in b/charms/kfp-api/requirements.in index 0ad8af7e..7a2bf1e0 100644 --- a/charms/kfp-api/requirements.in +++ b/charms/kfp-api/requirements.in @@ -1,4 +1,7 @@ charmed-kubeflow-chisme>=0.0.8 +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 lightkube lightkube-models>=1.25.4.4 ops>=2.2.0 diff --git a/charms/kfp-api/requirements.txt b/charms/kfp-api/requirements.txt index e39a1de1..7a7d220a 100644 --- a/charms/kfp-api/requirements.txt +++ b/charms/kfp-api/requirements.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # -anyio==3.7.0 +anyio==3.7.1 # via httpcore attrs==23.1.0 # via jsonschema @@ -14,16 +14,16 @@ certifi==2023.5.7 # httpx # requests charmed-kubeflow-chisme==0.0.11 - # via -r ./requirements.in -charset-normalizer==3.1.0 + # via -r requirements.in +charset-normalizer==3.2.0 # via requests deepdiff==6.2.1 # via charmed-kubeflow-chisme -exceptiongroup==1.1.1 +exceptiongroup==1.1.2 # via anyio h11==0.14.0 # via httpcore -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httpx==0.24.1 # via lightkube @@ -32,27 +32,29 @@ idna==3.4 # anyio # httpx # requests -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema jinja2==3.1.2 # via charmed-kubeflow-chisme jsonschema==4.17.3 - # via serialized-data-interface -lightkube==0.13.0 # via - # -r ./requirements.in + # -r requirements.in + # serialized-data-interface +lightkube==0.14.0 + # via + # -r requirements.in # charmed-kubeflow-chisme lightkube-models==1.27.1.4 # via - # -r ./requirements.in + # -r requirements.in # lightkube markupsafe==2.1.3 # via jinja2 oci-image==1.0.0 - # via -r ./requirements.in -ops==2.3.0 + # via -r requirements.in +ops==2.4.1 # via - # -r ./requirements.in + # -r requirements.in # charmed-kubeflow-chisme # serialized-data-interface ordered-set==4.1.0 @@ -61,7 +63,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema pyrsistent==0.19.3 # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # lightkube # ops @@ -73,7 +75,7 @@ ruamel-yaml==0.17.32 ruamel-yaml-clib==0.2.7 # via ruamel-yaml serialized-data-interface==0.6.0 - # via -r ./requirements.in + # via -r requirements.in sniffio==1.3.0 # via # anyio @@ -81,11 +83,11 @@ sniffio==1.3.0 # httpx tenacity==8.2.2 # via - # -r ./requirements.in + # -r requirements.in # charmed-kubeflow-chisme urllib3==2.0.3 # via requests -websocket-client==1.6.0 +websocket-client==1.6.1 # via ops -zipp==3.15.0 +zipp==3.16.2 # via importlib-resources diff --git a/charms/kfp-persistence/requirements.in b/charms/kfp-persistence/requirements.in index 287f7349..67c7c782 100644 --- a/charms/kfp-persistence/requirements.in +++ b/charms/kfp-persistence/requirements.in @@ -1,3 +1,6 @@ -ops==1.2.0 +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 +ops oci-image==1.0.0 serialized-data-interface<0.7 diff --git a/charms/kfp-persistence/requirements.txt b/charms/kfp-persistence/requirements.txt index dfb5c411..6bb6f130 100644 --- a/charms/kfp-persistence/requirements.txt +++ b/charms/kfp-persistence/requirements.txt @@ -2,39 +2,43 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # attrs==23.1.0 # via jsonschema certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests idna==3.4 # via requests -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema jsonschema==4.17.3 - # via serialized-data-interface + # via + # -r requirements.in + # serialized-data-interface oci-image==1.0.0 - # via -r ./requirements.in -ops==1.2.0 + # via -r requirements.in +ops==2.4.1 # via - # -r ./requirements.in + # -r requirements.in # serialized-data-interface pkgutil-resolve-name==1.3.10 # via jsonschema pyrsistent==0.19.3 # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # ops # serialized-data-interface requests==2.31.0 # via serialized-data-interface serialized-data-interface==0.6.0 - # via -r ./requirements.in + # via -r requirements.in urllib3==2.0.3 # via requests -zipp==3.15.0 +websocket-client==1.6.1 + # via ops +zipp==3.16.2 # via importlib-resources diff --git a/charms/kfp-profile-controller/requirements.in b/charms/kfp-profile-controller/requirements.in index b20943c4..ef3c7451 100644 --- a/charms/kfp-profile-controller/requirements.in +++ b/charms/kfp-profile-controller/requirements.in @@ -1,5 +1,6 @@ -ops==1.2.0 -# Unittest fails if not pinned. unknown config option: 'service-port' +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 +ops oci-image serialized-data-interface<0.7 -# Unittest fails if not pinned. unknown config option: 'service-port' diff --git a/charms/kfp-profile-controller/requirements.txt b/charms/kfp-profile-controller/requirements.txt index dfb5c411..6bb6f130 100644 --- a/charms/kfp-profile-controller/requirements.txt +++ b/charms/kfp-profile-controller/requirements.txt @@ -2,39 +2,43 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # attrs==23.1.0 # via jsonschema certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests idna==3.4 # via requests -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema jsonschema==4.17.3 - # via serialized-data-interface + # via + # -r requirements.in + # serialized-data-interface oci-image==1.0.0 - # via -r ./requirements.in -ops==1.2.0 + # via -r requirements.in +ops==2.4.1 # via - # -r ./requirements.in + # -r requirements.in # serialized-data-interface pkgutil-resolve-name==1.3.10 # via jsonschema pyrsistent==0.19.3 # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # ops # serialized-data-interface requests==2.31.0 # via serialized-data-interface serialized-data-interface==0.6.0 - # via -r ./requirements.in + # via -r requirements.in urllib3==2.0.3 # via requests -zipp==3.15.0 +websocket-client==1.6.1 + # via ops +zipp==3.16.2 # via importlib-resources diff --git a/charms/kfp-schedwf/requirements.in b/charms/kfp-schedwf/requirements.in index 7a994037..a2b5f271 100644 --- a/charms/kfp-schedwf/requirements.in +++ b/charms/kfp-schedwf/requirements.in @@ -1,2 +1,5 @@ +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 ops oci-image diff --git a/charms/kfp-schedwf/requirements.txt b/charms/kfp-schedwf/requirements.txt index 040893e5..3e692755 100644 --- a/charms/kfp-schedwf/requirements.txt +++ b/charms/kfp-schedwf/requirements.txt @@ -2,13 +2,25 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # +attrs==23.1.0 + # via jsonschema +importlib-resources==6.0.0 + # via jsonschema +jsonschema==4.17.3 + # via -r requirements.in oci-image==1.0.0 - # via -r ./requirements.in -ops==2.3.0 - # via -r ./requirements.in -pyyaml==6.0 + # via -r requirements.in +ops==2.4.1 + # via -r requirements.in +pkgutil-resolve-name==1.3.10 + # via jsonschema +pyrsistent==0.19.3 + # via jsonschema +pyyaml==6.0.1 # via ops -websocket-client==1.6.0 +websocket-client==1.6.1 # via ops +zipp==3.16.2 + # via importlib-resources diff --git a/charms/kfp-ui/requirements.in b/charms/kfp-ui/requirements.in index 729f752a..ef3c7451 100644 --- a/charms/kfp-ui/requirements.in +++ b/charms/kfp-ui/requirements.in @@ -1,3 +1,6 @@ +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 ops oci-image serialized-data-interface<0.7 diff --git a/charms/kfp-ui/requirements.txt b/charms/kfp-ui/requirements.txt index ae4ade09..6bb6f130 100644 --- a/charms/kfp-ui/requirements.txt +++ b/charms/kfp-ui/requirements.txt @@ -2,41 +2,43 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # attrs==23.1.0 # via jsonschema certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests idna==3.4 # via requests -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema jsonschema==4.17.3 - # via serialized-data-interface + # via + # -r requirements.in + # serialized-data-interface oci-image==1.0.0 - # via -r ./requirements.in -ops==2.3.0 + # via -r requirements.in +ops==2.4.1 # via - # -r ./requirements.in + # -r requirements.in # serialized-data-interface pkgutil-resolve-name==1.3.10 # via jsonschema pyrsistent==0.19.3 # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # ops # serialized-data-interface requests==2.31.0 # via serialized-data-interface serialized-data-interface==0.6.0 - # via -r ./requirements.in + # via -r requirements.in urllib3==2.0.3 # via requests -websocket-client==1.6.0 +websocket-client==1.6.1 # via ops -zipp==3.15.0 +zipp==3.16.2 # via importlib-resources diff --git a/charms/kfp-viewer/requirements.in b/charms/kfp-viewer/requirements.in index 7a994037..a2b5f271 100644 --- a/charms/kfp-viewer/requirements.in +++ b/charms/kfp-viewer/requirements.in @@ -1,2 +1,5 @@ +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 ops oci-image diff --git a/charms/kfp-viewer/requirements.txt b/charms/kfp-viewer/requirements.txt index 040893e5..3e692755 100644 --- a/charms/kfp-viewer/requirements.txt +++ b/charms/kfp-viewer/requirements.txt @@ -2,13 +2,25 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # +attrs==23.1.0 + # via jsonschema +importlib-resources==6.0.0 + # via jsonschema +jsonschema==4.17.3 + # via -r requirements.in oci-image==1.0.0 - # via -r ./requirements.in -ops==2.3.0 - # via -r ./requirements.in -pyyaml==6.0 + # via -r requirements.in +ops==2.4.1 + # via -r requirements.in +pkgutil-resolve-name==1.3.10 + # via jsonschema +pyrsistent==0.19.3 + # via jsonschema +pyyaml==6.0.1 # via ops -websocket-client==1.6.0 +websocket-client==1.6.1 # via ops +zipp==3.16.2 + # via importlib-resources diff --git a/charms/kfp-viz/requirements.in b/charms/kfp-viz/requirements.in index 8cf0f9b1..2a11e321 100644 --- a/charms/kfp-viz/requirements.in +++ b/charms/kfp-viz/requirements.in @@ -1,3 +1,6 @@ +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 ops oci-image serialized-data-interface diff --git a/charms/kfp-viz/requirements.txt b/charms/kfp-viz/requirements.txt index ae4ade09..6bb6f130 100644 --- a/charms/kfp-viz/requirements.txt +++ b/charms/kfp-viz/requirements.txt @@ -2,41 +2,43 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --resolver=backtracking ./requirements.in +# pip-compile --resolver=backtracking requirements.in # attrs==23.1.0 # via jsonschema certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests idna==3.4 # via requests -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema jsonschema==4.17.3 - # via serialized-data-interface + # via + # -r requirements.in + # serialized-data-interface oci-image==1.0.0 - # via -r ./requirements.in -ops==2.3.0 + # via -r requirements.in +ops==2.4.1 # via - # -r ./requirements.in + # -r requirements.in # serialized-data-interface pkgutil-resolve-name==1.3.10 # via jsonschema pyrsistent==0.19.3 # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # ops # serialized-data-interface requests==2.31.0 # via serialized-data-interface serialized-data-interface==0.6.0 - # via -r ./requirements.in + # via -r requirements.in urllib3==2.0.3 # via requests -websocket-client==1.6.0 +websocket-client==1.6.1 # via ops -zipp==3.15.0 +zipp==3.16.2 # via importlib-resources diff --git a/requirements-integration.in b/requirements-integration.in index 5f4f4c51..a1879092 100644 --- a/requirements-integration.in +++ b/requirements-integration.in @@ -1,7 +1,12 @@ aiohttp +# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117 +# and https://github.com/python-jsonschema/jsonschema/issues/1114 +jsonschema<4.18 # Pinning to <3.0 to ensure compatibility with the 2.9 controller version # Note: 3.0 is not being maintained anymore -juju<3.0 +# FIXME: This is a workaround for https://github.com/juju/python-libjuju/issues/913 +# please pin to a released python-libjuju ASAP. +juju @ git+https://github.com/DnPlas/python-libjuju@dnplas-pyyaml-6 # Need this version of kfp to work with kfp 2.0.0-alpha.7 kfp==1.8.22 lightkube diff --git a/requirements-integration.txt b/requirements-integration.txt index 9cc66a81..4165d40f 100644 --- a/requirements-integration.txt +++ b/requirements-integration.txt @@ -10,7 +10,7 @@ aiohttp==3.8.4 # via -r requirements-integration.in aiosignal==1.3.1 # via aiohttp -anyio==3.7.0 +anyio==3.7.1 # via httpcore asttokens==2.2.1 # via stack-data @@ -37,17 +37,17 @@ cffi==1.15.1 # via # cryptography # pynacl -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.5 # via # kfp # typer cloudpickle==2.2.1 # via kfp -cryptography==41.0.1 +cryptography==41.0.2 # via paramiko decorator==5.1.1 # via @@ -65,7 +65,7 @@ executing==1.2.0 # via stack-data fire==0.5.0 # via kfp -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -77,7 +77,7 @@ google-api-core==2.11.1 # kfp google-api-python-client==1.12.11 # via kfp -google-auth==2.21.0 +google-auth==2.22.0 # via # google-api-core # google-api-python-client @@ -88,7 +88,7 @@ google-auth==2.21.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage google-cloud-storage==2.10.0 # via kfp @@ -100,7 +100,7 @@ googleapis-common-protos==1.59.1 # via google-api-core h11==0.14.0 # via httpcore -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via @@ -114,7 +114,7 @@ idna==3.4 # httpx # requests # yarl -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via jsonschema iniconfig==2.0.0 # via pytest @@ -127,8 +127,10 @@ jedi==0.18.2 jinja2==3.1.2 # via pytest-operator jsonschema==4.17.3 - # via kfp -juju==2.9.42.4 + # via + # -r requirements-integration.in + # kfp +juju @ git+https://github.com/DnPlas/python-libjuju@dnplas-pyyaml-6 # via # -r requirements-integration.in # pytest-operator @@ -144,7 +146,7 @@ kubernetes==25.3.0 # via # juju # kfp -lightkube==0.13.0 +lightkube==0.14.0 # via -r requirements-integration.in lightkube-models==1.27.1.4 # via lightkube @@ -200,7 +202,7 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==1.10.10 +pydantic==1.10.11 # via kfp pygments==2.15.1 # via ipython @@ -224,7 +226,7 @@ pytest==7.4.0 # -r requirements-integration.in # pytest-asyncio # pytest-operator -pytest-asyncio==0.21.0 +pytest-asyncio==0.21.1 # via pytest-operator pytest-operator==0.28.0 # via -r requirements-integration.in @@ -234,7 +236,7 @@ python-dateutil==2.8.2 # kubernetes pytz==2023.3 # via pyrfc3339 -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements-integration.in # juju @@ -332,7 +334,7 @@ wrapt==1.15.0 # via deprecated yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via importlib-resources # The following packages are considered to be unsafe in a requirements file: