ci, build: use action to free space from GH runners and update depend…

…encies (#261)

* ci, build: use action to free space from GH runners and update dependencies

The action will remove unused software from the GH runners to free
some storage space in /dev/root. This will allow the heavy workload
that are functional tests to run properly without storage limitations.

At the same time, this commit is updating pyyaml (and other dependencies)
as a workaround for juju/python-libjuju#913

Fixes #250

.github/workflows/integrate.yaml

@@ -103,8 +103,24 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
+      # This is a workaround for https://github.com/canonical/kfp-operators/issues/250
+      # Ideally we'd use self-hosted runners, but this effort is still not stable
+      # This action will remove unused software (dotnet, haskell, android libs, codeql,
+      # and docker images) from the GH runner, which will liberate around 60 GB of storage
+      # distributed in 40GB for root and around 20 for a mnt point.
+      - name: Maximise GH runner space
+        uses: easimon/maximize-build-space@v7
+        with:
+          root-reserve-mb: 40960
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+
       - name: Check out code
         uses: actions/checkout@v3
+
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
@@ -150,6 +166,7 @@ jobs:
           mkdir ~/kfp-operators-debug-logs
           juju debug-log --replay | tee ~/kfp-operators-debug-logs/juju-debug-log.log
           juju status | tee ~/kfp-operators-debug-logs/juju-status.log
+          bash -c "cp ~/.local/state/charmcraft/log/* ~/kfp-operators-debug-logs/"
           sg microk8s -c "microk8s.kubectl describe deployments -A | tee ~/kfp-operators-debug-logs/deployments.log"
           sg microk8s -c "microk8s.kubectl describe pods -A | tee ~/kfp-operators-debug-logs/pods.log"
           sg microk8s -c "microk8s.kubectl describe workloads -A | tee ~/kfp-operators-debug-logs/workloads.log"

charms/kfp-api/requirements.in

@@ -1,4 +1,7 @@
 charmed-kubeflow-chisme>=0.0.8
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
 lightkube
 lightkube-models>=1.25.4.4
 ops>=2.2.0

charms/kfp-api/requirements.txt

@@ -2,9 +2,9 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --resolver=backtracking ./requirements.in
+#    pip-compile --resolver=backtracking requirements.in
 #
-anyio==3.7.0
+anyio==3.7.1
     # via httpcore
 attrs==23.1.0
     # via jsonschema
@@ -14,16 +14,16 @@ certifi==2023.5.7
     #   httpx
     #   requests
 charmed-kubeflow-chisme==0.0.11
-    # via -r ./requirements.in
-charset-normalizer==3.1.0
+    # via -r requirements.in
+charset-normalizer==3.2.0
     # via requests
 deepdiff==6.2.1
     # via charmed-kubeflow-chisme
-exceptiongroup==1.1.1
+exceptiongroup==1.1.2
     # via anyio
 h11==0.14.0
     # via httpcore
-httpcore==0.17.2
+httpcore==0.17.3
     # via httpx
 httpx==0.24.1
     # via lightkube
@@ -32,27 +32,29 @@ idna==3.4
     #   anyio
     #   httpx
     #   requests
-importlib-resources==5.12.0
+importlib-resources==6.0.0
     # via jsonschema
 jinja2==3.1.2
     # via charmed-kubeflow-chisme
 jsonschema==4.17.3
-    # via serialized-data-interface
-lightkube==0.13.0
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
+    #   serialized-data-interface
+lightkube==0.14.0
+    # via
+    #   -r requirements.in
     #   charmed-kubeflow-chisme
 lightkube-models==1.27.1.4
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   lightkube
 markupsafe==2.1.3
     # via jinja2
 oci-image==1.0.0
-    # via -r ./requirements.in
-ops==2.3.0
+    # via -r requirements.in
+ops==2.4.1
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   charmed-kubeflow-chisme
     #   serialized-data-interface
 ordered-set==4.1.0
@@ -61,7 +63,7 @@ pkgutil-resolve-name==1.3.10
     # via jsonschema
 pyrsistent==0.19.3
     # via jsonschema
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   lightkube
     #   ops
@@ -73,19 +75,19 @@ ruamel-yaml==0.17.32
 ruamel-yaml-clib==0.2.7
     # via ruamel-yaml
 serialized-data-interface==0.6.0
-    # via -r ./requirements.in
+    # via -r requirements.in
 sniffio==1.3.0
     # via
     #   anyio
     #   httpcore
     #   httpx
 tenacity==8.2.2
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   charmed-kubeflow-chisme
 urllib3==2.0.3
     # via requests
-websocket-client==1.6.0
+websocket-client==1.6.1
     # via ops
-zipp==3.15.0
+zipp==3.16.2
     # via importlib-resources

charms/kfp-persistence/requirements.in

@@ -1,3 +1,6 @@
-ops==1.2.0
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
+ops
 oci-image==1.0.0
 serialized-data-interface<0.7

charms/kfp-persistence/requirements.txt

@@ -2,39 +2,43 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --resolver=backtracking ./requirements.in
+#    pip-compile --resolver=backtracking requirements.in
 #
 attrs==23.1.0
     # via jsonschema
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 idna==3.4
     # via requests
-importlib-resources==5.12.0
+importlib-resources==6.0.0
     # via jsonschema
 jsonschema==4.17.3
-    # via serialized-data-interface
+    # via
+    #   -r requirements.in
+    #   serialized-data-interface
 oci-image==1.0.0
-    # via -r ./requirements.in
-ops==1.2.0
+    # via -r requirements.in
+ops==2.4.1
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   serialized-data-interface
 pkgutil-resolve-name==1.3.10
     # via jsonschema
 pyrsistent==0.19.3
     # via jsonschema
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
 serialized-data-interface==0.6.0
-    # via -r ./requirements.in
+    # via -r requirements.in
 urllib3==2.0.3
     # via requests
-zipp==3.15.0
+websocket-client==1.6.1
+    # via ops
+zipp==3.16.2
     # via importlib-resources

charms/kfp-profile-controller/requirements.in

@@ -1,5 +1,6 @@
-ops==1.2.0
-# Unittest fails if not pinned. unknown config option: 'service-port'
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
+ops
 oci-image
 serialized-data-interface<0.7
-# Unittest fails if not pinned. unknown config option: 'service-port'

charms/kfp-profile-controller/requirements.txt

@@ -2,39 +2,43 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --resolver=backtracking ./requirements.in
+#    pip-compile --resolver=backtracking requirements.in
 #
 attrs==23.1.0
     # via jsonschema
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 idna==3.4
     # via requests
-importlib-resources==5.12.0
+importlib-resources==6.0.0
     # via jsonschema
 jsonschema==4.17.3
-    # via serialized-data-interface
+    # via
+    #   -r requirements.in
+    #   serialized-data-interface
 oci-image==1.0.0
-    # via -r ./requirements.in
-ops==1.2.0
+    # via -r requirements.in
+ops==2.4.1
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   serialized-data-interface
 pkgutil-resolve-name==1.3.10
     # via jsonschema
 pyrsistent==0.19.3
     # via jsonschema
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
 serialized-data-interface==0.6.0
-    # via -r ./requirements.in
+    # via -r requirements.in
 urllib3==2.0.3
     # via requests
-zipp==3.15.0
+websocket-client==1.6.1
+    # via ops
+zipp==3.16.2
     # via importlib-resources

charms/kfp-schedwf/requirements.in

@@ -1,2 +1,5 @@
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
 ops
 oci-image

charms/kfp-schedwf/requirements.txt

@@ -2,13 +2,25 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --resolver=backtracking ./requirements.in
+#    pip-compile --resolver=backtracking requirements.in
 #
+attrs==23.1.0
+    # via jsonschema
+importlib-resources==6.0.0
+    # via jsonschema
+jsonschema==4.17.3
+    # via -r requirements.in
 oci-image==1.0.0
-    # via -r ./requirements.in
-ops==2.3.0
-    # via -r ./requirements.in
-pyyaml==6.0
+    # via -r requirements.in
+ops==2.4.1
+    # via -r requirements.in
+pkgutil-resolve-name==1.3.10
+    # via jsonschema
+pyrsistent==0.19.3
+    # via jsonschema
+pyyaml==6.0.1
     # via ops
-websocket-client==1.6.0
+websocket-client==1.6.1
     # via ops
+zipp==3.16.2
+    # via importlib-resources

charms/kfp-ui/requirements.in

@@ -1,3 +1,6 @@
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
 ops
 oci-image
 serialized-data-interface<0.7

charms/kfp-ui/requirements.txt

@@ -2,41 +2,43 @@
 # This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --resolver=backtracking ./requirements.in
+#    pip-compile --resolver=backtracking requirements.in
 #
 attrs==23.1.0
     # via jsonschema
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 idna==3.4
     # via requests
-importlib-resources==5.12.0
+importlib-resources==6.0.0
     # via jsonschema
 jsonschema==4.17.3
-    # via serialized-data-interface
+    # via
+    #   -r requirements.in
+    #   serialized-data-interface
 oci-image==1.0.0
-    # via -r ./requirements.in
-ops==2.3.0
+    # via -r requirements.in
+ops==2.4.1
     # via
-    #   -r ./requirements.in
+    #   -r requirements.in
     #   serialized-data-interface
 pkgutil-resolve-name==1.3.10
     # via jsonschema
 pyrsistent==0.19.3
     # via jsonschema
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   ops
     #   serialized-data-interface
 requests==2.31.0
     # via serialized-data-interface
 serialized-data-interface==0.6.0
-    # via -r ./requirements.in
+    # via -r requirements.in
 urllib3==2.0.3
     # via requests
-websocket-client==1.6.0
+websocket-client==1.6.1
     # via ops
-zipp==3.15.0
+zipp==3.16.2
     # via importlib-resources

charms/kfp-viewer/requirements.in

@@ -1,2 +1,5 @@
+# Workaround for https://github.com/python-jsonschema/jsonschema/issues/1117
+# and https://github.com/python-jsonschema/jsonschema/issues/1114
+jsonschema<4.18
 ops
 oci-image