Merge pull request #116 from lsst-sqre/tickets/DM-33044B

tickets/DM-33044B: adopt kubernetes_asyncio
lsst-sqre · Feb 23, 2022 · faa2e51 · faa2e51
2 parents 1eb430b + b5f641e
commit faa2e51
Show file tree

Hide file tree

Showing 9 changed files with 116 additions and 130 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM jupyterhub/jupyterhub:1.5.0 as base-image
+FROM jupyterhub/jupyterhub:2.1.1 as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .PHONY: update-deps
 update-deps:
-	pip install --upgrade pip-tools pip setuptools
+	pip install --upgrade pip-tools 'pip<22' setuptools
 	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
 	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in
 

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -228,9 +228,9 @@ frozenlist==1.3.0 \
     #   -c requirements/main.txt
     #   aiohttp
     #   aiosignal
-identify==2.4.10 \
-    --hash=sha256:7d10baf6ba6f1912a0a49f4c1c2c49fa1718765c3a37d72d13b07779567c5b85 \
-    --hash=sha256:e12b2aea3cf108de73ae055c2260783bde6601de09718f6768cf8e9f6f6322a6
+identify==2.4.11 \
+    --hash=sha256:2986942d3974c8f2e5019a190523b0b0e2a07cb8e89bf236727fb4b26f27f8fd \
+    --hash=sha256:fd906823ed1db23c7a48f9b176a1d71cb8abede1e21ebe614bac7bdd688d9213
     # via pre-commit
 idna==3.3 \
     --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \
@@ -339,7 +339,9 @@ nodeenv==1.6.0 \
 packaging==21.3 \
     --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
     --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
-    # via pytest
+    # via
+    #   -c requirements/main.txt
+    #   pytest
 platformdirs==2.5.1 \
     --hash=sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d \
     --hash=sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227
@@ -359,7 +361,9 @@ py==1.11.0 \
 pyparsing==3.0.7 \
     --hash=sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea \
     --hash=sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484
-    # via packaging
+    # via
+    #   -c requirements/main.txt
+    #   packaging
 pytest==7.0.1 \
     --hash=sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db \
     --hash=sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171

diff --git a/requirements/main.in b/requirements/main.in
@@ -11,19 +11,19 @@ cchardet
 inflect
 jinja2
 jupyterhub-idle-culler
-jupyterhub-kubespawner
 psycopg2
 ruamel.yaml
 tornado
 
-# Introduces a renaming of V1EndpointPort to CoreV1EndpointPort that
-# breaks jupyterhub-kubespawner.
-kubernetes!=21.7.0
+kubernetes-asyncio
+
+# Until changes are accepted upstream:
+rubin-kubespawner
 
 # Always pin jupyterhub to a specific version.  We don't want it to be
 # upgraded without our explicit approval, and it should be upgraded in
 # lockstep with the base image in Dockerfile.
-jupyterhub==1.5.0
+jupyterhub==2.1.1
 
 # Required by alembic for Python 3.8, so install it unconditionally until
 # Python 3.8 support is dropped so that we have consistent dependencies.

diff --git a/requirements/main.txt b/requirements/main.txt
@@ -81,7 +81,9 @@ aiohttp==3.8.1 \
     --hash=sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237 \
     --hash=sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642 \
     --hash=sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578
-    # via -r requirements/main.in
+    # via
+    #   -r requirements/main.in
+    #   kubernetes-asyncio
 aiosignal==1.2.0 \
     --hash=sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a \
     --hash=sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2
@@ -95,7 +97,7 @@ async-generator==1.10 \
     --hash=sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144
     # via
     #   jupyterhub
-    #   jupyterhub-kubespawner
+    #   rubin-kubespawner
 async-timeout==4.0.2 \
     --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \
     --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c
@@ -106,10 +108,6 @@ attrs==21.4.0 \
     # via
     #   aiohttp
     #   jsonschema
-cachetools==5.0.0 \
-    --hash=sha256:486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6 \
-    --hash=sha256:8fecd4203a38af17928be7b90689d8083603073622229ca7077b72d8e5a976e4
-    # via google-auth
 cchardet==2.1.7 \
     --hash=sha256:0b859069bbb9d27c78a2c9eb997e6f4b738db2d7039a03f8792b4058d61d1109 \
     --hash=sha256:228d2533987c450f39acf7548f474dd6814c446e9d6bd228e8f1d9a2d210f10b \
@@ -145,7 +143,7 @@ certifi==2021.10.8 \
     --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
     --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
     # via
-    #   kubernetes
+    #   kubernetes-asyncio
     #   requests
 certipy==0.1.3 \
     --hash=sha256:695704b7716b033375c9a1324d0d30f27110a28895c40151a90ec07ff1032859 \
@@ -240,7 +238,7 @@ entrypoints==0.4 \
 escapism==1.0.1 \
     --hash=sha256:73256bdfb4f22230f0428fc6efecee61cdc4fad531b6f98b849cb9c80711e4ec \
     --hash=sha256:d28f19edc3cb1ffc36fa238956ecc068695477e748f57157c6dde00a6b77f229
-    # via jupyterhub-kubespawner
+    # via rubin-kubespawner
 frozenlist==1.3.0 \
     --hash=sha256:006d3595e7d4108a12025ddf415ae0f6c9e736e726a5db0183326fd191b14c5e \
     --hash=sha256:01a73627448b1f2145bddb6e6c2259988bb8aee0fb361776ff8604b99616cd08 \
@@ -304,10 +302,6 @@ frozenlist==1.3.0 \
     # via
     #   aiohttp
     #   aiosignal
-google-auth==2.6.0 \
-    --hash=sha256:218ca03d7744ca0c8b6697b6083334be7df49b7bf76a69d555962fd1a7657b5f \
-    --hash=sha256:ad160fc1ea8f19e331a16a14a79f3d643d813a69534ba9611d2c80dc10439dad
-    # via kubernetes
 greenlet==1.1.2 \
     --hash=sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3 \
     --hash=sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711 \
@@ -389,7 +383,7 @@ jinja2==3.0.3 \
     # via
     #   -r requirements/main.in
     #   jupyterhub
-    #   jupyterhub-kubespawner
+    #   rubin-kubespawner
 jsonschema==4.4.0 \
     --hash=sha256:636694eb41b3535ed608fe04129f26542b59ed99808b4f688aa32dcf55317a83 \
     --hash=sha256:77281a1f71684953ee8b3d488371b162419767973789272434bbc3f29d9c8823
@@ -398,26 +392,21 @@ jupyter-telemetry==0.1.0 \
     --hash=sha256:1de3e423b23aa40ca4a4238d65c56dda544061ff5aedc3f7647220ed7e3b9589 \
     --hash=sha256:445c613ae3df70d255fe3de202f936bba8b77b4055c43207edf22468ac875314
     # via jupyterhub
-jupyterhub==1.5.0 \
-    --hash=sha256:98e3f29671be6f530433d654a02d5a2d4c0ca637f7999e42aeb1c836f418bbec \
-    --hash=sha256:dc618f657c23ba46280e36257e50931806674ba0e9e6498afb091efc6226d69d
+jupyterhub==2.1.1 \
+    --hash=sha256:386ae16dba48cb48f593bd1108cc01fb3c500cca607afae13e35d96461d7a19c \
+    --hash=sha256:ec93c94bd1a31917cb15269ee614c9b1cdda846cd20de5f25c5c7191cab19bb4
     # via
     #   -r requirements/main.in
-    #   jupyterhub-kubespawner
+    #   rubin-kubespawner
 jupyterhub-idle-culler==1.2.1 \
     --hash=sha256:c84e45a51932a34bd95c08b3b6e8330fb0ee3391fa4d0f1ac0d4458a40492fd9 \
     --hash=sha256:d80822982d2590cb876d01849cee028943fdbdcc991c0f02fce45af3f40dd415
     # via -r requirements/main.in
-jupyterhub-kubespawner==2.0.1 \
-    --hash=sha256:a05194c52f34aaab1982f6585c7afa6681b18ab3e2ef47e4a3f942e3a5bcbd7d \
-    --hash=sha256:a4136b3a27c98a991db8c37b32ad25bff115502164e5364f92d08b30af629002
-    # via -r requirements/main.in
-kubernetes==22.6.0 \
-    --hash=sha256:5d329ee9b40e333e32f7bb62339cdce3eac47ab7c50b305ec21c114c49447e20 \
-    --hash=sha256:8578afa0fd4e7e86cfa4eeec204e4bad8a2c109ce52f85f8f3c593e9b0d47960
+kubernetes-asyncio==19.15.1 \
+    --hash=sha256:39f01eb267d1fec52e2f37f9fbe6b63b095011a64ca6de53988daf307d319f28
     # via
     #   -r requirements/main.in
-    #   jupyterhub-kubespawner
+    #   rubin-kubespawner
 mako==1.1.6 \
     --hash=sha256:4e9e345a41924a954251b95b4b28e14a301145b544901332e658907a7464b6b2 \
     --hash=sha256:afaf8e515d075b22fad7d7b8b30e4a1c90624ff2f3733a06ec125f5a5f043a57
@@ -532,9 +521,11 @@ multidict==6.0.2 \
 oauthlib==3.2.0 \
     --hash=sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2 \
     --hash=sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe
-    # via
-    #   jupyterhub
-    #   requests-oauthlib
+    # via jupyterhub
+packaging==21.3 \
+    --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
+    --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
+    # via jupyterhub
 pamela==1.0.0 \
     --hash=sha256:65c9389bef7d1bb0b168813b6be21964df32016923aac7515bdf05366acbab6c \
     --hash=sha256:b54be508a13bb3d983d117f7b069b31545003ae989791f467cf941376c807f20
@@ -556,16 +547,6 @@ psycopg2==2.9.3 \
     --hash=sha256:cb10d44e6694d763fa1078a26f7f6137d69f555a78ec85dc2ef716c37447e4b2 \
     --hash=sha256:d3ca6421b942f60c008f81a3541e8faf6865a28d5a9b48544b0ee4f40cac7fca
     # via -r requirements/main.in
-pyasn1==0.4.8 \
-    --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
-    --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba
-    # via
-    #   pyasn1-modules
-    #   rsa
-pyasn1-modules==0.2.8 \
-    --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \
-    --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74
-    # via google-auth
 pycares==4.1.2 \
     --hash=sha256:03490be0e7b51a0c8073f877bec347eff31003f64f57d9518d419d9369452837 \
     --hash=sha256:056330275dea42b7199494047a745e1d9785d39fb8c4cd469dca043532240b80 \
@@ -607,6 +588,10 @@ pyopenssl==22.0.0 \
     --hash=sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf \
     --hash=sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0
     # via certipy
+pyparsing==3.0.7 \
+    --hash=sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea \
+    --hash=sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484
+    # via packaging
 pyrsistent==0.18.1 \
     --hash=sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c \
     --hash=sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc \
@@ -636,15 +621,15 @@ python-dateutil==2.8.2 \
     # via
     #   jupyterhub
     #   jupyterhub-idle-culler
-    #   kubernetes
+    #   kubernetes-asyncio
 python-json-logger==2.0.2 \
     --hash=sha256:202a4f29901a4b8002a6d1b958407eeb2dd1d83c18b18b816f5b64476dde9096 \
     --hash=sha256:99310d148f054e858cd5f4258794ed6777e7ad2c3fd7e1c1b527f1cba4d08420
     # via jupyter-telemetry
-python-slugify==6.0.1 \
-    --hash=sha256:89eec682c5180ba64811c9906a28184bbcc0a35792ba1bda3b5c2ab0cb2d0f67 \
-    --hash=sha256:ba72aa9d9f0514c0c3dd4430442f698ccc27a24d19630473663a71e3ec606bc1
-    # via jupyterhub-kubespawner
+python-slugify==6.1.0 \
+    --hash=sha256:2e3fad0bf38b11514f8de911ea04e7a6c6a08bb1bac18abd96d9566c34404d56 \
+    --hash=sha256:eff190e4dfac97d2f8c1890ee682709ecd23650742361687db82d95e1e5e25f5
+    # via rubin-kubespawner
 pyyaml==6.0 \
     --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
     --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
@@ -680,23 +665,12 @@ pyyaml==6.0 \
     --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
     --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
     # via
-    #   jupyterhub-kubespawner
-    #   kubernetes
+    #   kubernetes-asyncio
+    #   rubin-kubespawner
 requests==2.27.1 \
     --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \
     --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d
-    # via
-    #   jupyterhub
-    #   kubernetes
-    #   requests-oauthlib
-requests-oauthlib==1.3.1 \
-    --hash=sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5 \
-    --hash=sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a
-    # via kubernetes
-rsa==4.8 \
-    --hash=sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17 \
-    --hash=sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb
-    # via google-auth
+    # via jupyterhub
 ruamel-yaml==0.17.21 \
     --hash=sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7 \
     --hash=sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af
@@ -730,12 +704,15 @@ ruamel-yaml-clib==0.2.6 \
     --hash=sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed \
     --hash=sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c
     # via ruamel-yaml
+rubin-kubespawner==2.0.1.dev6 \
+    --hash=sha256:3602bb5791a1ae8c1c8c9c4b4b56007ab8e16c520b703e1d2bb5b8a16dc6d339 \
+    --hash=sha256:709d36528f7dd631556ed3544b2f697f18f5751128962469dd5c72a9636d6bc1
+    # via -r requirements/main.in
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
-    #   google-auth
-    #   kubernetes
+    #   kubernetes-asyncio
     #   python-dateutil
 sqlalchemy==1.4.31 \
     --hash=sha256:05fa14f279d43df68964ad066f653193187909950aa0163320b728edfc400167 \
@@ -837,13 +814,9 @@ urllib3==1.26.8 \
     --hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \
     --hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c
     # via
-    #   jupyterhub-kubespawner
-    #   kubernetes
+    #   kubernetes-asyncio
     #   requests
-websocket-client==1.2.3 \
-    --hash=sha256:1315816c0acc508997eb3ae03b9d3ff619c9d12d544c9a9b553704b1cc4f6af5 \
-    --hash=sha256:2eed4cc58e4d65613ed6114af2f380f7910ff416fc8c46947f6e76b6815f56c0
-    # via kubernetes
+    #   rubin-kubespawner
 yarl==1.7.2 \
     --hash=sha256:044daf3012e43d4b3538562da94a88fb12a6490652dbc29fb19adfa02cf72eac \
     --hash=sha256:0cba38120db72123db7c58322fa69e3c0efa933040ffb586c3a87c063ec7cae8 \

diff --git a/src/nublado2/hub_config.py b/src/nublado2/hub_config.py
@@ -31,6 +31,8 @@ def configure(self, c: JupyterHub) -> None:
         c.Spawner.options_from_form = hooks.options_from_form
         # Turn off restart after n consecutive failures
         c.Spawner.consecutive_failure_limit = 0
+        # Use JupyterLab by default
+        c.Spawner.default_url = "/lab"
 
         c.KubeSpawner.enable_user_namespaces = True
 

diff --git a/src/nublado2/options.py b/src/nublado2/options.py
@@ -110,7 +110,11 @@ def __init__(self) -> None:
 
     async def show_options_form(self, spawner: Spawner) -> str:
         base_url = self.nublado_config.base_url
-        url = urljoin(base_url, "cachemachine/jupyter/available")
+        # This should probably be configurable between 'available' and
+        # 'desired'.  Where we have image streaming, 'desired' is fine and
+        # will improve user experience during autoscales.
+        url = urljoin(base_url, "cachemachine/jupyter/desired")
+        # url = urljoin(base_url, "cachemachine/jupyter/available")
         (cached_images, all_images) = await self._get_images_from_url(url)
         cached_images.extend(self.nublado_config.pinned_images)