From 4e4ec11e764faae18323b9ef3cc59f26fa6a4a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Mon, 13 Aug 2018 13:53:02 -0400 Subject: [PATCH 1/5] Add API handler to post remote server port number --- batchspawner/__init__.py | 2 ++ batchspawner/api.py | 17 +++++++++++++++++ batchspawner/batchspawner.py | 16 ++++++++-------- batchspawner/singleuser.py | 21 +++++++++++++++++++++ scripts/batchspawner-singleuser | 6 ++++++ setup.py | 2 ++ 6 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 batchspawner/api.py create mode 100644 batchspawner/singleuser.py create mode 100644 scripts/batchspawner-singleuser diff --git a/batchspawner/__init__.py b/batchspawner/__init__.py index bba3352d..191c968e 100644 --- a/batchspawner/__init__.py +++ b/batchspawner/__init__.py @@ -1 +1,3 @@ from .batchspawner import * +from . import singleuser +from . import api \ No newline at end of file diff --git a/batchspawner/api.py b/batchspawner/api.py new file mode 100644 index 00000000..c2f1163c --- /dev/null +++ b/batchspawner/api.py @@ -0,0 +1,17 @@ +import json +from tornado import web +from jupyterhub.apihandlers import APIHandler, default_handlers + +class BatchSpawnerAPIHandler(APIHandler): + @web.authenticated + def post(self): + """POST set user's spawner port number""" + user = self.get_current_user() + data = self.get_json_body() + if user.spawner.port == 0: + port = data.get('port', 0) + user.spawner.port = int(port) + self.finish(json.dumps({"message": "BatchSpawner port configured"})) + self.set_status(201) + +default_handlers.append((r"/api/batchspawner", BatchSpawnerAPIHandler)) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 9f34854c..8acf0027 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -29,6 +29,7 @@ from tornado.iostream import StreamClosedError from jupyterhub.spawner import Spawner +from jupyterhub.traitlets import Command from traitlets import ( Integer, Unicode, Float, Dict, default ) @@ -73,6 +74,9 @@ class BatchSpawnerBase(Spawner): state_gethost """ + # override default since will need to set the listening port using the api + cmd = Command(['batchspawner-singleuser'], allow_none=True).tag(config=True) + # override default since batch systems typically need longer start_timeout = Integer(300).tag(config=True) @@ -342,14 +346,7 @@ def poll(self): @gen.coroutine def start(self): """Start the process""" - if self.user and self.user.server and self.user.server.port: - self.port = self.user.server.port - self.db.commit() - elif (jupyterhub.version_info < (0,7) and not self.user.server.port) or ( - jupyterhub.version_info >= (0,7) and not self.port - ): - self.port = random_port() - self.db.commit() + self.port = self.server.port = 0 job = yield self.submit_batch_script() # We are called with a timeout, and if the timeout expires this function will @@ -374,6 +371,9 @@ def start(self): yield gen.sleep(self.startup_poll_interval) self.current_ip = self.state_gethost() + while self.port == 0: + yield gen.sleep(self.startup_poll_interval) + if jupyterhub.version_info < (0,7): # store on user for pre-jupyterhub-0.7: self.user.server.port = self.port diff --git a/batchspawner/singleuser.py b/batchspawner/singleuser.py new file mode 100644 index 00000000..6d493262 --- /dev/null +++ b/batchspawner/singleuser.py @@ -0,0 +1,21 @@ +from jupyterhub.singleuser import SingleUserNotebookApp +from jupyterhub.utils import random_port, url_path_join +from traitlets import default + +class BatchSingleUserNotebookApp(SingleUserNotebookApp): + @default('port') + def _port(self): + return random_port() + + def start(self): + # Send Notebook app's port number to remote Spawner + self.hub_auth._api_request(method='POST', + url=url_path_join(self.hub_api_url, 'batchspawner'), + json={'port' : self.port}) + super().start() + +def main(argv=None): + return BatchSingleUserNotebookApp.launch_instance(argv) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/batchspawner-singleuser b/scripts/batchspawner-singleuser new file mode 100644 index 00000000..1b91fe47 --- /dev/null +++ b/scripts/batchspawner-singleuser @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from batchspawner.singleuser import main + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/setup.py b/setup.py index d84d92e1..2037998e 100755 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import sys from setuptools import setup +from glob import glob pjoin = os.path.join here = os.path.abspath(os.path.dirname(__file__)) @@ -28,6 +29,7 @@ setup_args = dict( name = 'batchspawner', + scripts = glob(pjoin('scripts', '*')), packages = ['batchspawner'], version = version_ns['__version__'], description = """Batchspawner: A spawner for Jupyterhub to spawn notebooks using batch resource managers.""", From d1052385f245a3c799c5b81d30c8e67f193963c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Mon, 13 Aug 2018 16:01:37 -0400 Subject: [PATCH 2/5] Add a 'current_port' traitlet instead of modifying self.port --- batchspawner/api.py | 5 ++--- batchspawner/batchspawner.py | 15 ++++++++++----- batchspawner/tests/test_spawners.py | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/batchspawner/api.py b/batchspawner/api.py index c2f1163c..f78693a4 100644 --- a/batchspawner/api.py +++ b/batchspawner/api.py @@ -8,9 +8,8 @@ def post(self): """POST set user's spawner port number""" user = self.get_current_user() data = self.get_json_body() - if user.spawner.port == 0: - port = data.get('port', 0) - user.spawner.port = int(port) + port = int(data.get('port', 0)) + user.spawner.current_port = port self.finish(json.dumps({"message": "BatchSpawner port configured"})) self.set_status(201) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 8acf0027..26b6debc 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -168,6 +168,9 @@ def _req_keepvars_default(self): # Will get the address of the server as reported by job manager current_ip = Unicode() + # Will get the port of the server as reported by singleserver + current_port = Integer() + # Prepare substitution variables for templates using req_xyz traits def get_req_subvars(self): reqlist = [ t for t in self.trait_names() if t.startswith('req_') ] @@ -346,7 +349,9 @@ def poll(self): @gen.coroutine def start(self): """Start the process""" - self.port = self.server.port = 0 + if self.server: + self.server.port = self.port + job = yield self.submit_batch_script() # We are called with a timeout, and if the timeout expires this function will @@ -371,19 +376,19 @@ def start(self): yield gen.sleep(self.startup_poll_interval) self.current_ip = self.state_gethost() - while self.port == 0: + while self.current_port == 0: yield gen.sleep(self.startup_poll_interval) if jupyterhub.version_info < (0,7): # store on user for pre-jupyterhub-0.7: - self.user.server.port = self.port + self.user.server.port = self.current_port self.user.server.ip = self.current_ip self.db.commit() self.log.info("Notebook server job {0} started at {1}:{2}".format( - self.job_id, self.current_ip, self.port) + self.job_id, self.current_ip, self.current_port) ) - return self.current_ip, self.port + return self.current_ip, self.current_port @gen.coroutine def stop(self, now=False): diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index 01873486..a4b63dc8 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -17,6 +17,7 @@ testhost = "userhost123" testjob = "12345" +testport = 54321 class BatchDummy(BatchSpawnerRegexStates): exec_prefix = '' @@ -61,6 +62,7 @@ def new_spawner(db, spawner_class=BatchDummy, **kwargs): user = User(user, {}) kwargs.setdefault('hub', hub) kwargs.setdefault('user', user) + kwargs.setdefault('current_port', testport) kwargs.setdefault('INTERRUPT_TIMEOUT', 1) kwargs.setdefault('TERM_TIMEOUT', 1) kwargs.setdefault('KILL_TIMEOUT', 1) From f5af94f116335d8c4824066957e4c93732b0b222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 14 Aug 2018 14:49:50 -0400 Subject: [PATCH 3/5] Init a Server object in new_spawner to fix tests with JHub 0.8.x --- batchspawner/tests/test_spawners.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/batchspawner/tests/test_spawners.py b/batchspawner/tests/test_spawners.py index a4b63dc8..e0e0bc27 100644 --- a/batchspawner/tests/test_spawners.py +++ b/batchspawner/tests/test_spawners.py @@ -10,7 +10,7 @@ from tornado import gen try: - from jupyterhub.objects import Hub + from jupyterhub.objects import Hub, Server from jupyterhub.user import User except: pass @@ -60,6 +60,8 @@ def new_spawner(db, spawner_class=BatchDummy, **kwargs): else: hub = Hub() user = User(user, {}) + server = Server() + kwargs.setdefault('server', server) kwargs.setdefault('hub', hub) kwargs.setdefault('user', user) kwargs.setdefault('current_port', testport) From 410f7d9e8de55791d1c36341f70ebd952f19e093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 14 Aug 2018 15:29:10 -0400 Subject: [PATCH 4/5] Add support for JupyterHub 0.7.x --- batchspawner/batchspawner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batchspawner/batchspawner.py b/batchspawner/batchspawner.py index 26b6debc..b1ecc19d 100644 --- a/batchspawner/batchspawner.py +++ b/batchspawner/batchspawner.py @@ -349,7 +349,7 @@ def poll(self): @gen.coroutine def start(self): """Start the process""" - if self.server: + if jupyterhub.version_info >= (0,8) and self.server: self.server.port = self.port job = yield self.submit_batch_script() From 2e024e805f2d2acebd88665d3df865aef4e5b1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 14 Aug 2018 17:06:37 -0400 Subject: [PATCH 5/5] Remove import of singleuser from __init__.py Avoid error when notebook is not installed with JupyterHub --- batchspawner/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/batchspawner/__init__.py b/batchspawner/__init__.py index 191c968e..e6b50a69 100644 --- a/batchspawner/__init__.py +++ b/batchspawner/__init__.py @@ -1,3 +1,2 @@ from .batchspawner import * -from . import singleuser from . import api \ No newline at end of file