From 0394b44fc44ac757cb54556835fb84154f92e677 Mon Sep 17 00:00:00 2001 From: Vinay Date: Fri, 11 Aug 2017 13:24:17 +0800 Subject: [PATCH 1/7] Added to_dict --- floyd/model/experiment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/floyd/model/experiment.py b/floyd/model/experiment.py index e8b783d..82daa48 100644 --- a/floyd/model/experiment.py +++ b/floyd/model/experiment.py @@ -68,6 +68,9 @@ def localize_date(self, date): date = utc.localize(date) return date.astimezone(PST_TIMEZONE) + def to_dict(self): + return self.__dict__ + @property def created_pretty(self): return pretty_date(self.created) From 5a3b49c197fdb27c6640c986dafb5647117ff651 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 13 Aug 2017 00:41:43 -0400 Subject: [PATCH 2/7] only catch 404 when checking project existence on run command --- floyd/client/project.py | 2 +- tests/cli/run/test_run.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/floyd/client/project.py b/floyd/client/project.py index fe781e4..e2e11e0 100644 --- a/floyd/client/project.py +++ b/floyd/client/project.py @@ -43,5 +43,5 @@ def exists(self, project_id): return True else: return False - except FloydException: + except NotFoundException: return False diff --git a/tests/cli/run/test_run.py b/tests/cli/run/test_run.py index 8232403..3f0a6a0 100644 --- a/tests/cli/run/test_run.py +++ b/tests/cli/run/test_run.py @@ -19,13 +19,15 @@ def setUp(self): @patch('floyd.cli.run.ExperimentConfigManager.set_config') @patch('floyd.cli.run.ModuleClient.create', return_value='module_id') @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id') + @patch('floyd.cli.run.ProjectClient.exists', return_value=True) def test_with_no_data(self, create_experiment, create_module, set_config, get_config, get_access_token, - get_all_env): + get_all_env, + exists): """ Simple experiment with no data attached """ @@ -37,12 +39,14 @@ def test_with_no_data(self, @patch('floyd.cli.run.ExperimentConfigManager.set_config') @patch('floyd.cli.run.ModuleClient.create', return_value='module_id') @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id') + @patch('floyd.cli.run.ProjectClient.exists', return_value=True) def test_with_multiple_data_ids(self, create_experiment, create_module, set_config, get_config, - get_access_token): + get_access_token, + exists): """ Simple experiment with no data attached """ From 91bef54723e50f2448748106222b557ae737299b Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 13 Aug 2017 20:50:20 -0400 Subject: [PATCH 3/7] make data and job name consistent with web UI --- floyd/cli/data.py | 9 ++++++--- floyd/cli/data_upload_utils.py | 3 ++- floyd/cli/experiment.py | 11 +++++++---- floyd/cli/run.py | 20 ++++++++++++-------- floyd/cli/utils.py | 23 +++++++++++++++++++++++ floyd/main.py | 1 - tests/cli/utils_test.py | 18 ++++++++++++++++++ 7 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 tests/cli/utils_test.py diff --git a/floyd/cli/data.py b/floyd/cli/data.py index 08781bf..1e298f8 100644 --- a/floyd/cli/data.py +++ b/floyd/cli/data.py @@ -14,6 +14,7 @@ opt_to_resume, upload_is_resumable, abort_previous_upload, initialize_new_upload, complete_upload ) +from floyd.cli.utils import normalize_data_name @click.group() @@ -97,7 +98,8 @@ def print_data(data_sources): headers = ["DATA NAME", "CREATED", "STATUS", "DISK USAGE"] data_list = [] for data_source in data_sources: - data_list.append([data_source.name, data_source.created_pretty, + data_list.append([normalize_data_name(data_source.name), + data_source.created_pretty, data_source.state, data_source.size]) floyd_logger.info(tabulate(data_list, headers=headers)) @@ -159,10 +161,11 @@ def delete(ids, yes): failures = True continue - if not yes and not click.confirm("Delete Data: {}?".format(data_source.name), + data_name = normalize_data_name(data_source.name) + if not yes and not click.confirm("Delete Data: {}?".format(data_name), abort=False, default=False): - floyd_logger.info("Data {}: Skipped".format(data_source.name)) + floyd_logger.info("Data %s: Skipped", data_name) continue if not DataClient().delete(id): diff --git a/floyd/cli/data_upload_utils.py b/floyd/cli/data_upload_utils.py index 01ad928..37d0f13 100644 --- a/floyd/cli/data_upload_utils.py +++ b/floyd/cli/data_upload_utils.py @@ -14,6 +14,7 @@ from floyd.log import logger as floyd_logger from floyd.manager.data_config import DataConfigManager from floyd.model.data import DataRequest +from floyd.cli.utils import normalize_data_name class ResourceWaitIter(object): @@ -169,7 +170,7 @@ def complete_upload(data_config): # Print output table_output = [["DATA ID", "NAME"], - [data_id, data_config.data_name]] + [data_id, normalize_data_name(data_config.data_name)]] floyd_logger.info(tabulate(table_output, headers="firstrow")) diff --git a/floyd/cli/experiment.py b/floyd/cli/experiment.py index 2ab41ad..8c194ca 100644 --- a/floyd/cli/experiment.py +++ b/floyd/cli/experiment.py @@ -5,7 +5,9 @@ import sys import floyd -from floyd.cli.utils import get_module_task_instance_id +from floyd.cli.utils import ( + get_module_task_instance_id, normalize_job_name, normalize_data_name +) from floyd.client.experiment import ExperimentClient from floyd.client.module import ModuleClient from floyd.client.project import ProjectClient @@ -65,7 +67,8 @@ def print_experiments(experiments): headers = ["JOB NAME", "CREATED", "STATUS", "DURATION(s)", "INSTANCE", "DESCRIPTION"] expt_list = [] for experiment in experiments: - expt_list.append([experiment.name, experiment.created_pretty, experiment.state, + expt_list.append([normalize_job_name(experiment.name), + experiment.created_pretty, experiment.state, experiment.duration_rounded, experiment.instance_type_trimmed, experiment.description]) floyd_logger.info(tabulate(expt_list, headers=headers)) @@ -99,8 +102,8 @@ def info(id): experiment = ExperimentClient().get(id) task_instance_id = get_module_task_instance_id(experiment.task_instances) task_instance = TaskInstanceClient().get(task_instance_id) if task_instance_id else None - table = [["Job name", experiment.name], - ["Output name", '%s/output' % experiment.name if task_instance else None], + table = [["Job name", normalize_job_name(experiment.name)], + ["Output name", normalize_data_name(experiment.name + '/output') if task_instance else None], ["Created", experiment.created_pretty], ["Status", experiment.state], ["Duration(s)", experiment.duration_rounded], ["Instance", experiment.instance_type_trimmed], diff --git a/floyd/cli/run.py b/floyd/cli/run.py index 21b5989..c2f42c8 100644 --- a/floyd/cli/run.py +++ b/floyd/cli/run.py @@ -8,7 +8,9 @@ from floyd.constants import DEFAULT_ENV from floyd.client.data import DataClient from floyd.client.project import ProjectClient -from floyd.cli.utils import get_mode_parameter, wait_for_url, get_data_name +from floyd.cli.utils import ( + get_mode_parameter, wait_for_url, get_data_name, normalize_job_name +) from floyd.client.experiment import ExperimentClient from floyd.client.module import ModuleClient from floyd.client.env import EnvClient @@ -133,9 +135,11 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): instance_type=instance_type) expt_cli = ExperimentClient() expt_info = expt_cli.create(experiment_request) - floyd_logger.debug("Created job : {}".format(expt_info['id'])) + floyd_logger.debug("Created job : %s", expt_info['id']) - table_output = [["JOB NAME"], [expt_info['name']]] + job_name = normalize_job_name(expt_info['name']) + floyd_logger.info("") + table_output = [["JOB NAME"], [job_name]] floyd_logger.info(tabulate(table_output, headers="firstrow")) floyd_logger.info("") @@ -147,9 +151,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): if experiment.task_instances: break except Exception: - floyd_logger.debug("Job not available yet: {}".format(expt_info['id'])) + floyd_logger.debug("Job not available yet: %s", expt_info['id']) - floyd_logger.debug("Job not available yet: {}".format(expt_info['id'])) + floyd_logger.debug("Job not available yet: %s", expt_info['id']) sleep(3) continue @@ -162,9 +166,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): if open: webbrowser.open(jupyter_url) else: - floyd_logger.info("\nPath to jupyter notebook: {}".format(jupyter_url)) + floyd_logger.info("\nPath to jupyter notebook: %s", jupyter_url) floyd_logger.info("Notebook is still loading. View logs to track progress") - floyd_logger.info(" floyd logs {}".format(expt_info['name'])) + floyd_logger.info(" floyd logs %s", job_name) # Print the path to serving endpoint if mode == 'serve': @@ -177,7 +181,7 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): else: floyd_logger.info("To view logs enter:") - floyd_logger.info(" floyd logs {}".format(expt_info['name'])) + floyd_logger.info(" floyd logs %s", job_name) def get_command_line(gpu, env, message, data, mode, open, tensorboard, command): diff --git a/floyd/cli/utils.py b/floyd/cli/utils.py index bfad5bb..fb97eaf 100644 --- a/floyd/cli/utils.py +++ b/floyd/cli/utils.py @@ -72,3 +72,26 @@ def get_data_id(data_str): return name_or_id else: return data_str + + +def normalize_data_name(data_name): + if data_name.endswith('/output'): + name_parts = data_name.split('/') + if len(name_parts) <= 4: + name_parts.insert(1, 'projects') + data_name = '/'.join(name_parts) + return data_name + else: + name_parts = data_name.split('/') + if len(name_parts) <= 3: + name_parts.insert(1, 'datasets') + data_name = '/'.join(name_parts) + return data_name + + +def normalize_job_name(job_name): + job_name_parts = job_name.split('/') + if len(job_name_parts) <= 3: + job_name_parts.insert(1, 'projects') + job_name = '/'.join(job_name_parts) + return job_name diff --git a/floyd/main.py b/floyd/main.py index 499409b..9567e8b 100644 --- a/floyd/main.py +++ b/floyd/main.py @@ -10,7 +10,6 @@ from floyd.cli.run import run from floyd.cli.version import upgrade, version from floyd.client.version import VersionClient -from floyd.exceptions import FloydException from floyd.log import configure_logger diff --git a/tests/cli/utils_test.py b/tests/cli/utils_test.py new file mode 100644 index 0000000..cbe02fd --- /dev/null +++ b/tests/cli/utils_test.py @@ -0,0 +1,18 @@ +import unittest + + +class TestCliUtil(unittest.TestCase): + """ + Tests cli utils helper functions + """ + def test_normalize_data_name(self): + from floyd.cli.utils import normalize_data_name + assert normalize_data_name('foo/bar/1') == 'foo/datasets/bar/1' + assert normalize_data_name('foo/datasets/bar/1') == 'foo/datasets/bar/1' + assert normalize_data_name('foo/bar/1/output') == 'foo/projects/bar/1/output' + assert normalize_data_name('foo/projects/bar/1/output') == 'foo/projects/bar/1/output' + + def test_normalize_job_name(self): + from floyd.cli.utils import normalize_job_name + assert normalize_job_name('foo/bar/1') == 'foo/projects/bar/1' + assert normalize_job_name('foo/projects/bar/1') == 'foo/projects/bar/1' From b29edf69c39c679bc801cd3208c300f92bbfe2ee Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 13 Aug 2017 21:19:32 -0400 Subject: [PATCH 4/7] do not allow job output deletion from data delete command --- floyd/cli/data.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/floyd/cli/data.py b/floyd/cli/data.py index 1e298f8..d91747b 100644 --- a/floyd/cli/data.py +++ b/floyd/cli/data.py @@ -162,13 +162,21 @@ def delete(ids, yes): continue data_name = normalize_data_name(data_source.name) + suffix = data_name.split('/')[-1] + if not suffix.isnumeric(): + failures = True + floyd_logger.error('%s is not a dataset, skipped.', id) + if suffix == 'output': + floyd_logger.error('To delete job output, please delete the job itself.') + continue + if not yes and not click.confirm("Delete Data: {}?".format(data_name), abort=False, default=False): floyd_logger.info("Data %s: Skipped", data_name) continue - if not DataClient().delete(id): + if not DataClient().delete(data_source.id): failures = True if failures: From 2df2d56177e8f73e1224f63d588365a7a03793c4 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 13 Aug 2017 21:27:06 -0400 Subject: [PATCH 5/7] hide data id from status and delete command --- floyd/cli/data.py | 2 ++ floyd/cli/data_upload_utils.py | 5 +++-- floyd/client/data.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/floyd/cli/data.py b/floyd/cli/data.py index d91747b..6c3398f 100644 --- a/floyd/cli/data.py +++ b/floyd/cli/data.py @@ -178,6 +178,8 @@ def delete(ids, yes): if not DataClient().delete(data_source.id): failures = True + else: + floyd_logger.info("Data %s: Deleted", data_name) if failures: sys.exit(1) diff --git a/floyd/cli/data_upload_utils.py b/floyd/cli/data_upload_utils.py index 37d0f13..89cf4c5 100644 --- a/floyd/cli/data_upload_utils.py +++ b/floyd/cli/data_upload_utils.py @@ -169,8 +169,9 @@ def complete_upload(data_config): DataConfigManager.set_config(data_config) # Print output - table_output = [["DATA ID", "NAME"], - [data_id, normalize_data_name(data_config.data_name)]] + table_output = [["NAME"], + [normalize_data_name(data_config.data_name)]] + floyd_logger.info('') floyd_logger.info(tabulate(table_output, headers="firstrow")) diff --git a/floyd/client/data.py b/floyd/client/data.py index 36ef1aa..6eca0dd 100644 --- a/floyd/client/data.py +++ b/floyd/client/data.py @@ -90,7 +90,6 @@ def delete(self, data_id): try: # data delete is a synchronous process, it can take a long time self.request("DELETE", self.url + data_id, timeout=60) - floyd_logger.info("Data %s: Deleted", data_id) return True except FloydException as e: floyd_logger.error("Data %s: ERROR! %s", data_id, e.message) From fec7ce2971bedfde41c50b99640d8ce52d5a43ae Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Mon, 14 Aug 2017 02:04:27 -0400 Subject: [PATCH 6/7] fix tests --- floyd/cli/data.py | 3 ++- floyd/client/experiment.py | 8 ++++---- floyd/main.py | 1 + tests/cli/data/mocks.py | 3 ++- tox.ini | 4 ++++ 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/floyd/cli/data.py b/floyd/cli/data.py index 6c3398f..ce10a4a 100644 --- a/floyd/cli/data.py +++ b/floyd/cli/data.py @@ -163,7 +163,7 @@ def delete(ids, yes): data_name = normalize_data_name(data_source.name) suffix = data_name.split('/')[-1] - if not suffix.isnumeric(): + if not suffix.isdigit(): failures = True floyd_logger.error('%s is not a dataset, skipped.', id) if suffix == 'output': @@ -184,6 +184,7 @@ def delete(ids, yes): if failures: sys.exit(1) + data.add_command(clone) data.add_command(delete) data.add_command(init) diff --git a/floyd/client/experiment.py b/floyd/client/experiment.py index a4875bb..03490d0 100644 --- a/floyd/client/experiment.py +++ b/floyd/client/experiment.py @@ -48,10 +48,10 @@ def delete(self, id): return True except NotFoundException as e: floyd_logger.info( - ("Job {}: ERROR! A deletable job with this " - "id was not found. Make sure you have the correct id and " - "that the experiment is not " - "queued or running.".format(id)) + ("Job {}: ERROR! A deletable job with this " + "id was not found. Make sure you have the correct id and " + "that the experiment is not " + "queued or running.".format(id)) ) return False except FloydException as e: diff --git a/floyd/main.py b/floyd/main.py index 9567e8b..d246ad5 100644 --- a/floyd/main.py +++ b/floyd/main.py @@ -67,4 +67,5 @@ def add_commands(cli): cli.add_command(upgrade) cli.add_command(version) + add_commands(cli) diff --git a/tests/cli/data/mocks.py b/tests/cli/data/mocks.py index 2cae37e..5ddddcc 100644 --- a/tests/cli/data/mocks.py +++ b/tests/cli/data/mocks.py @@ -1,5 +1,6 @@ def mock_data(data_id): class Data: id = data_id - name = 'test name' + name = 'test/name/123' + return Data() diff --git a/tox.ini b/tox.ini index d86fe92..5efffef 100644 --- a/tox.ini +++ b/tox.ini @@ -8,3 +8,7 @@ commands = deps = setuptools>=17.1 +[flake8] +exclude = .tox,./build +filename = *.py +ignore = E501 From 440bccf10b7749d6b6875360e55de2d1d9021bb4 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Thu, 17 Aug 2017 16:06:40 -0400 Subject: [PATCH 7/7] =?UTF-8?q?Bump=20version:=200.10.5=20=E2=86=92=200.10?= =?UTF-8?q?.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1e165ba..60cad97 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.10.5 +current_version = 0.10.6 commit = True tag = False diff --git a/setup.py b/setup.py index b267764..2a49c12 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages, setup project = "floyd-cli" -version = "0.10.5" +version = "0.10.6" with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: long_description = readme.read()