diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1e165ba..60cad97 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.10.5 +current_version = 0.10.6 commit = True tag = False diff --git a/floyd/cli/data.py b/floyd/cli/data.py index 08781bf..ce10a4a 100644 --- a/floyd/cli/data.py +++ b/floyd/cli/data.py @@ -14,6 +14,7 @@ opt_to_resume, upload_is_resumable, abort_previous_upload, initialize_new_upload, complete_upload ) +from floyd.cli.utils import normalize_data_name @click.group() @@ -97,7 +98,8 @@ def print_data(data_sources): headers = ["DATA NAME", "CREATED", "STATUS", "DISK USAGE"] data_list = [] for data_source in data_sources: - data_list.append([data_source.name, data_source.created_pretty, + data_list.append([normalize_data_name(data_source.name), + data_source.created_pretty, data_source.state, data_source.size]) floyd_logger.info(tabulate(data_list, headers=headers)) @@ -159,18 +161,30 @@ def delete(ids, yes): failures = True continue - if not yes and not click.confirm("Delete Data: {}?".format(data_source.name), + data_name = normalize_data_name(data_source.name) + suffix = data_name.split('/')[-1] + if not suffix.isdigit(): + failures = True + floyd_logger.error('%s is not a dataset, skipped.', id) + if suffix == 'output': + floyd_logger.error('To delete job output, please delete the job itself.') + continue + + if not yes and not click.confirm("Delete Data: {}?".format(data_name), abort=False, default=False): - floyd_logger.info("Data {}: Skipped".format(data_source.name)) + floyd_logger.info("Data %s: Skipped", data_name) continue - if not DataClient().delete(id): + if not DataClient().delete(data_source.id): failures = True + else: + floyd_logger.info("Data %s: Deleted", data_name) if failures: sys.exit(1) + data.add_command(clone) data.add_command(delete) data.add_command(init) diff --git a/floyd/cli/data_upload_utils.py b/floyd/cli/data_upload_utils.py index 01ad928..89cf4c5 100644 --- a/floyd/cli/data_upload_utils.py +++ b/floyd/cli/data_upload_utils.py @@ -14,6 +14,7 @@ from floyd.log import logger as floyd_logger from floyd.manager.data_config import DataConfigManager from floyd.model.data import DataRequest +from floyd.cli.utils import normalize_data_name class ResourceWaitIter(object): @@ -168,8 +169,9 @@ def complete_upload(data_config): DataConfigManager.set_config(data_config) # Print output - table_output = [["DATA ID", "NAME"], - [data_id, data_config.data_name]] + table_output = [["NAME"], + [normalize_data_name(data_config.data_name)]] + floyd_logger.info('') floyd_logger.info(tabulate(table_output, headers="firstrow")) diff --git a/floyd/cli/experiment.py b/floyd/cli/experiment.py index 2ab41ad..8c194ca 100644 --- a/floyd/cli/experiment.py +++ b/floyd/cli/experiment.py @@ -5,7 +5,9 @@ import sys import floyd -from floyd.cli.utils import get_module_task_instance_id +from floyd.cli.utils import ( + get_module_task_instance_id, normalize_job_name, normalize_data_name +) from floyd.client.experiment import ExperimentClient from floyd.client.module import ModuleClient from floyd.client.project import ProjectClient @@ -65,7 +67,8 @@ def print_experiments(experiments): headers = ["JOB NAME", "CREATED", "STATUS", "DURATION(s)", "INSTANCE", "DESCRIPTION"] expt_list = [] for experiment in experiments: - expt_list.append([experiment.name, experiment.created_pretty, experiment.state, + expt_list.append([normalize_job_name(experiment.name), + experiment.created_pretty, experiment.state, experiment.duration_rounded, experiment.instance_type_trimmed, experiment.description]) floyd_logger.info(tabulate(expt_list, headers=headers)) @@ -99,8 +102,8 @@ def info(id): experiment = ExperimentClient().get(id) task_instance_id = get_module_task_instance_id(experiment.task_instances) task_instance = TaskInstanceClient().get(task_instance_id) if task_instance_id else None - table = [["Job name", experiment.name], - ["Output name", '%s/output' % experiment.name if task_instance else None], + table = [["Job name", normalize_job_name(experiment.name)], + ["Output name", normalize_data_name(experiment.name + '/output') if task_instance else None], ["Created", experiment.created_pretty], ["Status", experiment.state], ["Duration(s)", experiment.duration_rounded], ["Instance", experiment.instance_type_trimmed], diff --git a/floyd/cli/run.py b/floyd/cli/run.py index 21b5989..c2f42c8 100644 --- a/floyd/cli/run.py +++ b/floyd/cli/run.py @@ -8,7 +8,9 @@ from floyd.constants import DEFAULT_ENV from floyd.client.data import DataClient from floyd.client.project import ProjectClient -from floyd.cli.utils import get_mode_parameter, wait_for_url, get_data_name +from floyd.cli.utils import ( + get_mode_parameter, wait_for_url, get_data_name, normalize_job_name +) from floyd.client.experiment import ExperimentClient from floyd.client.module import ModuleClient from floyd.client.env import EnvClient @@ -133,9 +135,11 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): instance_type=instance_type) expt_cli = ExperimentClient() expt_info = expt_cli.create(experiment_request) - floyd_logger.debug("Created job : {}".format(expt_info['id'])) + floyd_logger.debug("Created job : %s", expt_info['id']) - table_output = [["JOB NAME"], [expt_info['name']]] + job_name = normalize_job_name(expt_info['name']) + floyd_logger.info("") + table_output = [["JOB NAME"], [job_name]] floyd_logger.info(tabulate(table_output, headers="firstrow")) floyd_logger.info("") @@ -147,9 +151,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): if experiment.task_instances: break except Exception: - floyd_logger.debug("Job not available yet: {}".format(expt_info['id'])) + floyd_logger.debug("Job not available yet: %s", expt_info['id']) - floyd_logger.debug("Job not available yet: {}".format(expt_info['id'])) + floyd_logger.debug("Job not available yet: %s", expt_info['id']) sleep(3) continue @@ -162,9 +166,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): if open: webbrowser.open(jupyter_url) else: - floyd_logger.info("\nPath to jupyter notebook: {}".format(jupyter_url)) + floyd_logger.info("\nPath to jupyter notebook: %s", jupyter_url) floyd_logger.info("Notebook is still loading. View logs to track progress") - floyd_logger.info(" floyd logs {}".format(expt_info['name'])) + floyd_logger.info(" floyd logs %s", job_name) # Print the path to serving endpoint if mode == 'serve': @@ -177,7 +181,7 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command): else: floyd_logger.info("To view logs enter:") - floyd_logger.info(" floyd logs {}".format(expt_info['name'])) + floyd_logger.info(" floyd logs %s", job_name) def get_command_line(gpu, env, message, data, mode, open, tensorboard, command): diff --git a/floyd/cli/utils.py b/floyd/cli/utils.py index bfad5bb..fb97eaf 100644 --- a/floyd/cli/utils.py +++ b/floyd/cli/utils.py @@ -72,3 +72,26 @@ def get_data_id(data_str): return name_or_id else: return data_str + + +def normalize_data_name(data_name): + if data_name.endswith('/output'): + name_parts = data_name.split('/') + if len(name_parts) <= 4: + name_parts.insert(1, 'projects') + data_name = '/'.join(name_parts) + return data_name + else: + name_parts = data_name.split('/') + if len(name_parts) <= 3: + name_parts.insert(1, 'datasets') + data_name = '/'.join(name_parts) + return data_name + + +def normalize_job_name(job_name): + job_name_parts = job_name.split('/') + if len(job_name_parts) <= 3: + job_name_parts.insert(1, 'projects') + job_name = '/'.join(job_name_parts) + return job_name diff --git a/floyd/client/data.py b/floyd/client/data.py index 36ef1aa..6eca0dd 100644 --- a/floyd/client/data.py +++ b/floyd/client/data.py @@ -90,7 +90,6 @@ def delete(self, data_id): try: # data delete is a synchronous process, it can take a long time self.request("DELETE", self.url + data_id, timeout=60) - floyd_logger.info("Data %s: Deleted", data_id) return True except FloydException as e: floyd_logger.error("Data %s: ERROR! %s", data_id, e.message) diff --git a/floyd/client/experiment.py b/floyd/client/experiment.py index a4875bb..03490d0 100644 --- a/floyd/client/experiment.py +++ b/floyd/client/experiment.py @@ -48,10 +48,10 @@ def delete(self, id): return True except NotFoundException as e: floyd_logger.info( - ("Job {}: ERROR! A deletable job with this " - "id was not found. Make sure you have the correct id and " - "that the experiment is not " - "queued or running.".format(id)) + ("Job {}: ERROR! A deletable job with this " + "id was not found. Make sure you have the correct id and " + "that the experiment is not " + "queued or running.".format(id)) ) return False except FloydException as e: diff --git a/floyd/client/project.py b/floyd/client/project.py index fe781e4..e2e11e0 100644 --- a/floyd/client/project.py +++ b/floyd/client/project.py @@ -43,5 +43,5 @@ def exists(self, project_id): return True else: return False - except FloydException: + except NotFoundException: return False diff --git a/floyd/main.py b/floyd/main.py index 499409b..d246ad5 100644 --- a/floyd/main.py +++ b/floyd/main.py @@ -10,7 +10,6 @@ from floyd.cli.run import run from floyd.cli.version import upgrade, version from floyd.client.version import VersionClient -from floyd.exceptions import FloydException from floyd.log import configure_logger @@ -68,4 +67,5 @@ def add_commands(cli): cli.add_command(upgrade) cli.add_command(version) + add_commands(cli) diff --git a/floyd/model/experiment.py b/floyd/model/experiment.py index e8b783d..82daa48 100644 --- a/floyd/model/experiment.py +++ b/floyd/model/experiment.py @@ -68,6 +68,9 @@ def localize_date(self, date): date = utc.localize(date) return date.astimezone(PST_TIMEZONE) + def to_dict(self): + return self.__dict__ + @property def created_pretty(self): return pretty_date(self.created) diff --git a/setup.py b/setup.py index b267764..2a49c12 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages, setup project = "floyd-cli" -version = "0.10.5" +version = "0.10.6" with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: long_description = readme.read() diff --git a/tests/cli/data/mocks.py b/tests/cli/data/mocks.py index 2cae37e..5ddddcc 100644 --- a/tests/cli/data/mocks.py +++ b/tests/cli/data/mocks.py @@ -1,5 +1,6 @@ def mock_data(data_id): class Data: id = data_id - name = 'test name' + name = 'test/name/123' + return Data() diff --git a/tests/cli/run/test_run.py b/tests/cli/run/test_run.py index 8232403..3f0a6a0 100644 --- a/tests/cli/run/test_run.py +++ b/tests/cli/run/test_run.py @@ -19,13 +19,15 @@ def setUp(self): @patch('floyd.cli.run.ExperimentConfigManager.set_config') @patch('floyd.cli.run.ModuleClient.create', return_value='module_id') @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id') + @patch('floyd.cli.run.ProjectClient.exists', return_value=True) def test_with_no_data(self, create_experiment, create_module, set_config, get_config, get_access_token, - get_all_env): + get_all_env, + exists): """ Simple experiment with no data attached """ @@ -37,12 +39,14 @@ def test_with_no_data(self, @patch('floyd.cli.run.ExperimentConfigManager.set_config') @patch('floyd.cli.run.ModuleClient.create', return_value='module_id') @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id') + @patch('floyd.cli.run.ProjectClient.exists', return_value=True) def test_with_multiple_data_ids(self, create_experiment, create_module, set_config, get_config, - get_access_token): + get_access_token, + exists): """ Simple experiment with no data attached """ diff --git a/tests/cli/utils_test.py b/tests/cli/utils_test.py new file mode 100644 index 0000000..cbe02fd --- /dev/null +++ b/tests/cli/utils_test.py @@ -0,0 +1,18 @@ +import unittest + + +class TestCliUtil(unittest.TestCase): + """ + Tests cli utils helper functions + """ + def test_normalize_data_name(self): + from floyd.cli.utils import normalize_data_name + assert normalize_data_name('foo/bar/1') == 'foo/datasets/bar/1' + assert normalize_data_name('foo/datasets/bar/1') == 'foo/datasets/bar/1' + assert normalize_data_name('foo/bar/1/output') == 'foo/projects/bar/1/output' + assert normalize_data_name('foo/projects/bar/1/output') == 'foo/projects/bar/1/output' + + def test_normalize_job_name(self): + from floyd.cli.utils import normalize_job_name + assert normalize_job_name('foo/bar/1') == 'foo/projects/bar/1' + assert normalize_job_name('foo/projects/bar/1') == 'foo/projects/bar/1' diff --git a/tox.ini b/tox.ini index d86fe92..5efffef 100644 --- a/tox.ini +++ b/tox.ini @@ -8,3 +8,7 @@ commands = deps = setuptools>=17.1 +[flake8] +exclude = .tox,./build +filename = *.py +ignore = E501