Merge pull request #89 from houqp/release/0.10

release 0.10.6

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.5
+current_version = 0.10.6
 commit = True
 tag = False
 

floyd/cli/data.py

@@ -14,6 +14,7 @@
     opt_to_resume, upload_is_resumable, abort_previous_upload,
     initialize_new_upload, complete_upload
 )
+from floyd.cli.utils import normalize_data_name
 
 
 @click.group()
@@ -97,7 +98,8 @@ def print_data(data_sources):
     headers = ["DATA NAME", "CREATED", "STATUS", "DISK USAGE"]
     data_list = []
     for data_source in data_sources:
-        data_list.append([data_source.name, data_source.created_pretty,
+        data_list.append([normalize_data_name(data_source.name),
+                          data_source.created_pretty,
                           data_source.state, data_source.size])
     floyd_logger.info(tabulate(data_list, headers=headers))
 
@@ -159,18 +161,30 @@ def delete(ids, yes):
             failures = True
             continue
 
-        if not yes and not click.confirm("Delete Data: {}?".format(data_source.name),
+        data_name = normalize_data_name(data_source.name)
+        suffix = data_name.split('/')[-1]
+        if not suffix.isdigit():
+            failures = True
+            floyd_logger.error('%s is not a dataset, skipped.', id)
+            if suffix == 'output':
+                floyd_logger.error('To delete job output, please delete the job itself.')
+            continue
+
+        if not yes and not click.confirm("Delete Data: {}?".format(data_name),
                                          abort=False,
                                          default=False):
-            floyd_logger.info("Data {}: Skipped".format(data_source.name))
+            floyd_logger.info("Data %s: Skipped", data_name)
             continue
 
-        if not DataClient().delete(id):
+        if not DataClient().delete(data_source.id):
             failures = True
+        else:
+            floyd_logger.info("Data %s: Deleted", data_name)
 
     if failures:
         sys.exit(1)
 
+
 data.add_command(clone)
 data.add_command(delete)
 data.add_command(init)

floyd/cli/data_upload_utils.py

@@ -14,6 +14,7 @@
 from floyd.log import logger as floyd_logger
 from floyd.manager.data_config import DataConfigManager
 from floyd.model.data import DataRequest
+from floyd.cli.utils import normalize_data_name
 
 
 class ResourceWaitIter(object):
@@ -168,8 +169,9 @@ def complete_upload(data_config):
             DataConfigManager.set_config(data_config)
 
     # Print output
-    table_output = [["DATA ID", "NAME"],
-                    [data_id, data_config.data_name]]
+    table_output = [["NAME"],
+                    [normalize_data_name(data_config.data_name)]]
+    floyd_logger.info('')
     floyd_logger.info(tabulate(table_output, headers="firstrow"))
 
 

floyd/cli/experiment.py

@@ -5,7 +5,9 @@
 import sys
 
 import floyd
-from floyd.cli.utils import get_module_task_instance_id
+from floyd.cli.utils import (
+    get_module_task_instance_id, normalize_job_name, normalize_data_name
+)
 from floyd.client.experiment import ExperimentClient
 from floyd.client.module import ModuleClient
 from floyd.client.project import ProjectClient
@@ -65,7 +67,8 @@ def print_experiments(experiments):
     headers = ["JOB NAME", "CREATED", "STATUS", "DURATION(s)", "INSTANCE", "DESCRIPTION"]
     expt_list = []
     for experiment in experiments:
-        expt_list.append([experiment.name, experiment.created_pretty, experiment.state,
+        expt_list.append([normalize_job_name(experiment.name),
+                          experiment.created_pretty, experiment.state,
                           experiment.duration_rounded,
                           experiment.instance_type_trimmed, experiment.description])
     floyd_logger.info(tabulate(expt_list, headers=headers))
@@ -99,8 +102,8 @@ def info(id):
     experiment = ExperimentClient().get(id)
     task_instance_id = get_module_task_instance_id(experiment.task_instances)
     task_instance = TaskInstanceClient().get(task_instance_id) if task_instance_id else None
-    table = [["Job name", experiment.name],
-             ["Output name", '%s/output' % experiment.name if task_instance else None],
+    table = [["Job name", normalize_job_name(experiment.name)],
+             ["Output name", normalize_data_name(experiment.name + '/output') if task_instance else None],
              ["Created", experiment.created_pretty],
              ["Status", experiment.state], ["Duration(s)", experiment.duration_rounded],
              ["Instance", experiment.instance_type_trimmed],

floyd/cli/run.py

@@ -8,7 +8,9 @@
 from floyd.constants import DEFAULT_ENV
 from floyd.client.data import DataClient
 from floyd.client.project import ProjectClient
-from floyd.cli.utils import get_mode_parameter, wait_for_url, get_data_name
+from floyd.cli.utils import (
+    get_mode_parameter, wait_for_url, get_data_name, normalize_job_name
+)
 from floyd.client.experiment import ExperimentClient
 from floyd.client.module import ModuleClient
 from floyd.client.env import EnvClient
@@ -133,9 +135,11 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command):
                                            instance_type=instance_type)
     expt_cli = ExperimentClient()
     expt_info = expt_cli.create(experiment_request)
-    floyd_logger.debug("Created job : {}".format(expt_info['id']))
+    floyd_logger.debug("Created job : %s", expt_info['id'])
 
-    table_output = [["JOB NAME"], [expt_info['name']]]
+    job_name = normalize_job_name(expt_info['name'])
+    floyd_logger.info("")
+    table_output = [["JOB NAME"], [job_name]]
     floyd_logger.info(tabulate(table_output, headers="firstrow"))
     floyd_logger.info("")
 
@@ -147,9 +151,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command):
                 if experiment.task_instances:
                     break
             except Exception:
-                floyd_logger.debug("Job not available yet: {}".format(expt_info['id']))
+                floyd_logger.debug("Job not available yet: %s", expt_info['id'])
 
-            floyd_logger.debug("Job not available yet: {}".format(expt_info['id']))
+            floyd_logger.debug("Job not available yet: %s", expt_info['id'])
             sleep(3)
             continue
 
@@ -162,9 +166,9 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command):
                 if open:
                     webbrowser.open(jupyter_url)
             else:
-                floyd_logger.info("\nPath to jupyter notebook: {}".format(jupyter_url))
+                floyd_logger.info("\nPath to jupyter notebook: %s", jupyter_url)
                 floyd_logger.info("Notebook is still loading. View logs to track progress")
-                floyd_logger.info("   floyd logs {}".format(expt_info['name']))
+                floyd_logger.info("   floyd logs %s", job_name)
 
         # Print the path to serving endpoint
         if mode == 'serve':
@@ -177,7 +181,7 @@ def run(ctx, gpu, env, message, data, mode, open, tensorboard, command):
 
     else:
         floyd_logger.info("To view logs enter:")
-        floyd_logger.info("   floyd logs {}".format(expt_info['name']))
+        floyd_logger.info("   floyd logs %s", job_name)
 
 
 def get_command_line(gpu, env, message, data, mode, open, tensorboard, command):

floyd/cli/utils.py

@@ -72,3 +72,26 @@ def get_data_id(data_str):
         return name_or_id
     else:
         return data_str
+
+
+def normalize_data_name(data_name):
+    if data_name.endswith('/output'):
+        name_parts = data_name.split('/')
+        if len(name_parts) <= 4:
+            name_parts.insert(1, 'projects')
+            data_name = '/'.join(name_parts)
+        return data_name
+    else:
+        name_parts = data_name.split('/')
+        if len(name_parts) <= 3:
+            name_parts.insert(1, 'datasets')
+            data_name = '/'.join(name_parts)
+        return data_name
+
+
+def normalize_job_name(job_name):
+    job_name_parts = job_name.split('/')
+    if len(job_name_parts) <= 3:
+        job_name_parts.insert(1, 'projects')
+        job_name = '/'.join(job_name_parts)
+    return job_name

floyd/client/data.py

@@ -90,7 +90,6 @@ def delete(self, data_id):
         try:
             # data delete is a synchronous process, it can take a long time
             self.request("DELETE", self.url + data_id, timeout=60)
-            floyd_logger.info("Data %s: Deleted", data_id)
             return True
         except FloydException as e:
             floyd_logger.error("Data %s: ERROR! %s", data_id, e.message)

floyd/client/experiment.py

@@ -48,10 +48,10 @@ def delete(self, id):
             return True
         except NotFoundException as e:
             floyd_logger.info(
-                    ("Job {}: ERROR! A deletable job with this "
-                     "id was not found. Make sure you have the correct id and "
-                     "that the experiment is not "
-                     "queued or running.".format(id))
+                ("Job {}: ERROR! A deletable job with this "
+                 "id was not found. Make sure you have the correct id and "
+                 "that the experiment is not "
+                 "queued or running.".format(id))
             )
             return False
         except FloydException as e:

floyd/client/project.py

@@ -43,5 +43,5 @@ def exists(self, project_id):
                 return True
             else:
                 return False
-        except FloydException:
+        except NotFoundException:
             return False

floyd/main.py

@@ -10,7 +10,6 @@
 from floyd.cli.run import run
 from floyd.cli.version import upgrade, version
 from floyd.client.version import VersionClient
-from floyd.exceptions import FloydException
 from floyd.log import configure_logger
 
 
@@ -68,4 +67,5 @@ def add_commands(cli):
     cli.add_command(upgrade)
     cli.add_command(version)
 
+
 add_commands(cli)

floyd/model/experiment.py

@@ -68,6 +68,9 @@ def localize_date(self, date):
             date = utc.localize(date)
         return date.astimezone(PST_TIMEZONE)
 
+    def to_dict(self):
+        return self.__dict__
+
     @property
     def created_pretty(self):
         return pretty_date(self.created)

setup.py

@@ -3,7 +3,7 @@
 from setuptools import find_packages, setup
 
 project = "floyd-cli"
-version = "0.10.5"
+version = "0.10.6"
 
 with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme:
     long_description = readme.read()

tests/cli/data/mocks.py

@@ -1,5 +1,6 @@
 def mock_data(data_id):
     class Data:
         id = data_id
-        name = 'test name'
+        name = 'test/name/123'
+
     return Data()

tests/cli/run/test_run.py

@@ -19,13 +19,15 @@ def setUp(self):
     @patch('floyd.cli.run.ExperimentConfigManager.set_config')
     @patch('floyd.cli.run.ModuleClient.create', return_value='module_id')
     @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id')
+    @patch('floyd.cli.run.ProjectClient.exists', return_value=True)
     def test_with_no_data(self,
                           create_experiment,
                           create_module,
                           set_config,
                           get_config,
                           get_access_token,
-                          get_all_env):
+                          get_all_env,
+                          exists):
         """
         Simple experiment with no data attached
         """
@@ -37,12 +39,14 @@ def test_with_no_data(self,
     @patch('floyd.cli.run.ExperimentConfigManager.set_config')
     @patch('floyd.cli.run.ModuleClient.create', return_value='module_id')
     @patch('floyd.cli.run.ExperimentClient.create', return_value='expt_id')
+    @patch('floyd.cli.run.ProjectClient.exists', return_value=True)
     def test_with_multiple_data_ids(self,
                                     create_experiment,
                                     create_module,
                                     set_config,
                                     get_config,
-                                    get_access_token):
+                                    get_access_token,
+                                    exists):
         """
         Simple experiment with no data attached
         """

tests/cli/utils_test.py

@@ -0,0 +1,18 @@
+import unittest
+
+
+class TestCliUtil(unittest.TestCase):
+    """
+    Tests cli utils helper functions
+    """
+    def test_normalize_data_name(self):
+        from floyd.cli.utils import normalize_data_name
+        assert normalize_data_name('foo/bar/1') == 'foo/datasets/bar/1'
+        assert normalize_data_name('foo/datasets/bar/1') == 'foo/datasets/bar/1'
+        assert normalize_data_name('foo/bar/1/output') == 'foo/projects/bar/1/output'
+        assert normalize_data_name('foo/projects/bar/1/output') == 'foo/projects/bar/1/output'
+
+    def test_normalize_job_name(self):
+        from floyd.cli.utils import normalize_job_name
+        assert normalize_job_name('foo/bar/1') == 'foo/projects/bar/1'
+        assert normalize_job_name('foo/projects/bar/1') == 'foo/projects/bar/1'

tox.ini

@@ -8,3 +8,7 @@ commands =
 deps =
     setuptools>=17.1
 
+[flake8]
+exclude = .tox,./build
+filename = *.py
+ignore = E501