Merge pull request #154 from alces-software/feat/run-namespaces

Feat/run namespaces

README.md

@@ -16,34 +16,27 @@ exit - exits the CLI.
 help - displays help for the current level's commands.
 
 run - command group used for running tools - works in parallel across nodes.
- - run tool [NAMESPACE(S)] TOOL NODE(S) [ARGUMENTS]
+ - run <TOOL...> NODE(S) [ARGUMENTS]
     - Runs tool TOOL on NODE(S).
-    - A tool's namespaces must be specified before its name.
-    - If a tool is selected for a single node it will be automatically ran
-      in interactive mode.
     - If a tool marked as being interactive only (see 'Adding Tools') and you
       attempt to run it on more than one node it will cancel and an error will
       be thrown.
-    - Optionally, arguments can be provided.
- - run family FAMILY NODE(S)
-    - Runs tool-family FAMILY on NODE(S).
+    - Additionally a tool marked as interactive will open an interactive ssh
+      session with the node when ran
+    - Optionally, arguments for the shell command can be provided.
 
 view - inspect execution history, statuses, groups, and tools.
  - view group [GROUP]
     - Lists all the nodes in group GROUP
     - If no group is given, a list of all groups can be found in the command's
       help display
- - view tool [NAMESPACE(S)] [TOOL]
-    - Shows info about the tool at NAMESPACE(S)/TOOL
-    - Displays the tool's name, description, command, families, whether it must
+ - view tool [TOOL...]
+    - Shows info about tool TOOL
+    - Displays the tool's name, description, command, whether it must
       be ran interactively and the contents of its working directory
-    - If no tool is given, in the command's help display it lists the availible tools
-      and sub-namespaces of the given namespace(s).
+    - If a tool consisting of other tools is given, in the command's help display
+      it lists the sub-tools of that tool.
     - If NAMESPACE(S) is not given, it lists at the highest level `tools` directory.
- - view family [FAMILY]
-    - Displays the members of the tool family FAMILY, as well as their order of
-      execution
-    - If FAMILY is not given, it lists all the system's tool families.
  - view result JOB-ID
     - Shows the result (exit code, stdout, stderr) of an instance of a single
       tool running on a single node.
@@ -114,7 +107,7 @@ single line output. The full results are logged to the database.
 
 Tools are automatically picked up from config files stored at:
 
-`/var/lib/adminware/tools/[<optional-namespace>/].../<tool-name>/config.yaml`
+`/var/lib/adminware/tools/[<optional-directories>/].../<tool-name>/config.yaml`
 
 The config.yaml files cannot have directories as their siblings, although
 there can be other files in the same directories.
@@ -125,23 +118,13 @@ The config files should follow the following format:
 # `./script.rb`.
 command: command_to_run
 
-# Full help text for this tool, it will be picked up and displayed in full when `help` is
-# displayed for this tool in `run` commands.
+# Full help text for this tool, it will be picked up and displayed in full when `help
+# is displayed for this tool in `run` commands.
 help: command_help
 
-# A list of any families that the tool is in. A family is a group of tools that
-# can be executed with a single statement using `run family`. Tools within a
-# family are executed in alphanumeric order and each tool is executed on every node
-# before the second tool is executed on any.
-families:
- - family1
- - family2
- - etc..
-
-# A flag stating that this tool's command should never be ran in a non-interactive
-# shell. It's value must be "True" for this to take effect. If a tool is marked as
-# interactive only it will be excluded from tool families and from being run on more
-# than one node at once. If it is attempted to run in on more than one node an error
+# A flag stating that this tool's command is only to be ran in an interactive
+# shell. It's value must be "True" for this to take effect. If a tool marked as
+# interactive only is ran as part of another tool or on more than one node an error
 # will be thrown.
 interactive: True
 

src/appliance_cli/command_generation.py

@@ -131,7 +131,7 @@ def _parse_simple_command_config(ancestor_commands, config, callback):
 
     # Define function to be passed as 'callback' parameter to click.Command,
     # transforming its arguments suitable to be passed to our own callback.
-    def click_callback(**params):
+    def click_callback(ctx, **params):
         argument_values = [
             params[arg_name] for arg_name in arguments.keys()
         ]
@@ -147,11 +147,16 @@ def click_callback(**params):
         new_commands = deepcopy(ancestor_commands)
         new_options = deepcopy(options)
 
-        callback(new_commands, argument_values, new_options)
+        callback_args = [new_commands, argument_values, new_options]
+
+        if config.get('pass_context'):
+            callback(*callback_args, ctx=ctx)
+        else:
+            callback(*callback_args)
 
     return {
         'params': click_params,
-        'callback': click_callback
+        'callback': click.pass_context(click_callback)
     }
 
 
@@ -200,6 +205,15 @@ def _parse_group_command_config(ancestor_commands, config, callback):
         in config['commands'].items()
     }
 
-    return {
-        'commands': commands
-    }
+    config.setdefault('invoke_without_command', False)
+
+    return_hash = { 'commands': commands }
+
+    if config['invoke_without_command']:
+        return_hash['invoke_without_command'] = True
+        return_hash = {
+            **return_hash,
+            **_parse_simple_command_config(ancestor_commands, config, callback)
+        }
+
+    return return_hash

src/commands/run.py

@@ -19,10 +19,6 @@ def add_commands(appliance):
     def run():
         pass
 
-    @run.group(help='Run a tool over a batch of nodes')
-    def tool():
-        pass
-
     node_group_options = {
         ('--node', '-n'): {
             'help': 'Specify a node, repeat the flag for multiple',
@@ -42,50 +38,39 @@ def tool():
         'options': node_group_options
     }
     runner_group = {
-        'help': (lambda names: "Run further tools: '{}'".format(' '.join(names)))
+        'help': (lambda names: "Run tools in {}".format(' '.join(names))),
+        'invoke_without_command': True,
+        'options': node_group_options,
+        'pass_context': True
     }
 
-    @Config.commands(tool, command = runner_cmd, group = runner_group)
+    @Config.commands(run, command = runner_cmd, group = runner_group)
     @cli_utils.with__node__group
-    def runner(config, argv, _, nodes):
-        batch = Batch(config = config.path, arguments = (argv[0] or ''))
-        batch.build_jobs(*nodes)
-        if batch.is_interactive():
-            if len(batch.jobs) == 1:
-                execute_threaded_batches([batch], quiet = True)
-            elif batch.jobs:
-                raise ClickException('''
+    def runner(configs, argv, _, nodes):
+        if not argv: argv = [None]
+        if len(configs) > 1:
+            for config in configs:
+                if config.interactive():
+                    raise ClickException('''
+'{}' is an interactive tool and cannot be ran as part of a group
+'''.format(config.__name__()).strip())
+        for config in configs:
+            batch = Batch(config = config.path, arguments = (argv[0] or ''))
+            batch.build_jobs(*nodes)
+            if batch.is_interactive():
+                if len(batch.jobs) == 1:
+                    execute_threaded_batches([batch], quiet = True)
+                elif batch.jobs:
+                    raise ClickException('''
 '{}' is an interactive tool and can only be ran on a single node
 '''.format(config.name()).strip())
+                else:
+                    raise ClickException('Please specify a node with --node')
+            elif batch.jobs:
+                report = batch.config_model.report
+                execute_threaded_batches([batch], quiet = report)
             else:
-                raise ClickException('Please specify a node with --node')
-        elif batch.jobs:
-            report = batch.config_model.report
-            execute_threaded_batches([batch], quiet = report)
-        else:
-            raise ClickException('Please give either --node or --group')
-
-    @run.group(help='Run a family of tools on node(s) or group(s)')
-    def family(): pass
-
-    family_runner = {
-        'help': 'Runs the tool over the group',
-        'options': node_group_options
-    }
-
-    @Config.family_commands(family, command = family_runner)
-    @cli_utils.with__node__group
-    def family_runner(callstack, _a, _o, nodes):
-        family = callstack[0]
-        if not nodes:
-            raise ClickException('Please give either --node or --group')
-        batches = []
-        for config in Config.all_families()[family]:
-            #create batch w/ relevant config for tool
-            batch = Batch(config = config.path)
-            batch.build_jobs(*nodes)
-            batches += [batch]
-        execute_threaded_batches(batches)
+                raise ClickException('Please give either --node or --group')
 
     def execute_threaded_batches(batches, quiet = False):
         def run_print(string):

src/commands/view.py

@@ -53,29 +53,17 @@ def tool():
     tool_cmd = { 'help': "See tool's details" }
     tool_grp = { 'help': 'List details for further tools' }
     @Config.commands(tool, command = tool_cmd, group = tool_grp)
-    def get_tool_info(config, _a, _o):
+    def get_tool_info(configs, _a, _o):
+        config = configs[0]
         table_data = [
             ['Name', config.name()],
             ['Description', config.help()],
             ['Shell Command', config.command()],
             ['Interactive', 'Yes' if config.interactive_only() else 'No'],
-            ['Families', '\n'.join(config.families())],
             ['Working Directory', '\n'.join(config.working_files())]
         ]
         display_table([], table_data)
 
-    @view.group(help="See more details about your tool families")
-    def family():
-        pass
-
-    family_view_command = { 'help': 'View the tools in this family' }
-    @Config.family_commands(family, command = family_view_command)
-    def get_family_info(callstack, _a, _o):
-        family = callstack[0]
-        output = "{}\n{}".format(family, " --> ".join(list(map(lambda x: x.__name__(),
-                                                               Config.all_families()[family]))))
-        click.echo_via_pager(output)
-
     @view.command(name='node-status', help='View the execution history of a single node')
     @click.argument('node', type=str)
     # note: this works on tool location, not tool name.
@@ -109,7 +97,8 @@ def tool_status():
     tool_status_cmd = { 'help': 'List the status across the nodes' }
     tool_status_grp = { 'help': 'See the status of further tools' }
     @Config.commands(tool_status, command = tool_status_cmd, group = tool_status_grp)
-    def tool_status_runner(config, _a, _o):
+    def tool_status_runner(configs, _a, _o):
+        config = configs[0]
         session = Session()
         # Returns the most recent job for each node and the number of times the tool's been ran
         # => [(latest_job1, count1), (lastest_job2, count2), ...]
@@ -153,7 +142,8 @@ def tool_history():
     tool_history_cmd = { 'help': 'List the history across the nodes' }
     tool_history_grp = { 'help': 'See the history of further tools' }
     @Config.commands(tool_history, command = tool_history_cmd, group = tool_history_grp)
-    def tool_history_runner(config, _a, _o):
+    def tool_history_runner(configs, _a, _o):
+        config = configs[0]
         session = Session()
         job_data = session.query(Job)\
                           .select_from(Batch)\

src/models/config.py

@@ -1,5 +1,6 @@
 import yaml
 import re
+import click
 from os.path import basename, dirname
 
 import os.path
@@ -19,42 +20,35 @@ class ConfigCallback():
             def __init__(self, callback_func):
                 self.callback = callback_func
 
-            def run(self, callstack, *a):
-                path = os.path.join(CONFIG_DIR, *callstack, 'config.yaml')
-                self.callback(Config(path), *a)
+            def run(self, callstack, *a, ctx = None):
+                if not ctx:
+                    path = os.path.join(CONFIG_DIR, *callstack, 'config.yaml')
+                    self.callback([Config(path)], *a)
+                if ctx and not ctx.invoked_subcommand:
+                    parts = [CONFIG_DIR, *callstack, '*/config.yaml']
+                    paths = glob(os.path.join(*parts))
+                    if not paths:
+                        raise click.ClickException("""
+No tools found in '{}'
+""".format('/'.join(callstack)).strip())
+                    configs = list(map(lambda x: Config(x), paths))
+                    self.callback(configs, *a)
 
         def __commands(config_callback):
             config_hash = Config.hashify_all(subcommand_key = 'commands', **kwargs)
             callback = ConfigCallback(config_callback).run
             generate_commands(root_command, config_hash, callback)
         return __commands
 
-    def family_commands(root_command, **kwargs):
-        def __family_commands(callback):
-            families_hash = Config.hashify_all_families(**kwargs)
-            generate_commands(root_command, families_hash, callback)
-        return __family_commands
-
     @lru_cache()
     def cache(*a, **kw): return Config(*a, **kw)
 
     def all():
         glob_path = os.path.join(CONFIG_DIR, '**/*/config.yaml')
         return list(map(lambda p: Config.cache(p), glob(glob_path, recursive=True)))
 
-
-    @lru_cache()
-    def all_families():
-        combined_hash = {}
-        sorted_configs = sorted(Config.all(), key = lambda x: x.__name__())
-        for config in sorted_configs:
-            for family in config.families():
-                combined_hash.setdefault(family, [])
-                combined_hash[family] += [config]
-        return combined_hash
-
     # The commands are hashed into the following structure
-    # NOTES: `command` and `group both supports callable objects as a means
+    # NOTES: `command` and `group` both supports callable objects as a means
     #        to customize the hashes. They are called with:
     #          - command: The config object
     #          - group: The current name
@@ -85,19 +79,6 @@ def build_group_hashes():
 
         return combined_hash[subcommand_key]
 
-    # Generates a similar hash as above but for the command families
-    # Callable objects are called with the family name
-    #   {
-    #       familyX: **<command>,
-    #       ...
-    #   }
-    def hashify_all_families(command = {}):
-        combined_hash = {}
-        for family in Config.all_families():
-            family_hash = combined_hash.setdefault(family, {})
-            Config.__copy_values(command, family_hash, family)
-        return combined_hash
-
     def __copy_values(source, target, args):
         for k, v in source.items():
             target[k] = (v(args) if callable(v) else v)
@@ -138,12 +119,6 @@ def help(self):
         if not self.data['help']: self.data['help'] = default
         return self.data['help']
 
-    def families(self):
-        default = ''
-        self.data.setdefault('families', default)
-        if not self.data['families']: self.data['families'] = default
-        return self.data['families']
-
     # TODO: Deprecated, avoid usage
     def interactive_only(self):
         return self.interactive()