Implement dry-run for install/upgrade process

kubemarine/apt.py

@@ -32,15 +32,17 @@ def backup_repo(group: NodeGroup) -> Optional[RunnersGroupResult]:
     if not group.cluster.inventory['services']['packages']['package_manager']['replace-repositories']:
         group.cluster.log.debug("Skipped - repos replacement disabled in configuration")
         return None
+    dry_run = utils.check_dry_run_status_active(group.cluster)
     # all files in directory will be renamed: xxx.repo -> xxx.repo.bak
     # if there already any files with ".bak" extension, they should not be renamed to ".bak.bak"!
     return group.sudo("find /etc/apt/ -type f -name '*.list' | "
-                      "sudo xargs -t -iNAME mv -bf NAME NAME.bak")
+                      "sudo xargs -t -iNAME mv -bf NAME NAME.bak", dry_run=dry_run)
 
 
 def add_repo(group: NodeGroup, repo_data: Union[List[str], Dict[str, dict], str]) -> RunnersGroupResult:
-    create_repo_file(group, repo_data, get_repo_file_name())
-    return group.sudo(DEBIAN_HEADERS + 'apt clean && sudo apt update')
+    dry_run = utils.check_dry_run_status_active(group.cluster)
+    create_repo_file(group, repo_data, get_repo_file_name(), dry_run)
+    return group.sudo(DEBIAN_HEADERS + 'apt clean && sudo apt update', dry_run=dry_run)
 
 
 def get_repo_file_name() -> str:
@@ -49,17 +51,16 @@ def get_repo_file_name() -> str:
 
 def create_repo_file(group: AbstractGroup[RunResult],
                      repo_data: Union[List[str], Dict[str, dict], str],
-                     repo_file: str) -> None:
+                     repo_file: str, dry_run=False) -> None:
     # if repo_data is list, then convert it to string using join
     if isinstance(repo_data, list):
         repo_data_str = "\n".join(repo_data) + "\n"
     elif isinstance(repo_data, dict):
         raise Exception("Not supported repositories format for apt package manager")
     else:
         repo_data_str = utils.read_external(repo_data)
-
     repo_data_stream = io.StringIO(repo_data_str)
-    group.put(repo_data_stream, repo_file, sudo=True)
+    group.put(repo_data_stream, repo_file, sudo=True, dry_run=dry_run)
 
 
 def clean(group: NodeGroup) -> RunnersGroupResult:
@@ -90,7 +91,7 @@ def install(group: AbstractGroup[GROUP_RUN_TYPE], include: Union[str, List[str]]
 
     command = get_install_cmd(include, exclude)
 
-    return group.sudo(command, callback=callback)
+    return group.sudo(command, callback=callback, dry_run=utils.check_dry_run_status_active(group.cluster))
 
 
 def remove(group: AbstractGroup[GROUP_RUN_TYPE], include: Union[str, List[str]] = None, exclude: Union[str, List[str]] = None,
@@ -107,7 +108,7 @@ def remove(group: AbstractGroup[GROUP_RUN_TYPE], include: Union[str, List[str]]
             exclude = ','.join(exclude)
         command += ' --exclude=%s' % exclude
 
-    return group.sudo(command, warn=warn, hide=hide)
+    return group.sudo(command, warn=warn, hide=hide, dry_run=utils.check_dry_run_status_active(group.cluster))
 
 
 def upgrade(group: AbstractGroup[GROUP_RUN_TYPE], include: Union[str, List[str]] = None,
@@ -125,7 +126,7 @@ def upgrade(group: AbstractGroup[GROUP_RUN_TYPE], include: Union[str, List[str]]
             exclude = ','.join(exclude)
         command += ' --exclude=%s' % exclude
 
-    return group.sudo(command)
+    return group.sudo(command, dry_run=utils.check_dry_run_status_active(group.cluster))
 
 
 def no_changes_found(action: str, result: RunnersResult) -> bool:

kubemarine/audit.py

@@ -68,6 +68,9 @@ def install(group: NodeGroup) -> Optional[RunnersGroupResult]:
     else:
         log.debug(f'Auditd package is not installed on {not_installed_hosts}, installing...')
 
+    if utils.check_dry_run_status_active(cluster):
+        return None
+
     collector = CollectorCallback(cluster)
     with cluster.make_group(not_installed_hosts).new_executor() as exe:
         for node in exe.group.get_ordered_members_list():
@@ -93,7 +96,10 @@ def apply_audit_rules(group: NodeGroup) -> RunnersGroupResult:
     rules_content = " \n".join(cluster.inventory['services']['audit']['rules'])
     utils.dump_file(cluster, rules_content, 'audit.rules')
 
+    if utils.check_dry_run_status_active(group.cluster):
+        return None
     collector = CollectorCallback(cluster)
+
     with group.new_executor() as exe:
         for node in exe.group.get_ordered_members_list():
             host = node.get_host()

kubemarine/core/cluster.py

@@ -341,7 +341,8 @@ def dump_finalized_inventory(self) -> None:
         data = yaml.dump(inventory_for_dump)
         finalized_filename = "cluster_finalized.yaml"
         utils.dump_file(self, data, finalized_filename)
-        utils.dump_file(self, data, finalized_filename, dump_location=False)
+        if not utils.check_dry_run_status_active(self):
+            utils.dump_file(self, data, finalized_filename, dump_location=False)
 
     def preserve_inventory(self) -> None:
         self.log.debug("Start preserving of the information about the procedure.")

kubemarine/core/executor.py

@@ -388,7 +388,6 @@ def get_last_results(self) -> Dict[str, TokenizedResult]:
     def flush(self) -> None:
         """
         Flushes the connections' queue and returns grouped result
-
         :return: grouped tokenized results per connection.
         """
         self._check_closed()

kubemarine/core/flow.py

@@ -133,8 +133,8 @@ def run_actions(resources: res.DynamicResources, actions: Sequence[action.Action
                     timestamp = utils.get_current_timestamp_formatted()
                     inventory_file_basename = os.path.basename(resources.inventory_filepath)
                     utils.dump_file(context, stream, "%s_%s" % (inventory_file_basename, str(timestamp)))
-
-            resources.recreate_inventory()
+            if not utils.check_dry_run_status_active(last_cluster):
+                resources.recreate_inventory()
             _post_process_actions_group(last_cluster, context, successfully_performed)
             successfully_performed = []
             last_cluster = None
@@ -186,9 +186,8 @@ def run_tasks(resources: res.DynamicResources, tasks, cumulative_points=None, ta
     cluster = resources.cluster()
 
     if args.get('without_act', False):
+        cluster.context["dry_run"] = True
         resources.context['preserve_inventory'] = False
-        cluster.log.debug('\nFurther acting manually disabled')
-        return
 
     init_tasks_flow(cluster)
     run_tasks_recursive(tasks, final_list, cluster, cumulative_points, [])
@@ -320,7 +319,7 @@ def new_common_parser(cli_help: str) -> argparse.ArgumentParser:
                         help='define main cluster configuration file')
 
     parser.add_argument('--ansible-inventory-location',
-                        default='./ansible-inventory.ini',
+                        default='ansible-inventory.ini',
                         help='auto-generated ansible-compatible inventory file location')
 
     parser.add_argument('--dump-location',

kubemarine/core/group.py

@@ -24,6 +24,8 @@
     Callable, Dict, List, Union, Any, TypeVar, Mapping, Iterator, Optional, Iterable, Generic, Set, cast
 )
 
+import fabric
+
 from kubemarine.core import utils, log, errors
 from kubemarine.core.executor import (
     RawExecutor, Token, GenericResult, RunnersResult, HostToResult, Callback, TokenizedResult,
@@ -266,6 +268,36 @@ def result(self) -> RunnersGroupResult:
 GROUP_SELF = TypeVar('GROUP_SELF', bound='AbstractGroup[Union[RunnersGroupResult, Token]]')
 
 
+def _handle_dry_run(fn: Callable) -> Callable:
+    """
+    Method is a decorator that handles internal streaming of output (hide=False) of fabric (invoke).
+    Note! This decorator should be the outermost.
+    :param fn: Origin function to apply annotation to
+    :return: Validation wrapper function
+    """
+    def do_dry_run(self: NodeGroup, *args, **kwargs) -> NodeGroupResult:
+        results: Dict[str, str] = {}
+
+        if kwargs.get("dry_run"):
+            if fn.__name__ == "put":
+                self.cluster.log.verbose("Local file \"%s\" is being transferred to remote file \"%s\" on nodes %s with options %s"
+                                         % (args[0], args[1], list(self.nodes), kwargs))
+            else:
+                self.cluster.log.verbose('Performing %s %s on nodes %s with options: %s' % (
+                    fn.__name__, args[0], list(self.nodes), kwargs))
+            return NodeGroupResult(self.cluster, results)
+        elif "dry_run" in kwargs.keys():
+            del kwargs["dry_run"]
+        try:
+            results = fn(self, *args, **kwargs)
+            return results
+        except fabric.group.GroupException as e:
+            results = e.result
+            raise
+
+    return do_dry_run
+
+
 class AbstractGroup(Generic[GROUP_RUN_TYPE], ABC):
     def __init__(self, ips: Iterable[Union[str, GROUP_SELF]], cluster: object):
         from kubemarine.core.cluster import KubernetesCluster
@@ -297,6 +329,7 @@ def __eq__(self, other: object) -> bool:
     def __ne__(self, other: object) -> bool:
         return not self == other
 
+    @_handle_dry_run
     def run(self, command: str,
             warn: bool = False, hide: bool = True,
             env: Dict[str, str] = None, timeout: int = None,
@@ -308,6 +341,7 @@ def run(self, command: str,
         return self._run("run", command, caller,
                          warn=warn, hide=hide, env=env, timeout=timeout, callback=callback)
 
+    @_handle_dry_run
     def sudo(self, command: str,
              warn: bool = False, hide: bool = True,
              env: Dict[str, str] = None, timeout: int = None,
@@ -328,9 +362,11 @@ def _run(self, do_type: str, command: str, caller: Optional[Dict[str, object]],
     def get(self, remote_file: str, local_file: str) -> None:
         pass
 
+    @_handle_dry_run
     def put(self, local_file: Union[io.StringIO, str], remote_file: str,
             backup: bool = False, sudo: bool = False,
             mkdir: bool = False, immutable: bool = False) -> None:
+
         if isinstance(local_file, io.StringIO):
             self.cluster.log.verbose("Text is being transferred to remote file \"%s\" on nodes %s"
                                      % (remote_file, list(self.nodes)))

kubemarine/core/utils.py

@@ -147,7 +147,11 @@ def make_ansible_inventory(location, c):
             config_compiled += '\n' + string
         config_compiled += '\n\n'
 
-    dump_file({}, config_compiled, location, dump_location=False)
+    dump_location = False
+    args = cluster.context.get('execution_arguments')
+    if args.get('without_act', None):
+        dump_location = True
+    dump_file(cluster, config_compiled, location, dump_location=dump_location)
 
 
 def get_current_timestamp_formatted():
@@ -203,7 +207,7 @@ def dump_file(context, data: object, filename: str,
             cluster = context
             context = cluster.context
 
-        args = context['execution_arguments']
+        args = context.get('execution_arguments', {})
         if args.get('disable_dump', True) \
                 and not (filename in ClusterStorage.PRESERVED_DUMP_FILES and context['preserve_inventory']):
             return
@@ -229,11 +233,14 @@ def get_dump_filepath(context, filename):
     return get_external_resource_path(os.path.join(context['execution_arguments']['dump_location'], 'dump', filename))
 
 
-def wait_command_successful(g, command, retries=15, timeout=5, warn=True, hide=False):
+def wait_command_successful(g, command, retries=15, timeout=5, warn=True, hide=False, dry_run=False):
     from kubemarine.core.group import NodeGroup
     group: NodeGroup = g
 
     log = group.cluster.log
+    if dry_run:
+        log.debug("[dry-run] Command succeeded")
+        return
 
     while retries > 0:
         log.debug("Waiting for command to succeed, %s retries left" % retries)
@@ -470,6 +477,10 @@ def _test_version(version: str, numbers_amount: int) -> List[int]:
     raise ValueError(f'Incorrect version \"{version}\" format, expected version pattern is \"{expected_pattern}\"')
 
 
+def check_dry_run_status_active(cluster):
+    return cluster.context.get("dry_run")
+
+
 class ClusterStorage:
     """
     File preservation:

kubemarine/coredns.py

@@ -148,18 +148,18 @@ def generate_configmap(inventory: dict) -> str:
     return config + '\n'
 
 
-def apply_configmap(cluster: KubernetesCluster, config: str) -> RunnersGroupResult:
+def apply_configmap(cluster: KubernetesCluster, config: str, dry_run=False) -> RunnersGroupResult:
     utils.dump_file(cluster, config, 'coredns-configmap.yaml')
 
     group = cluster.make_group_from_roles(['control-plane', 'worker']).get_final_nodes()
-    group.put(io.StringIO(config), '/etc/kubernetes/coredns-configmap.yaml', backup=True, sudo=True)
+    group.put(io.StringIO(config), '/etc/kubernetes/coredns-configmap.yaml', backup=True, sudo=True, dry_run=dry_run)
 
     return cluster.nodes['control-plane'].get_final_nodes().get_first_member()\
         .sudo('kubectl apply -f /etc/kubernetes/coredns-configmap.yaml && '
-             'sudo kubectl rollout restart -n kube-system deployment/coredns')
+             'sudo kubectl rollout restart -n kube-system deployment/coredns', dry_run=dry_run)
 
 
-def apply_patch(cluster: KubernetesCluster) -> Union[RunnersGroupResult, str]:
+def apply_patch(cluster: KubernetesCluster, dry_run=False) -> Union[RunnersGroupResult, str]:
     apply_command = ''
 
     for config_type in ['deployment']:
@@ -177,11 +177,11 @@ def apply_patch(cluster: KubernetesCluster) -> Union[RunnersGroupResult, str]:
         utils.dump_file(cluster, config, filename)
 
         group = cluster.make_group_from_roles(['control-plane', 'worker']).get_final_nodes()
-        group.put(io.StringIO(config), filepath, backup=True, sudo=True)
+        group.put(io.StringIO(config), filepath, backup=True, sudo=True, dry_run=dry_run)
 
         apply_command = 'kubectl patch %s coredns -n kube-system --type merge -p \"$(sudo cat %s)\"' % (config_type, filepath)
 
     if apply_command == '':
         return 'Nothing to patch'
 
-    return cluster.nodes['control-plane'].get_final_nodes().get_first_member().sudo(apply_command)
+    return cluster.nodes['control-plane'].get_final_nodes().get_first_member().sudo(apply_command, dry_run=dry_run)

kubemarine/cri/containerd.py

@@ -26,11 +26,15 @@
 
 
 def install(group: NodeGroup) -> RunnersGroupResult:
+
+    if utils.check_dry_run_status_active(group.cluster):
+        group.cluster.log.debug("[dry-run] Installing Containerd")
+        return None
     collector = CollectorCallback(group.cluster)
+
     with group.new_executor() as exe:
         for node in exe.group.get_ordered_members_list():
             os_specific_associations = exe.cluster.get_associations_for_node(node.get_host(), 'containerd')
-
             exe.cluster.log.debug("Installing latest containerd and podman on %s node" % node.get_node_name())
             # always install latest available containerd and podman
             packages.install(node, include=os_specific_associations['package_name'], callback=collector)
@@ -52,8 +56,10 @@ def configure(group: NodeGroup) -> RunnersGroupResult:
 
     log.debug("Uploading crictl configuration for containerd...")
     crictl_config = yaml.dump({"runtime-endpoint": "unix:///run/containerd/containerd.sock"})
+
     utils.dump_file(cluster, crictl_config, 'crictl.yaml')
-    group.put(StringIO(crictl_config), '/etc/crictl.yaml', backup=True, sudo=True)
+    dry_run = utils.check_dry_run_status_active(cluster)
+    group.put(StringIO(crictl_config), '/etc/crictl.yaml', backup=True, sudo=True, dry_run=dry_run)
 
     config_string = ""
     # double loop is used to make sure that no "simple" `key: value` pairs are accidentally assigned to sections
@@ -93,20 +99,23 @@ def configure(group: NodeGroup) -> RunnersGroupResult:
             if registry_configs[auth_registry].get('auth', {}).get('auth', ''):
                 auth_registries['auths'][auth_registry]['auth'] = registry_configs[auth_registry]['auth']['auth']
         auth_json = json.dumps(auth_registries)
-        group.put(StringIO(auth_json), "/etc/containers/auth.json", backup=True, sudo=True)
-        group.sudo("chmod 600 /etc/containers/auth.json")
+        group.put(StringIO(auth_json), "/etc/containers/auth.json", backup=True, sudo=True, dry_run=dry_run)
+        group.sudo("chmod 600 /etc/containers/auth.json", dry_run=dry_run)
     if insecure_registries:
         log.debug("Uploading podman configuration...")
         podman_registries = f"[registries.insecure]\nregistries = {insecure_registries}\n"
-        utils.dump_file(cluster, podman_registries, 'podman_registries.conf')
-        group.sudo("mkdir -p /etc/containers/")
-        group.put(StringIO(podman_registries), "/etc/containers/registries.conf", backup=True, sudo=True)
+        utils.dump_file(group.cluster, podman_registries, 'podman_registries.conf')
+        group.sudo("mkdir -p /etc/containers/", dry_run=dry_run)
+        group.put(StringIO(podman_registries), "/etc/containers/registries.conf", backup=True, sudo=True, dry_run=dry_run)
     else:
         log.debug("Removing old podman configuration...")
-        group.sudo("rm -f /etc/containers/registries.conf")
+        group.sudo("rm -f /etc/containers/registries.conf", dry_run=dry_run)
 
     utils.dump_file(cluster, config_string, 'containerd-config.toml')
+    if dry_run:
+        return None
     collector = CollectorCallback(cluster)
+
     with group.new_executor() as exe:
         for node in exe.group.get_ordered_members_list():
             os_specific_associations = exe.cluster.get_associations_for_node(node.get_host(), 'containerd')

kubemarine/cri/docker.py

@@ -22,6 +22,9 @@
 
 def install(group: NodeGroup) -> RunnersGroupResult:
     cluster = group.cluster
+    if utils.check_dry_run_status_active(cluster):
+        cluster.log.debug("[dry-run] Installing Docker")
+        return None
     collector = CollectorCallback(cluster)
     with group.new_executor() as exe:
         for node in exe.group.get_ordered_members_list():
@@ -58,6 +61,9 @@ def configure(group: NodeGroup) -> RunnersGroupResult:
 
     settings_json = json.dumps(cluster.inventory["services"]['cri']['dockerConfig'], sort_keys=True, indent=4)
     utils.dump_file(cluster, settings_json, 'docker-daemon.json')
+    if utils.check_dry_run_status_active(cluster):
+        group.cluster.log.debug("[dry-run] Configuring Docker")
+        return
 
     collector = CollectorCallback(cluster)
     with group.new_executor() as exe:
@@ -76,9 +82,10 @@ def configure(group: NodeGroup) -> RunnersGroupResult:
     return collector.result
 
 
-def prune(group: NodeGroup) -> RunnersGroupResult:
+def prune(group: NodeGroup, dry_run=False) -> RunnersGroupResult:
     return group.sudo('docker container stop $(sudo docker container ls -aq); '
                       'sudo docker container rm $(sudo docker container ls -aq); '
                       'sudo docker system prune -a -f; '
-                      # kill all containerd-shim processes, so that no orphan containers remain 
-                      'sudo pkill -9 -f "^containerd-shim"', warn=True)
+                      # kill all containerd-shim processes, so that no orphan containers remain
+                      'sudo pkill -9 -f "^containerd-shim"', warn=True,
+                      dry_run=dry_run)