MANOPD-70719 Added DaemonSets and Deployments expect

kubemarine/kubernetes/daemonset.py

@@ -0,0 +1,46 @@
+# Copyright 2021-2022 NetCracker Technology Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from core.cluster import KubernetesCluster
+from kubemarine.kubernetes.object import KubernetesObject
+
+
+class DaemonSet(KubernetesObject):
+
+    def __init__(self, cluster: KubernetesCluster, name=None, namespace=None, obj=None):
+        super().__init__(cluster, kind='DaemonSet', name=name, namespace=namespace, obj=obj)
+
+    def is_actual_and_ready(self) -> bool:
+        return self.is_ready() and self.is_up_to_date()
+
+    def is_up_to_date(self) -> bool:
+        desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled')
+        updated_number_scheduled = self._obj.get('status', {}).get('updatedNumberScheduled')
+        return desired_number_scheduled is not None \
+               and updated_number_scheduled is not None \
+               and desired_number_scheduled == updated_number_scheduled
+
+    def is_ready(self) -> bool:
+        desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled')
+        number_ready = self._obj.get('status', {}).get('numberReady')
+        return desired_number_scheduled is not None \
+               and number_ready is not None \
+               and desired_number_scheduled == number_ready
+
+    def is_scheduled(self) -> bool:
+        desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled')
+        current_number_scheduled = self._obj.get('status', {}).get('currentNumberScheduled')
+        return desired_number_scheduled is not None \
+               and current_number_scheduled is not None \
+               and desired_number_scheduled == current_number_scheduled

kubemarine/kubernetes/deployment.py

@@ -0,0 +1,39 @@
+# Copyright 2021-2022 NetCracker Technology Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from core.cluster import KubernetesCluster
+from kubemarine.kubernetes.object import KubernetesObject
+
+
+class Deployment(KubernetesObject):
+
+    def __init__(self, cluster: KubernetesCluster, name=None, namespace=None, obj=None):
+        super().__init__(cluster, kind='Deployment', name=name, namespace=namespace, obj=obj)
+
+    def is_actual_and_ready(self) -> bool:
+        return self.is_ready() and self.is_up_to_date()
+
+    def is_up_to_date(self) -> bool:
+        desired_number_scheduled = self._obj.get('spec', {}).get('replicas')
+        updated_number_scheduled = self._obj.get('status', {}).get('updatedReplicas')
+        return desired_number_scheduled is not None \
+               and updated_number_scheduled is not None \
+               and desired_number_scheduled == updated_number_scheduled
+
+    def is_ready(self) -> bool:
+        desired_number_scheduled = self._obj.get('spec', {}).get('replicas')
+        number_ready = self._obj.get('status', {}).get('readyReplicas')
+        return desired_number_scheduled is not None \
+               and number_ready is not None \
+               and desired_number_scheduled == number_ready

kubemarine/kubernetes/object.py

@@ -0,0 +1,101 @@
+# Copyright 2021-2022 NetCracker Technology Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import io
+import json
+import uuid
+import time
+
+import yaml
+
+from core.cluster import KubernetesCluster
+
+
+class KubernetesObject:
+    def __init__(self, cluster: KubernetesCluster, kind=None, name=None, namespace=None, obj=None):
+
+        self._cluster = cluster
+        self._reload_t = -1
+
+        if not kind and not name and not namespace and not obj:
+            raise RuntimeError('Not enough parameter values to construct the object')
+        if obj:
+            self._obj = obj
+        else:
+            if not kind or not name or not namespace:
+                raise RuntimeError('An unsynchronized object has not enough parameters '
+                                   'to be reloaded')
+            self._obj = {
+                "kind": kind,
+                "metadata": {
+                    "name": name,
+                    "namespace": namespace,
+                }
+            }
+
+    def __str__(self):
+        return self.to_yaml()
+
+    @property
+    def uid(self):
+        uid = self._obj.get('metadata', {}).get('uid')
+        if uid:
+            return uid
+
+        return str(uuid.uuid4())
+
+    @property
+    def kind(self):
+        return self._obj.get('kind').lower()
+
+    @property
+    def namespace(self):
+        return self._obj.get('metadata', {}).get('namespace').lower()
+
+    @property
+    def name(self):
+        return self._obj.get('metadata', {}).get('name').lower()
+
+    def to_json(self):
+        return json.dumps(self._obj)
+
+    def to_yaml(self):
+        return yaml.dump(self._obj)
+
+    def is_reloaded(self):
+        return self._reload_t > -1
+
+    def reload(self, master=None, suppress_exceptions=False) -> KubernetesObject:
+        if not master:
+            master = self._cluster.nodes['master'].get_any_member()
+        cmd = f'kubectl get {self.kind} -n {self.namespace} {self.name} -o json'
+        result = master.sudo(cmd, warn=suppress_exceptions)
+        self._cluster.log.verbose(result)
+        if not result.is_any_failed():
+            self._obj = json.loads(result.get_simple_out())
+            self._reload_t = time.time()
+        return self
+
+    def apply(self, master=None):
+        if not master:
+            master = self._cluster.nodes['master'].get_any_member()
+
+        json_str = self.to_json()
+        obj_filename = "_".join([self.kind, self.namespace, self.name, self.uid]) + '.json'
+        obj_path = f'/tmp/{obj_filename}'
+
+        master.put(io.StringIO(json_str), obj_path, sudo=True)
+        master.sudo(f'kubectl apply -f {obj_path} && sudo rm -f {obj_path}')

kubemarine/plugins/__init__.py

@@ -36,8 +36,11 @@
 from kubemarine.core.yaml_merger import default_merger
 from kubemarine.core.group import NodeGroup, NodeGroupResult
 from kubemarine.core.cluster import KubernetesCluster
+from kubemarine.kubernetes.daemonset import DaemonSet
+from kubemarine.kubernetes.deployment import Deployment
 
 # list of plugins owned and managed by kubemarine
+
 oob_plugins = [
     "calico",
     "flannel",
@@ -177,6 +180,78 @@ def install_plugin(cluster, plugin_name, installation_procedure):
         del cluster.context['cached_expected_pods']
 
 
+def expect_daemonset(cluster, daemonsets_names, plugin_name=None, timeout=None, retries=None,
+                     node=None, apply_filter=None):
+    log = cluster.log
+
+    if timeout is None:
+        timeout = cluster.globals['expect']['plugins']['timeout']
+    if retries is None:
+        retries = cluster.globals['expect']['plugins']['retries']
+
+    log.debug(f"Expecting the following DaemonSets to be up to date: {daemonsets_names}")
+    log.verbose("Max expectation time: %ss" % (timeout * retries))
+
+    log.debug("Waiting for DaemonSets...")
+
+    daemonsets = []
+    for name in daemonsets_names:
+        if isinstance(name, str):
+            daemonsets.append(DaemonSet(cluster, name=name, namespace='kube-system'))
+        elif isinstance(name, dict):
+            daemonsets.append(DaemonSet(cluster, name=name['name'], namespace=name['namespace']))
+
+    while retries > 0:
+        up_to_date = True
+        for daemonset in daemonsets:
+            if not daemonset.reload(master=node, suppress_exceptions=True).is_up_to_date():
+                up_to_date = False
+
+        if up_to_date:
+            cluster.log.debug("DaemonSets are up to date")
+            return
+        else:
+            retries -= 1
+            cluster.log.debug("DaemonSets are not up to date yet... (%ss left)" % (retries * timeout))
+            time.sleep(timeout)
+
+
+def expect_deployment(cluster, deployments_names, plugin_name=None, timeout=None, retries=None,
+                      node=None, apply_filter=None):
+    log = cluster.log
+
+    if timeout is None:
+        timeout = cluster.globals['expect']['plugins']['timeout']
+    if retries is None:
+        retries = cluster.globals['expect']['plugins']['retries']
+
+    log.debug(f"Expecting the following Deployments to be up to date: {deployments_names}")
+    log.verbose("Max expectation time: %ss" % (timeout * retries))
+
+    log.debug("Waiting for Deployments...")
+
+    deployments = []
+    for name in deployments_names:
+        if isinstance(name, str):
+            deployments.append(Deployment(cluster, name=name, namespace='kube-system'))
+        elif isinstance(name, dict):
+            deployments.append(Deployment(cluster, name=name['name'], namespace=name['namespace']))
+
+    while retries > 0:
+        up_to_date = True
+        for deployment in deployments:
+            if not deployment.reload(master=node, suppress_exceptions=True).is_actual_and_ready():
+                up_to_date = False
+
+        if up_to_date:
+            cluster.log.debug("Deployments are up to date!")
+            return
+        else:
+            retries -= 1
+            cluster.log.debug("Deployments are not up to date yet... (%ss left)" % (retries * timeout))
+            time.sleep(timeout)
+
+
 def expect_pods(cluster, pods, plugin_name=None, timeout=None, retries=None, node=None, apply_filter=None):
 
     if isinstance(cluster, NodeGroup):
@@ -426,6 +501,14 @@ def apply_template(cluster, config, plugin_name=None):
 # **** EXPECT ****
 
 def convert_expect(cluster, config):
+    if config.get('daemonsets') is not None and isinstance(config['daemonsets'], list):
+        config['daemonsets'] = {
+            'list': config['daemonsets']
+        }
+    if config.get('deployments') is not None and isinstance(config['pods'], list):
+        config['deployments'] = {
+            'list': config['deployments']
+        }
     if config.get('pods') is not None and isinstance(config['pods'], list):
         config['pods'] = {
             'list': config['pods']
@@ -436,19 +519,41 @@ def convert_expect(cluster, config):
 def verify_expect(cluster, config):
     if not config:
         raise Exception('Expect procedure is empty, but it should not be')
+
+    if config.get('daemonsets') is not None and config['daemonsets'].get('list') is None:
+        raise Exception('DaemonSet expectation defined, but DaemonSets list is missing')
+
+    if config.get('deployments') is not None and config['deployments'].get('list') is None:
+        raise Exception('Deployment expectation defined, but Deployments list is missing')
+
     if config.get('pods') is not None and config['pods'].get('list') is None:
-        raise Exception('Pod expectation defined, but pods list is missing')
+        raise Exception('Pod expectation defined, but Pods list is missing')
 
 
 def apply_expect(cluster, config, plugin_name=None):
     # TODO: Add support for expect services and expect nodes
-    if config.get('pods') is not None:
-        timeout = cluster.globals['pods']['expect']['plugins']['timeout']
-        retries = cluster.globals['pods']['expect']['plugins']['retries']
-        return expect_pods(cluster, config['pods']['list'], plugin_name,
-                           timeout=config['pods'].get('timeout', timeout),
-                           retries=config['pods'].get('retries', retries))
 
+    plugins_timeout = cluster.globals['pods']['expect']['plugins']['timeout']
+    plugins_retries = cluster.globals['pods']['expect']['plugins']['retries']
+
+    for expect_type, expect_conf in config.items():
+        if expect_type == 'daemonsets':
+            expect_daemonset(cluster, config['daemonsets']['list'], plugin_name,
+                             timeout=config['daemonsets'].get('timeout', plugins_timeout),
+                             retries=config['daemonsets'].get('retries', plugins_retries))
+
+        elif expect_type == 'deployments':
+            expect_deployment(cluster, config['deployments']['list'], plugin_name,
+                              timeout=config['deployments'].get('timeout', plugins_timeout),
+                              retries=config['deployments'].get('retries', plugins_retries))
+
+        elif expect_type == 'pods':
+            expect_pods(cluster, config['pods']['list'], plugin_name,
+                        timeout=config['pods'].get('timeout', plugins_timeout),
+                        retries=config['pods'].get('retries', plugins_retries))
+
+        else:
+            raise Exception(f'Unknown expectation type "{expect_type}"')
 
 # **** PYTHON ****
 

kubemarine/resources/configurations/defaults.yaml

@@ -344,6 +344,10 @@ plugins:
       procedures:
         - template: 'templates/plugins/calico-{{ plugins.calico.version|minorversion }}.yaml.j2'
         - expect:
+            daemonsets:
+              - calico-node
+            deployments:
+              - calico-kube-controllers
             pods:
               - coredns
               - calico-kube-controllers
@@ -439,6 +443,9 @@ plugins:
             method: manage_custom_certificate
         - template: 'templates/plugins/nginx-ingress-controller-{{ plugins["nginx-ingress-controller"].version|minorversion }}.yaml.j2'
         - expect:
+            daemonsets:
+              - name: ingress-nginx-controller
+                namespace: ingress-nginx
             pods:
               - '{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion|minorversion]["pod-name"] }}'
     controller:
@@ -493,6 +500,11 @@ plugins:
       procedures:
         - template: 'templates/plugins/dashboard-{{ plugins["kubernetes-dashboard"].version|minorversion }}.yaml.j2'
         - expect:
+            deployments:
+              - name: kubernetes-dashboard
+                namespace: kubernetes-dashboard
+              - name: dashboard-metrics-scraper
+                namespace: kubernetes-dashboard
             pods:
               - kubernetes-dashboard
               - dashboard-metrics-scraper