From c5882a506385e93b6b6c10ef7e6418a1689a81c4 Mon Sep 17 00:00:00 2001 From: Leonid Fedotov <3584432+iLeonidze@users.noreply.github.com> Date: Tue, 25 Jan 2022 15:29:38 +0300 Subject: [PATCH] MANOPD-70719 Added DaemonSets and Deployments expect --- .../{kubernetes.py => kubernetes/__init__.py} | 0 kubemarine/kubernetes/daemonset.py | 46 +++++++ kubemarine/kubernetes/deployment.py | 39 ++++++ kubemarine/kubernetes/object.py | 101 +++++++++++++++ kubemarine/plugins/__init__.py | 119 ++++++++++++++++-- .../resources/configurations/defaults.yaml | 12 ++ 6 files changed, 310 insertions(+), 7 deletions(-) rename kubemarine/{kubernetes.py => kubernetes/__init__.py} (100%) create mode 100644 kubemarine/kubernetes/daemonset.py create mode 100644 kubemarine/kubernetes/deployment.py create mode 100644 kubemarine/kubernetes/object.py diff --git a/kubemarine/kubernetes.py b/kubemarine/kubernetes/__init__.py similarity index 100% rename from kubemarine/kubernetes.py rename to kubemarine/kubernetes/__init__.py diff --git a/kubemarine/kubernetes/daemonset.py b/kubemarine/kubernetes/daemonset.py new file mode 100644 index 000000000..64fc9b1d8 --- /dev/null +++ b/kubemarine/kubernetes/daemonset.py @@ -0,0 +1,46 @@ +# Copyright 2021-2022 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from core.cluster import KubernetesCluster +from kubemarine.kubernetes.object import KubernetesObject + + +class DaemonSet(KubernetesObject): + + def __init__(self, cluster: KubernetesCluster, name=None, namespace=None, obj=None): + super().__init__(cluster, kind='DaemonSet', name=name, namespace=namespace, obj=obj) + + def is_actual_and_ready(self) -> bool: + return self.is_ready() and self.is_up_to_date() + + def is_up_to_date(self) -> bool: + desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled') + updated_number_scheduled = self._obj.get('status', {}).get('updatedNumberScheduled') + return desired_number_scheduled is not None \ + and updated_number_scheduled is not None \ + and desired_number_scheduled == updated_number_scheduled + + def is_ready(self) -> bool: + desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled') + number_ready = self._obj.get('status', {}).get('numberReady') + return desired_number_scheduled is not None \ + and number_ready is not None \ + and desired_number_scheduled == number_ready + + def is_scheduled(self) -> bool: + desired_number_scheduled = self._obj.get('status', {}).get('desiredNumberScheduled') + current_number_scheduled = self._obj.get('status', {}).get('currentNumberScheduled') + return desired_number_scheduled is not None \ + and current_number_scheduled is not None \ + and desired_number_scheduled == current_number_scheduled diff --git a/kubemarine/kubernetes/deployment.py b/kubemarine/kubernetes/deployment.py new file mode 100644 index 000000000..a79321f54 --- /dev/null +++ b/kubemarine/kubernetes/deployment.py @@ -0,0 +1,39 @@ +# Copyright 2021-2022 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from core.cluster import KubernetesCluster +from kubemarine.kubernetes.object import KubernetesObject + + +class Deployment(KubernetesObject): + + def __init__(self, cluster: KubernetesCluster, name=None, namespace=None, obj=None): + super().__init__(cluster, kind='Deployment', name=name, namespace=namespace, obj=obj) + + def is_actual_and_ready(self) -> bool: + return self.is_ready() and self.is_up_to_date() + + def is_up_to_date(self) -> bool: + desired_number_scheduled = self._obj.get('spec', {}).get('replicas') + updated_number_scheduled = self._obj.get('status', {}).get('updatedReplicas') + return desired_number_scheduled is not None \ + and updated_number_scheduled is not None \ + and desired_number_scheduled == updated_number_scheduled + + def is_ready(self) -> bool: + desired_number_scheduled = self._obj.get('spec', {}).get('replicas') + number_ready = self._obj.get('status', {}).get('readyReplicas') + return desired_number_scheduled is not None \ + and number_ready is not None \ + and desired_number_scheduled == number_ready diff --git a/kubemarine/kubernetes/object.py b/kubemarine/kubernetes/object.py new file mode 100644 index 000000000..80eb2d83e --- /dev/null +++ b/kubemarine/kubernetes/object.py @@ -0,0 +1,101 @@ +# Copyright 2021-2022 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import io +import json +import uuid +import time + +import yaml + +from core.cluster import KubernetesCluster + + +class KubernetesObject: + def __init__(self, cluster: KubernetesCluster, kind=None, name=None, namespace=None, obj=None): + + self._cluster = cluster + self._reload_t = -1 + + if not kind and not name and not namespace and not obj: + raise RuntimeError('Not enough parameter values to construct the object') + if obj: + self._obj = obj + else: + if not kind or not name or not namespace: + raise RuntimeError('An unsynchronized object has not enough parameters ' + 'to be reloaded') + self._obj = { + "kind": kind, + "metadata": { + "name": name, + "namespace": namespace, + } + } + + def __str__(self): + return self.to_yaml() + + @property + def uid(self): + uid = self._obj.get('metadata', {}).get('uid') + if uid: + return uid + + return str(uuid.uuid4()) + + @property + def kind(self): + return self._obj.get('kind').lower() + + @property + def namespace(self): + return self._obj.get('metadata', {}).get('namespace').lower() + + @property + def name(self): + return self._obj.get('metadata', {}).get('name').lower() + + def to_json(self): + return json.dumps(self._obj) + + def to_yaml(self): + return yaml.dump(self._obj) + + def is_reloaded(self): + return self._reload_t > -1 + + def reload(self, master=None, suppress_exceptions=False) -> KubernetesObject: + if not master: + master = self._cluster.nodes['master'].get_any_member() + cmd = f'kubectl get {self.kind} -n {self.namespace} {self.name} -o json' + result = master.sudo(cmd, warn=suppress_exceptions) + self._cluster.log.verbose(result) + if not result.is_any_failed(): + self._obj = json.loads(result.get_simple_out()) + self._reload_t = time.time() + return self + + def apply(self, master=None): + if not master: + master = self._cluster.nodes['master'].get_any_member() + + json_str = self.to_json() + obj_filename = "_".join([self.kind, self.namespace, self.name, self.uid]) + '.json' + obj_path = f'/tmp/{obj_filename}' + + master.put(io.StringIO(json_str), obj_path, sudo=True) + master.sudo(f'kubectl apply -f {obj_path} && sudo rm -f {obj_path}') diff --git a/kubemarine/plugins/__init__.py b/kubemarine/plugins/__init__.py index 94bc6e06a..c8b5e91bd 100755 --- a/kubemarine/plugins/__init__.py +++ b/kubemarine/plugins/__init__.py @@ -36,8 +36,11 @@ from kubemarine.core.yaml_merger import default_merger from kubemarine.core.group import NodeGroup, NodeGroupResult from kubemarine.core.cluster import KubernetesCluster +from kubemarine.kubernetes.daemonset import DaemonSet +from kubemarine.kubernetes.deployment import Deployment # list of plugins owned and managed by kubemarine + oob_plugins = [ "calico", "flannel", @@ -177,6 +180,78 @@ def install_plugin(cluster, plugin_name, installation_procedure): del cluster.context['cached_expected_pods'] +def expect_daemonset(cluster, daemonsets_names, plugin_name=None, timeout=None, retries=None, + node=None, apply_filter=None): + log = cluster.log + + if timeout is None: + timeout = cluster.globals['expect']['plugins']['timeout'] + if retries is None: + retries = cluster.globals['expect']['plugins']['retries'] + + log.debug(f"Expecting the following DaemonSets to be up to date: {daemonsets_names}") + log.verbose("Max expectation time: %ss" % (timeout * retries)) + + log.debug("Waiting for DaemonSets...") + + daemonsets = [] + for name in daemonsets_names: + if isinstance(name, str): + daemonsets.append(DaemonSet(cluster, name=name, namespace='kube-system')) + elif isinstance(name, dict): + daemonsets.append(DaemonSet(cluster, name=name['name'], namespace=name['namespace'])) + + while retries > 0: + up_to_date = True + for daemonset in daemonsets: + if not daemonset.reload(master=node, suppress_exceptions=True).is_up_to_date(): + up_to_date = False + + if up_to_date: + cluster.log.debug("DaemonSets are up to date") + return + else: + retries -= 1 + cluster.log.debug("DaemonSets are not up to date yet... (%ss left)" % (retries * timeout)) + time.sleep(timeout) + + +def expect_deployment(cluster, deployments_names, plugin_name=None, timeout=None, retries=None, + node=None, apply_filter=None): + log = cluster.log + + if timeout is None: + timeout = cluster.globals['expect']['plugins']['timeout'] + if retries is None: + retries = cluster.globals['expect']['plugins']['retries'] + + log.debug(f"Expecting the following Deployments to be up to date: {deployments_names}") + log.verbose("Max expectation time: %ss" % (timeout * retries)) + + log.debug("Waiting for Deployments...") + + deployments = [] + for name in deployments_names: + if isinstance(name, str): + deployments.append(Deployment(cluster, name=name, namespace='kube-system')) + elif isinstance(name, dict): + deployments.append(Deployment(cluster, name=name['name'], namespace=name['namespace'])) + + while retries > 0: + up_to_date = True + for deployment in deployments: + if not deployment.reload(master=node, suppress_exceptions=True).is_actual_and_ready(): + up_to_date = False + + if up_to_date: + cluster.log.debug("Deployments are up to date!") + return + else: + retries -= 1 + cluster.log.debug("Deployments are not up to date yet... (%ss left)" % (retries * timeout)) + time.sleep(timeout) + + def expect_pods(cluster, pods, plugin_name=None, timeout=None, retries=None, node=None, apply_filter=None): if isinstance(cluster, NodeGroup): @@ -426,6 +501,14 @@ def apply_template(cluster, config, plugin_name=None): # **** EXPECT **** def convert_expect(cluster, config): + if config.get('daemonsets') is not None and isinstance(config['daemonsets'], list): + config['daemonsets'] = { + 'list': config['daemonsets'] + } + if config.get('deployments') is not None and isinstance(config['pods'], list): + config['deployments'] = { + 'list': config['deployments'] + } if config.get('pods') is not None and isinstance(config['pods'], list): config['pods'] = { 'list': config['pods'] @@ -436,19 +519,41 @@ def convert_expect(cluster, config): def verify_expect(cluster, config): if not config: raise Exception('Expect procedure is empty, but it should not be') + + if config.get('daemonsets') is not None and config['daemonsets'].get('list') is None: + raise Exception('DaemonSet expectation defined, but DaemonSets list is missing') + + if config.get('deployments') is not None and config['deployments'].get('list') is None: + raise Exception('Deployment expectation defined, but Deployments list is missing') + if config.get('pods') is not None and config['pods'].get('list') is None: - raise Exception('Pod expectation defined, but pods list is missing') + raise Exception('Pod expectation defined, but Pods list is missing') def apply_expect(cluster, config, plugin_name=None): # TODO: Add support for expect services and expect nodes - if config.get('pods') is not None: - timeout = cluster.globals['pods']['expect']['plugins']['timeout'] - retries = cluster.globals['pods']['expect']['plugins']['retries'] - return expect_pods(cluster, config['pods']['list'], plugin_name, - timeout=config['pods'].get('timeout', timeout), - retries=config['pods'].get('retries', retries)) + plugins_timeout = cluster.globals['pods']['expect']['plugins']['timeout'] + plugins_retries = cluster.globals['pods']['expect']['plugins']['retries'] + + for expect_type, expect_conf in config.items(): + if expect_type == 'daemonsets': + expect_daemonset(cluster, config['daemonsets']['list'], plugin_name, + timeout=config['daemonsets'].get('timeout', plugins_timeout), + retries=config['daemonsets'].get('retries', plugins_retries)) + + elif expect_type == 'deployments': + expect_deployment(cluster, config['deployments']['list'], plugin_name, + timeout=config['deployments'].get('timeout', plugins_timeout), + retries=config['deployments'].get('retries', plugins_retries)) + + elif expect_type == 'pods': + expect_pods(cluster, config['pods']['list'], plugin_name, + timeout=config['pods'].get('timeout', plugins_timeout), + retries=config['pods'].get('retries', plugins_retries)) + + else: + raise Exception(f'Unknown expectation type "{expect_type}"') # **** PYTHON **** diff --git a/kubemarine/resources/configurations/defaults.yaml b/kubemarine/resources/configurations/defaults.yaml index 699cfb2a8..d9054a70e 100644 --- a/kubemarine/resources/configurations/defaults.yaml +++ b/kubemarine/resources/configurations/defaults.yaml @@ -344,6 +344,10 @@ plugins: procedures: - template: 'templates/plugins/calico-{{ plugins.calico.version|minorversion }}.yaml.j2' - expect: + daemonsets: + - calico-node + deployments: + - calico-kube-controllers pods: - coredns - calico-kube-controllers @@ -439,6 +443,9 @@ plugins: method: manage_custom_certificate - template: 'templates/plugins/nginx-ingress-controller-{{ plugins["nginx-ingress-controller"].version|minorversion }}.yaml.j2' - expect: + daemonsets: + - name: ingress-nginx-controller + namespace: ingress-nginx pods: - '{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion|minorversion]["pod-name"] }}' controller: @@ -493,6 +500,11 @@ plugins: procedures: - template: 'templates/plugins/dashboard-{{ plugins["kubernetes-dashboard"].version|minorversion }}.yaml.j2' - expect: + deployments: + - name: kubernetes-dashboard + namespace: kubernetes-dashboard + - name: dashboard-metrics-scraper + namespace: kubernetes-dashboard pods: - kubernetes-dashboard - dashboard-metrics-scraper