Skip to content

Commit

Permalink
fix: Fixed update process for K8s NuvlaEdges.
Browse files Browse the repository at this point in the history
* fix: Update K8s validation node.

* fix: Update K8s validation node.
  • Loading branch information
ignacio-penas authored Nov 18, 2024
1 parent b83710c commit 8c63a67
Show file tree
Hide file tree
Showing 11 changed files with 45 additions and 31 deletions.
4 changes: 2 additions & 2 deletions conf/targets/ubuntu_vm_k8s.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ alias = "ubuntu_VM"
coe = "kubernetes"

# Network data
address = "194.182.163.87"
address = "85.217.161.47"
port = 22
hostname = ""

# Naming data
user = "ubuntu"
user = "root"

# Security data
pub_key_path = "~/.ssh/id_rsa.pub"
Expand Down
10 changes: 5 additions & 5 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions validation_framework/deployer/coe/coe_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,11 @@ def engine_running(self) -> bool:
pass

@abstractmethod
def remove_engine(self):
def remove_engine(self, uuid: NuvlaUUID = None):
pass

@abstractmethod
def purge_engine(self):
def purge_engine(self, uuid: NuvlaUUID = None):
pass

@abstractmethod
Expand Down
6 changes: 3 additions & 3 deletions validation_framework/deployer/coe/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def engine_running(self) -> bool:
result: Result = self.device.run_command(f'docker ps | grep {cte.PROJECT_NAME}')
return result.stdout.strip() != ''

def remove_engine(self):
def remove_engine(self, uuid: NuvlaUUID = None):
"""
Removes docker containers, services, volumes and networks from the device
:return: None
Expand All @@ -205,9 +205,9 @@ def get_coe_type(self):
def finish_tasks(self):
pass

def purge_engine(self):
def purge_engine(self, uuid: NuvlaUUID = None):
self.stop_engine()
self.remove_engine()
self.remove_engine(uuid)

def download_files(self, source, version) -> list[str]:
self.engine_folder = cte.ROOT_PATH + cte.ENGINE_PATH + version
Expand Down
17 changes: 10 additions & 7 deletions validation_framework/deployer/coe/kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ def start_engine(self,
idparts = uuid.split('/')
self.nuvla_uuid = idparts[1]

add_repo_cmd = f'sudo helm repo add {cte.NUVLAEDGE_KUBE_LOCAL_REPO_NAME} {cte.NUVLAEDGE_KUBE_REPO}'
add_repo_cmd = f'helm repo add {cte.NUVLAEDGE_KUBE_LOCAL_REPO_NAME} {cte.NUVLAEDGE_KUBE_REPO}'
result: Result = self.device.run_sudo_command(add_repo_cmd, envs=_KUBECONFIG_ENV)
if result.failed:
self.logger.error(f'Could not add repo to helm {cte.NUVLAEDGE_KUBE_REPO}: {result.stderr}')

update_repo_cmd = f'sudo helm repo update nuvlaedge {cte.NUVLAEDGE_KUBE_LOCAL_REPO_NAME}'
update_repo_cmd = f'helm repo update nuvlaedge {cte.NUVLAEDGE_KUBE_LOCAL_REPO_NAME}'
result: Result = self.device.run_sudo_command(update_repo_cmd, envs=_KUBECONFIG_ENV)
if result.failed:
self.logger.error(f'Could not update helm repo {cte.NUVLAEDGE_KUBE_LOCAL_REPO_NAME}: {result.stderr}')
Expand Down Expand Up @@ -188,21 +188,24 @@ def finish_tasks(self):
self.logger.debug("Waiting for credentials check thread to close")
self.cred_check_thread.join(50)

def remove_engine(self):
def remove_engine(self, uuid: NuvlaUUID = None):
self.logger.debug(f'Removing engine in device {self.device}')
self.finish_tasks()

if not self.nuvla_uuid:
self.nuvla_uuid = uuid
ne_id = self.nuvla_uuid.split('/')[1]
self.namespaces_running = self.__get_namespaces_running()
commands: list = ['sudo kubectl delete '
'clusterrolebindings.rbac.authorization.k8s.io '
'nuvla-crb '
f'nuvlaedge-service-account-cluster-role-binding-{self.nuvla_uuid}']
f'nuvlaedge-service-account-cluster-role-binding-{ne_id}']

for namespace in self.namespaces_running:
if namespace.__contains__('kube') or namespace == 'default':
continue
commands.append(f'sudo kubectl delete ns {namespace}')
commands.append(f'sudo helm uninstall {namespace}')
commands.append(f'helm uninstall {namespace}')

for cmd in commands:
try:
Expand Down Expand Up @@ -230,13 +233,13 @@ def peripherals_running(self, peripherals: set) -> bool:
peripherals.remove(res.group(1))
return not bool(peripherals)

def purge_engine(self):
def purge_engine(self, uuid: NuvlaUUID = None):
"""
This will remove all pods and namespaces for kubernetes
:return:
"""
self.stop_engine()
self.remove_engine()
self.remove_engine(uuid)

def __get_current_nuvlaedge_namespace_running(self):
get_curr_nuvlaedge_namespace_cmd = ('sudo kubectl get namespaces -o json | jq '
Expand Down
6 changes: 3 additions & 3 deletions validation_framework/deployer/engine_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def start_engine(self, nuvlaedge_uuid: NuvlaUUID,
if remove_old_installation:
# 1. - Clean target
self.logger.info('Removing possible old installations of NuvlaEdge')
self.coe.purge_engine()
self.coe.purge_engine(nuvlaedge_uuid)

# 2. Start Engine
self.logger.info('Download NuvlaEdge related files and images')
Expand All @@ -99,7 +99,7 @@ def get_system_up_time_in_engine(self) -> float:
def restart_engine(self) -> Result:
return self.coe.restart_system()

def stop_engine(self, retrieve_logs: bool = False) -> bool:
def stop_engine(self, retrieve_logs: bool = False, uuid: NuvlaUUID = None) -> bool:
"""
:return:
Expand All @@ -112,7 +112,7 @@ def stop_engine(self, retrieve_logs: bool = False) -> bool:
if retrieve_logs:
self.coe.get_engine_logs()

self.coe.purge_engine()
self.coe.purge_engine(uuid)

def check_if_peripherals_running(self, peripherals: set) -> bool:
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
"""
import time
import unittest

from nuvla.api.resources import Deployment

from validation_framework.validators import ValidationBase
from validation_framework.validators.tests.nuvla_operations import validator
from validation_framework.common.constants import DEFAULT_DEPLOYMENTS_TIMEOUT
from nuvla.api.models import CimiResponse, CimiResource, CimiCollection
from nuvla.api.models import CimiResource, CimiCollection


@validator('BasicAppDeployment')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
from validation_framework.validators.tests.nuvla_operations import validator
from nuvla.api.models import CimiResource, CimiResponse
import time
from fabric import Result
from pprint import pprint as pp


@validator('EngineReboot')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
"""
import time
from pprint import pprint as pp
from nuvla.api.models import CimiResource, CimiResponse, CimiCollection
from nuvla.api.models import CimiResource, CimiResponse

from validation_framework.validators.tests.nuvla_operations import validator
from validation_framework.common.constants import DEFAULT_JOBS_TIMEOUT
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from nuvla.api import Api as NuvlaClient
from nuvla.api.models import CimiResource, CimiResponse
from packaging.version import Version
from pydantic import BaseModel, ConfigDict

from validation_framework.common.constants import DEFAULT_JOBS_TIMEOUT
Expand All @@ -29,6 +30,7 @@

GH_API_URL = "https://api.github.com/repos/{repo}/releases"
NUVLA_RELEASE_ENDPOINT = "https://nuvla.io/api/nuvlabox-release"
MINIMUM_K8S_VERSION = "2.18.0"

@dataclass
class FutureResult:
Expand All @@ -45,10 +47,19 @@ def __init__(self, *args, **kwargs):

self.engines: dict[str, EngineHandler] = {}
self.uuids: dict[str, str] = {}
self.is_skip: bool = False

def run_update(self, origin, target):
self.logger.info(f"Starting update from {origin}(Minimum {MINIMUM_K8S_VERSION}) to {target}")
self.logger.info("Engine keys are: " + str(self.engines.keys()))
self.custom_setup(origin, target)
engine = self.engines[origin]

if Version(origin) < Version(MINIMUM_K8S_VERSION) and engine.coe_type == 'kubernetes':
self.logger.info(f"Skipping {origin} to {target} update. Minimum NuvlaEdge version for Kubernetes validation update is {MINIMUM_K8S_VERSION}")
self.is_skip = True
self.skipTest("Minimum NuvlaEdge version for Kubernetes validation update is not met")

uuid = self.uuids[origin]

self.wait_for_commissioned(engine=engine, uuid=NuvlaUUID(uuid))
Expand Down Expand Up @@ -196,7 +207,9 @@ def setUp(self) -> None:
self.nuvla_client: NuvlaClient = NuvlaClient(reauthenticate=True)
self.nuvla_client.login_apikey(self.nuvla_api_key, self.nuvla_api_secret)


def tearDown(self) -> None:
if not self.is_skip:
super().tearDown()

def underscore_to_hyphen(field_name: str) -> str:
"""
Expand Down
6 changes: 4 additions & 2 deletions validation_framework/validators/validation_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def parametrize(testcase_class,
for release in releases:
method_name = f"test_update_{release[0].replace(".", "_")}_to_{nuvlaedge_branch}"
method = create_update_method(release[0], nuvlaedge_branch)
logging.info("Adding method %s to %s", method_name, testcase_class)
setattr(testcase_class, method_name, method)
setattr(testcase_class, 'release_id', release[1])

Expand Down Expand Up @@ -314,5 +315,6 @@ def tearDown(self) -> None:
self.retrieve_logs = True
self.logger.error(f'Test failed with exception: {self.failureException}')

self.engine_handler.stop_engine(retrieve_logs=self.retrieve_logs)
self.remove_nuvlaedge_from_nuvla()
if self.engine_handler and self.uuid:
self.engine_handler.stop_engine(retrieve_logs=self.retrieve_logs, uuid=self.uuid)
self.remove_nuvlaedge_from_nuvla()

0 comments on commit 8c63a67

Please sign in to comment.