diff --git a/almalinux_sign_node.py b/almalinux_sign_node.py index ec59aba..6571dac 100755 --- a/almalinux_sign_node.py +++ b/almalinux_sign_node.py @@ -7,17 +7,42 @@ CloudLinux Build System builds sign node. """ +import argparse +import logging import sys import sentry_sdk -from sign_node.errors import ConfigurationError from sign_node.config import SignNodeConfig -from sign_node.cli import init_args_parser, init_working_dir +from sign_node.errors import ConfigurationError from sign_node.signer import Signer from sign_node.utils.config import locate_config_file -from sign_node.utils.log import configure_logger -from sign_node.utils.pgp_utils import init_gpg, PGPPasswordDB +from sign_node.utils.file_utils import clean_dir, safe_mkdir +from sign_node.utils.pgp_utils import PGPPasswordDB, init_gpg + + +def init_arg_parser(): + parser = argparse.ArgumentParser( + prog="sign_node", description="CloudLinux Build System builds sign node" + ) + parser.add_argument("-c", "--config", help="configuration file path") + parser.add_argument( + "-v", "--verbose", action="store_true", help="enable additional debug output" + ) + return parser + + +def init_logger(verbose): + level = logging.DEBUG if verbose else logging.INFO + handler = logging.StreamHandler() + handler.setLevel(level) + log_format = "%(asctime)s %(levelname)-8s [%(threadName)s]: %(message)s" + formatter = logging.Formatter(log_format, "%y.%m.%d %H:%M:%S") + handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(level) + return logger def init_sentry(config: SignNodeConfig): @@ -30,10 +55,19 @@ def init_sentry(config: SignNodeConfig): ) +def init_working_dir(config): + working_dir = config.working_dir + if not safe_mkdir(working_dir): + logging.debug("cleaning up the %s working directory", working_dir) + clean_dir(working_dir) + else: + logging.debug("working directory %s was created", working_dir) + + def main(): - args_parser = init_args_parser() + args_parser = init_arg_parser() args = args_parser.parse_args() - logger = configure_logger(args.verbose) + logger = init_logger(args.verbose) try: config_file = locate_config_file('sign_node', args.config) logger.debug("Loading %s", config_file if config_file else 'default configuration') @@ -55,7 +89,7 @@ def main(): except ConfigurationError as e: args_parser.error(str(e)) - init_working_dir(config.working_dir) + init_working_dir(config) signer = Signer(config, password_db, gpg) signer.sign_loop() diff --git a/sign_node/cli.py b/sign_node/cli.py deleted file mode 100644 index b4ff2b5..0000000 --- a/sign_node/cli.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- mode:python; coding:utf-8; -*- -# author: Eugene Zamriy -# created: 2018-03-31 - -""" -CloudLinux Build System builds sign node command line tool functions. -""" - -import argparse -import logging - -from .utils.file_utils import safe_mkdir, clean_dir - -__all__ = ["init_args_parser", "init_working_dir"] - - -def init_working_dir(working_dir): - """ - The working directory initialization function. It removes files from - previous executions and creates the necessary directories. - - Parameters - ---------- - working_dir : str - Working directory path. - """ - # TODO: move this function to a common module like utils since its - # used by the sign node and we have something very similar in the - # build node's code. - if not safe_mkdir(working_dir): - logging.debug("cleaning up the %s working directory", working_dir) - clean_dir(working_dir) - else: - logging.debug("working directory %s was created", working_dir) - - -def init_args_parser(): - """ - Sign daemon command line arguments parser initialization. - - Returns - ------- - argparse.ArgumentParser - """ - parser = argparse.ArgumentParser( - prog="sign_node", description="CloudLinux Build System builds sign node" - ) - parser.add_argument("-c", "--config", help="configuration file path") - parser.add_argument( - "-v", "--verbose", action="store_true", help="enable additional debug output" - ) - return parser diff --git a/sign_node/utils/log.py b/sign_node/utils/log.py deleted file mode 100644 index 5d5d143..0000000 --- a/sign_node/utils/log.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- mode:python; coding:utf-8; -*- -# author: Eugene Zamriy -# created: 2017-10-19 - -""" -Cloud Linux Build System logging functions. -""" - -import logging - -__all__ = ["log_put", "configure_logger"] - - -def configure_logger(verbose, name=None, filelog=None): - """ - Initializes a program root logger. - - Parameters - ---------- - verbose : bool - Enable DEBUG messages output if True, print only INFO and higher - otherwise. - name : str, optional - Logger name. A root logger will be used if omitted. - filelog : str, optional - Logger path for file. - - Returns - ------- - logging.Logger - Configured root logger. - """ - level = logging.DEBUG if verbose else logging.INFO - handler = logging.StreamHandler() - handler.setLevel(level) - log_format = "%(asctime)s %(levelname)-8s [%(threadName)s]: %(message)s" - formatter = logging.Formatter(log_format, "%y.%m.%d %H:%M:%S") - handler.setFormatter(formatter) - logger = logging.getLogger(name) - logger.addHandler(handler) - logger.setLevel(level) - if filelog is not None: - fh = logging.FileHandler(filelog) - fh.setLevel(level) - fh.setFormatter(formatter) - logger.addHandler(fh) - return logger - - -def create_log_dict(log, status): - """ - Create dict for log - - Parameters - ---------- - log : str or unicode - String for log - status : str or unicode - Status - Returns - ------- - dict - Dict with status and message - - """ - if status not in ["info", "debug", "error"]: - status = "error" - return {"status": status, "log": log} - - -def log_put(logs_queue, message, log_type="info"): - log_type = log_type if log_type in ["info", "debug", "error"] else "info" - logs_queue.put(create_log_dict(message, log_type), True) diff --git a/sign_node/utils/test_utils.py b/sign_node/utils/test_utils.py deleted file mode 100644 index cb285b1..0000000 --- a/sign_node/utils/test_utils.py +++ /dev/null @@ -1,164 +0,0 @@ -# -*- mode:python; coding:utf-8; -*- -# author: Eugene Zamriy -# created: 2018-06-09 - -""" -CloudLinux Build System testing utility functions. -""" - -import contextlib -import json -import os -import shutil -import sys -import tempfile -import unittest - -__all__ = [ - "MOCK_COMMAND_TEMPLATE", - "MockShellCommand", - "unload_plumbum_modules", - "change_cwd", -] - - -MOCK_COMMAND_TEMPLATE = """#!{python} -import json, os, sys - -with open({output_file!r}, 'a') as fd: - json.dump( - {{'argv': sys.argv, 'cwd': os.getcwd(), 'env': dict(os.environ)}}, - fd) - fd.write('\\x1e') - -{user_code} -""" - - -class MockShellCommand(object): - - """ - Context manager to mock a shell command. - - Notes - ----- - The code is inspired by MockCommand class from the testpath project - (https://github.com/jupyter/testpath). There are some differences through: - - - the original code doesn't remove commands directory after a context - manager exit. - - the original code doesn't populate a user provided content with a - "recording_file" variable but creates the file and its parent directory - anyway. - - the original code uses global variables and contains unnecessary code - for compatibility with MS Windows. - """ - - def __init__(self, name, user_code=None, tmp_dir=None): - """ - Parameters - ---------- - name : str - Mocked shell command name. - user_code : str, optional - Additional Python code to be executed in the mocked command (e.g. - print some data). - tmp_dir : str, optional - Base directory for temporary files. - """ - self.__name = name - self.__user_code = user_code - self.__tmp_dir = tmp_dir - self.__command_dir = None - self.__output_file = None - - def __enter__(self): - self.__command_dir = tempfile.mkdtemp(prefix="castor_msc_", dir=self.__tmp_dir) - fd, self.__output_file = tempfile.mkstemp( - prefix="castor_msc_", dir=self.__tmp_dir - ) - os.close(fd) - self.__create_command_file() - self.modify_env_path(self.__command_dir) - return self - - def __create_command_file(self): - command_path = os.path.join(self.__command_dir, self.__name) - command = MOCK_COMMAND_TEMPLATE.format( - python=sys.executable, - output_file=self.__output_file, - user_code=self.__user_code or "", - ) - with open(command_path, "w") as fd: - fd.write(command) - os.chmod(command_path, 0o755) - - def get_calls(self): - if not self.__output_file or not os.path.isfile(self.__output_file): - return [] - with open(self.__output_file, "r") as fd: - return [json.loads(chunk) for chunk in fd.read().split("\x1e")[:-1]] - - @staticmethod - def modify_env_path(command_dir): - os.environ["PATH"] = "{0}{1}{2}".format( - command_dir, os.pathsep, os.environ["PATH"] - ) - - @staticmethod - def revert_env_path(command_dir): - paths = os.environ["PATH"].split(os.pathsep) - if command_dir in paths: - paths.remove(command_dir) - os.environ["PATH"] = os.pathsep.join(paths) - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.__command_dir: - self.revert_env_path(self.__command_dir) - if os.path.exists(self.__command_dir): - shutil.rmtree(self.__command_dir) - if self.__output_file and os.path.exists(self.__output_file): - os.remove(self.__output_file) - - -def unload_plumbum_modules(test_module): - """ - Unloads a `test_module` and all plumbum submodules except - `plumbum.commands.processes`. - - This is required because plumbum loads environment variables once when - module is imported, so we need to reload it after each test case to make - MockShellCommand work. - - Parameters - ---------- - test_module : str - Name of a module to unload alon with plumbum submodules. - """ - for mod in list(sys.modules.keys()): - if (mod == test_module) or ( - "plumbum" in mod and mod != "plumbum.commands.processes" - ): - del sys.modules[mod] - - -@contextlib.contextmanager -def change_cwd(path): - """ - Context manager that temporarily changes current working directory. - - Parameters - ---------- - path : str - Temporary working directory path. - - Returns - ------- - generator - """ - old_cwd = os.getcwd() - os.chdir(path) - try: - yield os.getcwd() - finally: - os.chdir(old_cwd)