diff --git a/SECI_Config_Analyser/Directory_Operations.py b/SECI_Config_Analyser/Directory_Operations.py index e55c800..9bd57cd 100644 --- a/SECI_Config_Analyser/Directory_Operations.py +++ b/SECI_Config_Analyser/Directory_Operations.py @@ -3,23 +3,20 @@ # 3. for each file, parse XML for required tag and add value to list # 4. remove duplicates from list -from os import listdir -from fnmatch import fnmatch import xml.etree.ElementTree as ET +from fnmatch import fnmatch +from os import listdir class ReadConfigFiles: - """ - Reads SECI configuration files and extracts VI names + """Reads SECI configuration files and extracts VI names """ def __init__(self, search_path): - """ - create lists for filenames and initialise to empty lists + """Create lists for filenames and initialise to empty lists call methods for reading directory contents and searching for config filenames :param search_path: the search path for files """ - self.full_path = search_path self.directory_contents = listdir(self.full_path) self.config_filenames = [] @@ -28,11 +25,9 @@ def __init__(self, search_path): self._extract_config_filenames() def _extract_config_filenames(self): - """ - extract SECI config (*.conf) & component (*.comp) filenames + """Extract SECI config (*.conf) & component (*.comp) filenames add to separate lists """ - for filename in self.directory_contents: if fnmatch(filename, "*.conf"): self.config_filenames.append(filename) @@ -41,12 +36,10 @@ def _extract_config_filenames(self): self.comp_filenames.append(filename) def _parse_vis_from_files(self, filenames): - """ - extracts VI names and paths from config files + """Extracts VI names and paths from config files :param filenames: list of filenames to process :return: vis_in_files: list of VIs with fullpaths """ - vis_in_files = [] for filename in filenames: @@ -58,12 +51,10 @@ def _parse_vis_from_files(self, filenames): @staticmethod def _parse_file(filename_and_path): - """ - reads XML file and creates list of values for "FilePath" tag + """Reads XML file and creates list of values for "FilePath" tag :param filename_and_path: absolute path of file to be parsed :return: vis_in_file: list of VIs in file """ - vis_in_file = [] tree = ET.parse(filename_and_path) root = tree.getroot() @@ -75,12 +66,10 @@ def _parse_file(filename_and_path): @staticmethod def _remove_duplicates(input_list): - """ - remove duplicates from list, then sort alphabetically + """Remove duplicates from list, then sort alphabetically :param input_list: input list :return: output_list: original list with duplicates removed """ - # maintain list of "encountered" items in dummy list # if item not in this list, add item to it and to output list @@ -99,12 +88,10 @@ def _remove_duplicates(input_list): return output_list def analyse_config_files(self): - """ - call methods to process files + """Call methods to process files :return: config_vis: list containing absolute paths of VIs within SECI config file :return: comp_vis: list containing absolute paths of VIs within SECI sub-config file """ - config_vis = self._remove_duplicates(self._parse_vis_from_files(self.config_filenames)) comp_vis = self._remove_duplicates(self._parse_vis_from_files(self.comp_filenames)) diff --git a/SECI_Config_Analyser/SECI_Config_Analyser.py b/SECI_Config_Analyser/SECI_Config_Analyser.py index 85d9db2..3a08422 100644 --- a/SECI_Config_Analyser/SECI_Config_Analyser.py +++ b/SECI_Config_Analyser/SECI_Config_Analyser.py @@ -1,11 +1,11 @@ -""" -Script to analyse SECI configurations and components and output results to file +"""Script to analyse SECI configurations and components and output results to file """ -from Directory_Operations import ReadConfigFiles import argparse from os import system +from Directory_Operations import ReadConfigFiles + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Analyse the SECI configuration files of specified instrument and output results to file" diff --git a/build_tools/check_builds_are_recent.py b/build_tools/check_builds_are_recent.py index f94502d..40b3593 100644 --- a/build_tools/check_builds_are_recent.py +++ b/build_tools/check_builds_are_recent.py @@ -1,8 +1,8 @@ +import json import os -from typing import Union -from datetime import datetime import sys -import json +from datetime import datetime +from typing import Union kits_root = r"\\isis\inst$\kits$\CompGroup\ICP" @@ -85,7 +85,7 @@ def check_build_dirs(build_dirs): if __name__ == "__main__": builds_to_check = os.getenv("BUILDS_TO_CHECK") if builds_to_check is None: - print(f"ERROR: BUILDS_TO_CHECK enviroment variable not set") + print("ERROR: BUILDS_TO_CHECK enviroment variable not set") sys.exit(1) try: builds_by_stale_times = json.loads(builds_to_check) diff --git a/build_tools/purge_archive.py b/build_tools/purge_archive.py index d10599f..97f6f7d 100644 --- a/build_tools/purge_archive.py +++ b/build_tools/purge_archive.py @@ -1,7 +1,7 @@ +import glob import os import shutil import stat -import glob from datetime import datetime, timedelta max_build_age_in_days = 30 diff --git a/data_gathering_scripts/data_sizes.py b/data_gathering_scripts/data_sizes.py index ee767f8..3695ffb 100644 --- a/data_gathering_scripts/data_sizes.py +++ b/data_gathering_scripts/data_sizes.py @@ -1,11 +1,11 @@ -""" -Look at the size of +"""Look at the size of """ import getpass import os import subprocess + from six.moves import input areas = { @@ -32,16 +32,14 @@ class CalibrationsFolder: - """ - Context manager for accessing calibration folders on remote instruments. + """Context manager for accessing calibration folders on remote instruments. """ DRIVE_LETTER = "q" @staticmethod def disconnect_from_drive(): - """ - Returns: True if disconnect is successful, else False. + """Returns: True if disconnect is successful, else False. """ return ( subprocess.call( @@ -53,8 +51,7 @@ def disconnect_from_drive(): ) def connect_to_drive(self): - """ - Returns: True if the connection is successful, else False + """Returns: True if the connection is successful, else False """ print("connecting") net_use_cmd_line = [ @@ -73,8 +70,7 @@ def __init__(self, instrument_host, username, password): self.password = password def __enter__(self): - """ - Returns: A git repository for the remote calibration folder if connection is successful, else None. + """Returns: A git repository for the remote calibration folder if connection is successful, else None. """ self.connect_to_drive() return self diff --git a/data_gathering_scripts/nexus_file_sample_data_log.py b/data_gathering_scripts/nexus_file_sample_data_log.py index c194f09..a1850d5 100644 --- a/data_gathering_scripts/nexus_file_sample_data_log.py +++ b/data_gathering_scripts/nexus_file_sample_data_log.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- -""" -Uses h5py to extract given log values from an ISIS NeXus file over a given time period. It is assumed +"""Uses h5py to extract given log values from an ISIS NeXus file over a given time period. It is assumed that the control software has logged when values changed. """ # std imports -from argparse import ArgumentParser -from collections import OrderedDict -from datetime import datetime, timedelta import logging import os.path as osp import sys +from argparse import ArgumentParser +from collections import OrderedDict +from datetime import datetime, timedelta + +import numpy as np # third party imports from h5py import File as HDF5File -import numpy as np from six import iteritems # Logger for this module @@ -48,8 +48,7 @@ def parse_args(): def fatal(msg, exit_code=1): - """ - End program by logging message and calling sys.exit with exit code = 1 + """End program by logging message and calling sys.exit with exit code = 1 :param msg: Message to display at error log level :param exit_code: Exit code for program. Default=1 @@ -59,8 +58,7 @@ def fatal(msg, exit_code=1): def logs_info(h5file, rel_time_start, names=None): - """ - Return values of log parameters when they change, i.e do not use every + """Return values of log parameters when they change, i.e do not use every log entry but only display values when they change. :param h5file: Reference to H5File open at root level @@ -82,8 +80,7 @@ def logs_info(h5file, rel_time_start, names=None): def find_log_info(h5group, start_timestamp, rel_time_start): - """ - Assumes that the times in the time array are relative to run start + """Assumes that the times in the time array are relative to run start :param h5group: The open log entry group :param start_timestamp: Start datetime of run :param rel_time_start: A time relative to run start to begin processing logs @@ -125,8 +122,7 @@ def append_value(value_list, time_in_seconds, value): def write_out(info, out_file): - """ - :param info: The dictionary of log information + """:param info: The dictionary of log information :param out_file: Destination to write output """ for log_name, values in iteritems(info): diff --git a/data_gathering_scripts/nexus_files_with_changing_blocks.py b/data_gathering_scripts/nexus_files_with_changing_blocks.py index ac6507a..859fdec 100644 --- a/data_gathering_scripts/nexus_files_with_changing_blocks.py +++ b/data_gathering_scripts/nexus_files_with_changing_blocks.py @@ -1,11 +1,10 @@ -""" -Utility +"""Utility """ +import os from datetime import datetime, timedelta import h5py -import os # List of machines to perform analysis for machines = ( @@ -32,8 +31,7 @@ def check(machine, cycle, outf): - """ - Check that every data file contains the same blocks as previous data files or that the last data file is 1000s ago + """Check that every data file contains the same blocks as previous data files or that the last data file is 1000s ago :param machine: machine to look at :param cycle: cycle to use :param outf: file to output results to diff --git a/developer_support_script/build_dependencies.py b/developer_support_script/build_dependencies.py index 2d46deb..74e2968 100644 --- a/developer_support_script/build_dependencies.py +++ b/developer_support_script/build_dependencies.py @@ -1,9 +1,7 @@ -""" -Utility to help create build dependencies +"""Utility to help create build dependencies """ import argparse - import os import re @@ -301,8 +299,7 @@ def find(pattern, path): - """ - Find a file which matches a regex + """Find a file which matches a regex Args: pattern: regex to find path: path to start in @@ -319,8 +316,7 @@ def find(pattern, path): def macro_dependencies(ioc_dir): - """ - Get a list of possible dependenices for the macros + """Get a list of possible dependenices for the macros Args: ioc_dir: @@ -355,12 +351,10 @@ def macro_dependencies(ioc_dir): def build_dependencies(ioc_dir): - """ - Create a list of dependencies from build.mak + """Create a list of dependencies from build.mak :param ioc_dir: path for the IOC :return: set of dependencies """ - file_list = find(r"(build.mak)|(Makefile)", ioc_dir) if len(file_list) < 1: @@ -388,8 +382,7 @@ def build_dependencies(ioc_dir): def get_entries(dependencies, macros): - """ - Get entries to add to release file + """Get entries to add to release file Args: dependencies: dependencies for the IOC macros: macros to add (yes I know it is not nice but it works) @@ -415,15 +408,14 @@ def get_entries(dependencies, macros): ) print("A list of possible dependencies can be found in EPICS\\configure\\MASTER_RELEASE)") exit(1) - sorted_lines = sorted([line for line in lines if line is not ""]) + sorted_lines = sorted([line for line in lines if line != ""]) to_print = "\n".join(sorted_lines) print(" - Adding lines:\n {}".format(to_print.replace("\n", "\n "))) return sorted_lines def write_extra_lines(outfile, lines): - """ - Add extra lines in to build file + """Add extra lines in to build file Args: outfile: file to out to lines: lines to add @@ -435,8 +427,8 @@ def write_extra_lines(outfile, lines): def replace_config_lines(lines, ioc_dir): - """ - Replace the expected config lines with generated lines. + """Replace the expected config lines with generated lines. + Args: lines: lines to replace. ioc_dir: ioc directory @@ -465,8 +457,8 @@ def replace_config_lines(lines, ioc_dir): def replace_dependencies_in_release_file(base_dir): - """ - Replaces the dependencies in the RELEASE file with dependencies from the base directory. + """Replaces the dependencies in the RELEASE file with dependencies from the base directory. + Args: base_dir: the base directory which contains the release file """ diff --git a/developer_support_script/check_build_stability.py b/developer_support_script/check_build_stability.py index 4503020..f0132b2 100644 --- a/developer_support_script/check_build_stability.py +++ b/developer_support_script/check_build_stability.py @@ -1,20 +1,18 @@ -""" -A utility script to retrieve tests data from Jenkins jobs. +"""A utility script to retrieve tests data from Jenkins jobs. Checks for common failures as Jenkins only tracks continuous failures. """ -import requests from collections import Counter, defaultdict from typing import Any +import requests WARNING_THRESHOLD_PERCENTAGE = 10 ERROR_THRESHOLD_PERCENTAGE = 50 def request_json(url: str) -> Any: - """ - Utility function to get Json data from Jenkins. + """Utility function to get Json data from Jenkins. Args: url: The URL to request. @@ -29,8 +27,7 @@ def request_json(url: str) -> Any: def calculate_level(percentage: int, error_percentage: int, warning_percentage: int) -> str: - """ - Utility function to calculate log level based on a percentage. + """Utility function to calculate log level based on a percentage. Args: percentage: The percentage to calculate. @@ -46,8 +43,7 @@ def calculate_level(percentage: int, error_percentage: int, warning_percentage: class JobData: - """ - Calculates and prints common test failures and number of aborted builds in a Jenkins job. + """Calculates and prints common test failures and number of aborted builds in a Jenkins job. """ def __init__(self, name: str) -> None: @@ -64,8 +60,7 @@ def __init__(self, name: str) -> None: self.num_aborted_builds = self._get_num_aborted_builds() def add_job(self, job): - """ - add another job's results to here so we can get a summary across jobs + """Add another job's results to here so we can get a summary across jobs """ self.no_test_report_failures += job.no_test_report_failures self.test_reports.append(job.test_reports) @@ -74,8 +69,7 @@ def add_job(self, job): self.num_aborted_builds += job.num_aborted_builds def _get_builds(self) -> defaultdict[str, Any]: - """ - Gets all the build data grouped by status. Ignores builds currently in progress. + """Gets all the build data grouped by status. Ignores builds currently in progress. Returns: defaultdict: Key is build status, value is the builds Json data. Defaults to empty list. @@ -92,8 +86,7 @@ def _get_builds(self) -> defaultdict[str, Any]: return builds def _get_num_evaluate_builds(self) -> int: - """ - Gets the number of builds that have valid test results. + """Gets the number of builds that have valid test results. """ num = 0 @@ -105,8 +98,7 @@ def _get_num_evaluate_builds(self) -> int: return num def _get_num_aborted_builds(self) -> int: - """ - Gets the number of aborted builds. Ignores builds that were aborted manually. + """Gets the number of aborted builds. Ignores builds that were aborted manually. This will generally be the number of builds that timed out. """ aborted_manually = 0 @@ -133,8 +125,7 @@ def _get_num_aborted_builds(self) -> int: return len(self.builds["ABORTED"]) - aborted_manually def _get_test_reports(self) -> list[Any]: - """ - Gets the test reports of the builds that have failing tests results. + """Gets the test reports of the builds that have failing tests results. Returns: list: A list of test reports Json data. @@ -152,8 +143,7 @@ def _get_test_reports(self) -> list[Any]: return test_reports_json def _get_failed_tests(self) -> Counter: - """ - Gets all the failed tests. The name is generated by using the the class name and the test name. + """Gets all the failed tests. The name is generated by using the the class name and the test name. Returns: Counter: Key is test case name, value is number of failures. @@ -169,8 +159,7 @@ def _get_failed_tests(self) -> Counter: return counter def print_results(self) -> None: - """ - Prints the percentage of aborted builds for the job, the percentage of failures with no test report, and the percentage failure of each failing test. + """Prints the percentage of aborted builds for the job, the percentage of failures with no test report, and the percentage failure of each failing test. """ if not self.buildable: print("WARNING: build is currently disabled") diff --git a/developer_support_script/install_files.py b/developer_support_script/install_files.py index ee767f8..3695ffb 100644 --- a/developer_support_script/install_files.py +++ b/developer_support_script/install_files.py @@ -1,11 +1,11 @@ -""" -Look at the size of +"""Look at the size of """ import getpass import os import subprocess + from six.moves import input areas = { @@ -32,16 +32,14 @@ class CalibrationsFolder: - """ - Context manager for accessing calibration folders on remote instruments. + """Context manager for accessing calibration folders on remote instruments. """ DRIVE_LETTER = "q" @staticmethod def disconnect_from_drive(): - """ - Returns: True if disconnect is successful, else False. + """Returns: True if disconnect is successful, else False. """ return ( subprocess.call( @@ -53,8 +51,7 @@ def disconnect_from_drive(): ) def connect_to_drive(self): - """ - Returns: True if the connection is successful, else False + """Returns: True if the connection is successful, else False """ print("connecting") net_use_cmd_line = [ @@ -73,8 +70,7 @@ def __init__(self, instrument_host, username, password): self.password = password def __enter__(self): - """ - Returns: A git repository for the remote calibration folder if connection is successful, else None. + """Returns: A git repository for the remote calibration folder if connection is successful, else None. """ self.connect_to_drive() return self diff --git a/galil_ini_parser/change_home_conventions.py b/galil_ini_parser/change_home_conventions.py index 4e223a7..1a08165 100644 --- a/galil_ini_parser/change_home_conventions.py +++ b/galil_ini_parser/change_home_conventions.py @@ -1,6 +1,5 @@ import argparse import pathlib - from distutils.util import strtobool from galil_ini_parser import ( diff --git a/galil_ini_parser/galil_ini_parser.py b/galil_ini_parser/galil_ini_parser.py index 0cf8dfa..caabeeb 100644 --- a/galil_ini_parser/galil_ini_parser.py +++ b/galil_ini_parser/galil_ini_parser.py @@ -1,7 +1,6 @@ import re - from collections import OrderedDict -from typing import Optional, Any, Dict, List +from typing import Any, Dict, List, Optional AXIS_NUMBER_REGEX = r"(?<=Axis )\S+" @@ -20,8 +19,7 @@ def apply_home_shift( old_homeval: float, old_offset: float, old_user_offset: float, old_hlim: float, old_llim: float ) -> Dict[str, Optional[str]]: - """ - Combines both the user and axis offsets into one (the new axis offset). + """Combines both the user and axis offsets into one (the new axis offset). Shifts the axis limits by the difference between the old and new home position Args: @@ -33,7 +31,6 @@ def apply_home_shift( Returns: new_settings: Dictionary containing the string representation of the updated values. """ - old_combined_offset = old_offset + old_user_offset # Set offset = homeval, so home position written to device is (homeval - offset) = 0 @@ -70,8 +67,7 @@ def __init__(self, crate_index: int): self.axes = {} def parse_line(self, line) -> None: - """ - Adds a line of the galil ini file to this class + """Adds a line of the galil ini file to this class """ axis_index = self.get_axis_letter_from_line(line) if axis_index is None: @@ -90,8 +86,7 @@ def parse_line(self, line) -> None: self.axes[axis_index] = new_axis def get_axis_letter_from_line(self, ini_line: str) -> str: - """ - Extracts the letter of the axis referred to in a line of the ini file + """Extracts the letter of the axis referred to in a line of the ini file """ matches = re.search(AXIS_NUMBER_REGEX, ini_line) if matches is not None: @@ -100,8 +95,7 @@ def get_axis_letter_from_line(self, ini_line: str) -> str: return None def get_save_strings(self) -> List[str]: - """ - Returns a strings can containing all settings for all axes on this crate + """Returns a strings can containing all settings for all axes on this crate """ settings = ["[{}]".format(self.crate_index)] for crate_setting, setting_value in self.settings.items(): @@ -126,8 +120,7 @@ def __init__(self, axis_index: str): self.axis_line_prefix = "Axis {}".format(axis_index) def scrub_axis_prefix(self, ini_line: str) -> str: - """ - Removes the axis identifier from a line in the ini file. Returns a string which defines its setting + """Removes the axis identifier from a line in the ini file. Returns a string which defines its setting Args: ini_line: Line from the ini file @@ -138,8 +131,7 @@ def scrub_axis_prefix(self, ini_line: str) -> str: return ini_line[len(self.axis_line_prefix) :].strip() def add_setting_from_ini_line(self, ini_line: str): - """ - Imports a new setting from a line in the galil ini file + """Imports a new setting from a line in the galil ini file """ setting = self.scrub_axis_prefix(ini_line) split_line = setting.split("=") @@ -150,8 +142,7 @@ def add_setting_from_ini_line(self, ini_line: str): self.set_value(key, value, make_new=True) def get_value(self, setting: str, caster: Any, default_value: Any = None) -> Optional[Any]: - """ - Attempts to get the setting for this axis and cast it to type 'type'. + """Attempts to get the setting for this axis and cast it to type 'type'. Args: setting: The name of the setting to get @@ -161,7 +152,6 @@ def get_value(self, setting: str, caster: Any, default_value: Any = None) -> Opt Returns: value: The setting, cast using supplied caster. Equals default_value if setting does not exist on this axis """ - try: value = caster(self.settings[setting]) except KeyError: @@ -170,8 +160,7 @@ def get_value(self, setting: str, caster: Any, default_value: Any = None) -> Opt return value def set_value(self, setting: str, value: str, make_new: bool = False) -> None: - """ - Sets the setting with the specified value + """Sets the setting with the specified value Args: setting: The setting to set @@ -183,8 +172,7 @@ def set_value(self, setting: str, value: str, make_new: bool = False) -> None: def extract_galil_settings_from_file(file: List[str]) -> Dict[str, Galil]: - """ - Given a file contatining galil settings, extract the settings as Galil objects + """Given a file contatining galil settings, extract the settings as Galil objects Args: file: List of lines from the settings files diff --git a/galil_ini_parser/parser_tests.py b/galil_ini_parser/parser_tests.py index 924374b..e284a5e 100644 --- a/galil_ini_parser/parser_tests.py +++ b/galil_ini_parser/parser_tests.py @@ -1,16 +1,17 @@ import unittest +from typing import Dict + +from mock import patch +from parameterized import parameterized -from galil_ini_parser import Galil, Axis from galil_ini_parser import ( + Axis, + Galil, apply_home_shift, common_setting_names, extract_galil_settings_from_file, ) -from mock import patch, call -from parameterized import parameterized -from typing import Dict - GALIL_CRATE_INDEX = "G1" AXIS_INDEX = "A" @@ -21,8 +22,7 @@ def POPULATE_AXIS( offset: float, user_offset: float, homeval: float, hlim: float, llim: float ) -> Dict[str, float]: - """ - Creates a dictionary with key/value pairs corresponding to axis parameters to be changed + """Creates a dictionary with key/value pairs corresponding to axis parameters to be changed """ axis = { common_setting_names["OFFSET"]: offset, @@ -62,8 +62,7 @@ def POPULATE_AXIS( class GalilTests(unittest.TestCase): - """ - Tests for the galil crate class + """Tests for the galil crate class """ def setUp(self): @@ -166,8 +165,7 @@ def test_GIVEN_settings_on_crate_and_axes_WHEN_save_string_requested_THEN_settin class AxisTests(unittest.TestCase): - """ - Tests for the galil axis class + """Tests for the galil axis class """ def setUp(self): @@ -252,8 +250,7 @@ def test_GIVEN_new_setting_is_set_WHEN_make_new_is_true_THEN_new_setting_is_adde class IniParserTests(unittest.TestCase): - """ - Tests for the code which changes SECI offsets to IBEX offsets + """Tests for the code which changes SECI offsets to IBEX offsets """ @parameterized.expand(TEST_AXIS_CASES) diff --git a/installation_and_upgrade/IBEX_upgrade.py b/installation_and_upgrade/IBEX_upgrade.py index 35a11e0..e548e6e 100644 --- a/installation_and_upgrade/IBEX_upgrade.py +++ b/installation_and_upgrade/IBEX_upgrade.py @@ -1,18 +1,17 @@ -""" -Script to install IBEX to various machines +"""Script to install IBEX to various machines """ import argparse import os import re import sys -import semantic_version -from ibex_install_utils.install_tasks import UpgradeInstrument, UPGRADE_TYPES -from ibex_install_utils.exceptions import UserStop, ErrorInTask -from ibex_install_utils.user_prompt import UserPrompt +import semantic_version +from ibex_install_utils.exceptions import ErrorInTask, UserStop from ibex_install_utils.file_utils import FileUtils +from ibex_install_utils.install_tasks import UPGRADE_TYPES, UpgradeInstrument from ibex_install_utils.logger import Logger +from ibex_install_utils.user_prompt import UserPrompt def _get_latest_release_path(release_dir): diff --git a/installation_and_upgrade/calibration_files_updater.py b/installation_and_upgrade/calibration_files_updater.py index 45a707b..8207f66 100644 --- a/installation_and_upgrade/calibration_files_updater.py +++ b/installation_and_upgrade/calibration_files_updater.py @@ -1,13 +1,13 @@ -""" -Script to update calibration files on instruments +"""Script to update calibration files on instruments """ -import git -import subprocess +import getpass import json import logging import os -import getpass +import subprocess + +import git from genie_python import genie as g from genie_python.utilities import dehex_and_decompress from six.moves import input @@ -16,16 +16,14 @@ class CalibrationsFolder: - """ - Context manager for accessing calibration folders on remote instruments. + """Context manager for accessing calibration folders on remote instruments. """ DRIVE_LETTER = "q" @staticmethod def disconnect_from_drive(): - """ - Returns: True if disconnect is successful, else False. + """Returns: True if disconnect is successful, else False. """ return ( subprocess.call( @@ -37,8 +35,7 @@ def disconnect_from_drive(): ) def connect_to_drive(self): - """ - Returns: True if the connection is successful, else False + """Returns: True if the connection is successful, else False """ return ( subprocess.call( @@ -62,8 +59,7 @@ def __init__(self, instrument_host, username, password): self.password = password def __enter__(self): - """ - Returns: A git repository for the remote calibration folder if connection is successful, else None. + """Returns: A git repository for the remote calibration folder if connection is successful, else None. """ self.disconnect_from_drive() return git.Repo(self.network_location) if self.connect_to_drive() else None @@ -73,15 +69,13 @@ def __exit__(self, *args): def get_instrument_hosts(): - """ - Returns: A collection of instrument host names + """Returns: A collection of instrument host names """ return (inst["hostName"] for inst in json.loads(dehex_and_decompress(g.get_pv("CS:INSTLIST")))) def update_instrument(host, username, password, logger, dry_run=False): - """ - Updates the calibration files on the named host. + """Updates the calibration files on the named host. Args: host: The instrument host to update diff --git a/installation_and_upgrade/ibex_install_utils/admin_runner.py b/installation_and_upgrade/ibex_install_utils/admin_runner.py index e9aead0..5b1535b 100644 --- a/installation_and_upgrade/ibex_install_utils/admin_runner.py +++ b/installation_and_upgrade/ibex_install_utils/admin_runner.py @@ -1,19 +1,19 @@ import contextlib +import os import tempfile from time import sleep -import os class AdminRunner: @staticmethod def run_command(command, parameters, expected_return_val=0): try: - from win32com.shell.shell import ShellExecuteEx - from win32com.shell import shellcon - import win32event - import win32process import win32api import win32con + import win32event + import win32process + from win32com.shell import shellcon + from win32com.shell.shell import ShellExecuteEx except ImportError: raise OSError("Can only elevate privileges on windows") @@ -36,8 +36,7 @@ def run_command(command, parameters, expected_return_val=0): class AdminCommandBuilder: - """ - Builder for running multiple commands sequentially as admin. + """Builder for running multiple commands sequentially as admin. """ def __init__(self): diff --git a/installation_and_upgrade/ibex_install_utils/ca_utils.py b/installation_and_upgrade/ibex_install_utils/ca_utils.py index 520f032..f69cafe 100644 --- a/installation_and_upgrade/ibex_install_utils/ca_utils.py +++ b/installation_and_upgrade/ibex_install_utils/ca_utils.py @@ -2,13 +2,11 @@ class CaWrapper: - """ - Wrapper around genie python's channel access class providing some useful abstractions. + """Wrapper around genie python's channel access class providing some useful abstractions. """ def __init__(self): - """ - Setting instrument is necessary because genie_python is being run from a network drive so it may not know + """Setting instrument is necessary because genie_python is being run from a network drive so it may not know where it is. """ self.g = None @@ -25,8 +23,7 @@ def _get_genie(self): return g def get_local_pv(self, name): - """ - Get PV with the local PV prefix appended + """Get PV with the local PV prefix appended Args: name (str): Name of the PV to get. @@ -37,8 +34,7 @@ def get_local_pv(self, name): return self._get_genie().get_pv(name, is_local=True) def get_object_from_compressed_hexed_json(self, name): - """ - Gets an object from a compressed hexed json PV + """Gets an object from a compressed hexed json PV Args: name (str): Name of the PV to get @@ -55,21 +51,18 @@ def get_object_from_compressed_hexed_json(self, name): return dehex_and_decompress(data) def get_blocks(self): - """ - Returns: - A collection of blocks, or None if the PV was not connected + """Returns: + A collection of blocks, or None if the PV was not connected """ return self._get_genie().get_blocks() def cget(self, block): - """ - Returns: - A collection of blocks, or None if the PV was not connected. + """Returns: + A collection of blocks, or None if the PV was not connected. """ return self._get_genie().cget(block) def set_pv(self, *args, **kwargs): - """ - Sets the value of a PV. + """Sets the value of a PV. """ return self._get_genie().set_pv(*args, **kwargs) diff --git a/installation_and_upgrade/ibex_install_utils/exceptions.py b/installation_and_upgrade/ibex_install_utils/exceptions.py index 3d82348..d51d7b4 100644 --- a/installation_and_upgrade/ibex_install_utils/exceptions.py +++ b/installation_and_upgrade/ibex_install_utils/exceptions.py @@ -1,19 +1,16 @@ -""" -Exceptions thrown by the upgrade utilities +"""Exceptions thrown by the upgrade utilities """ class UserStop(Exception): - """ - Exception when a task stops because of a user request + """Exception when a task stops because of a user request """ pass class ErrorInTask(Exception): - """ - Exception if there is an error when running a task. + """Exception if there is an error when running a task. """ def __init__(self, message): @@ -21,8 +18,7 @@ def __init__(self, message): class ErrorInRun(ErrorInTask): - """ - Exception if there is an error when running a process. + """Exception if there is an error when running a process. """ def __init__(self, message): @@ -30,8 +26,7 @@ def __init__(self, message): class ErrorWithFile(ErrorInTask): - """ - Exception if there is an error when doing something with a file + """Exception if there is an error when doing something with a file """ def __init__(self, message): diff --git a/installation_and_upgrade/ibex_install_utils/file_utils.py b/installation_and_upgrade/ibex_install_utils/file_utils.py index e9d003a..e29493a 100644 --- a/installation_and_upgrade/ibex_install_utils/file_utils.py +++ b/installation_and_upgrade/ibex_install_utils/file_utils.py @@ -1,20 +1,20 @@ -""" -Filesystem utility classes +"""Filesystem utility classes """ import binascii +import logging +import msilib import os import shutil import tempfile import time -import zlib import zipfile -import logging +import zlib +from contextlib import contextmanager + import win32api -import msilib from ibex_install_utils.exceptions import UserStop from ibex_install_utils.run_process import RunProcess -from contextlib import contextmanager LABVIEW_DAE_DIR = os.path.join("C:\\", "LabVIEW modules", "DAE") @@ -63,16 +63,14 @@ def get_size(path="."): class FileUtils: - """ - Various utilities for interacting with the file system + """Various utilities for interacting with the file system """ @staticmethod def is_junction(path: str) -> bool: - """ - is path a junction - Args: - path: path to check + """Is path a junction + Args: + path: path to check """ try: return bool(os.readlink(path)) @@ -81,8 +79,7 @@ def is_junction(path: str) -> bool: @staticmethod def remove_tree(path, prompt, use_robocopy=True, retries=10, leave_top_if_link=False): - """ - Delete a file path if it exists + """Delete a file path if it exists Args: path: path to delete prompt (ibex_install_utils.user_prompt.UserPrompt): prompt object to communicate with user @@ -192,8 +189,7 @@ def winapi_path(dos_path): return _winapi_path(dos_path) def mkdir_recursive(self, path): - """ - Make a directory and all its ancestors + """Make a directory and all its ancestors Args: path: path of directory @@ -208,8 +204,7 @@ def mkdir_recursive(self, path): @staticmethod def move_dir(src, dst, prompt): - """ - Moves a dir. Better to copy remove so we can handle permissions issues + """Moves a dir. Better to copy remove so we can handle permissions issues Args: src: Source directory @@ -221,8 +216,7 @@ def move_dir(src, dst, prompt): @staticmethod def move_file(source, destination, prompt): - """ - Move a file from the source to destination + """Move a file from the source to destination Args: source: source path destination: destination path @@ -311,6 +305,7 @@ def get_version(path: str): Args: path: The path to the file. + Returns: The string version (x.x.x.x) on successful read, None otherwise. """ @@ -332,6 +327,7 @@ def get_msi_property(path: str, property: str) -> str: Args: path: The path to the file. property: The property to read. + Returns: The string value of the property on successful read, None otherwise. """ @@ -350,10 +346,9 @@ def get_msi_property(path: str, property: str) -> str: @contextmanager def file_in_zip(zipname, peek_at_filename: str): - """ - Usage: - with file_in_zip(zipname, filename_within_zip) as file: - do something + """Usage: + with file_in_zip(zipname, filename_within_zip) as file: + do something """ _, file_extension = os.path.splitext(peek_at_filename) try: diff --git a/installation_and_upgrade/ibex_install_utils/install_tasks.py b/installation_and_upgrade/ibex_install_utils/install_tasks.py index a381bcf..7a2f2de 100644 --- a/installation_and_upgrade/ibex_install_utils/install_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/install_tasks.py @@ -1,13 +1,12 @@ -""" -Tasks associated with install +"""Tasks associated with install """ import json import os import warnings -from genie_python import genie as g -from ibex_install_utils.file_utils import FileUtils, LABVIEW_DAE_DIR +from genie_python import genie as g +from ibex_install_utils.file_utils import LABVIEW_DAE_DIR, FileUtils from ibex_install_utils.tasks.backup_tasks import BackupTasks from ibex_install_utils.tasks.client_tasks import ClientTasks from ibex_install_utils.tasks.git_tasks import GitTasks @@ -19,8 +18,7 @@ class UpgradeInstrument: - """ - Class to upgrade the instrument installation to the given version of IBEX. + """Class to upgrade the instrument installation to the given version of IBEX. """ def __init__( @@ -32,8 +30,8 @@ def __init__( ibex_version, file_utils=FileUtils(), ): - """ - Initializer. + """Initializer. + Args: user_prompt: a object to allow prompting of the user server_source_dir: directory to install ibex server from @@ -116,16 +114,14 @@ def __init__( @staticmethod def icp_in_labview_modules(): - """ - Condition on which to install ICP_Binaries or + """Condition on which to install ICP_Binaries or :return: True if the ICP is installed in labview modules, False otherwise """ return os.path.exists(LABVIEW_DAE_DIR) def run_test_update(self): - """ - Run a complete test upgrade on the current system + """Run a complete test upgrade on the current system """ self._system_tasks.user_confirm_upgrade_type_on_machine("Training Machine") self._system_tasks.install_or_upgrade_git() @@ -143,8 +139,7 @@ def run_test_update(self): self._system_tasks.upgrade_notepad_pp() def remove_all_and_install_client_and_server(self): - """ - Either install or upgrade the ibex client and server + """Either install or upgrade the ibex client and server """ self._system_tasks.confirm( "This script removes IBEX client and server and installs the latest build of both, and upgrade the " @@ -162,16 +157,14 @@ def remove_all_and_install_client_and_server(self): self._server_tasks.install_shared_scripts_repository() def run_instrument_tests(self): - """ - Run through client and server tests once installation / deployment has completed. + """Run through client and server tests once installation / deployment has completed. """ self._client_tasks.perform_client_tests() self._server_tasks.perform_server_tests() self._system_tasks.inform_instrument_scientists() def run_instrument_install(self): - """ - Do a first installation of IBEX on a new instrument. + """Do a first installation of IBEX on a new instrument. """ self._system_tasks.confirm( "This script performs a first-time full installation of the IBEX server and client" @@ -213,8 +206,7 @@ def save_motor_params(self): self._server_tasks.save_motor_parameters_to_file() def run_instrument_deploy(self): - """ - Deploy a full IBEX upgrade on an existing instrument. + """Deploy a full IBEX upgrade on an existing instrument. """ self._system_tasks.confirm( "This script performs a full upgrade of the IBEX server and client on an existing instrument. Proceed?" @@ -224,8 +216,7 @@ def run_instrument_deploy(self): self.run_instrument_deploy_post_start() def run_instrument_deploy_post_start(self): - """ - Upgrade an instrument. Steps to do after ibex has been started. + """Upgrade an instrument. Steps to do after ibex has been started. Current the server can not be started in this python script. """ @@ -240,8 +231,7 @@ def run_instrument_deploy_post_start(self): self._system_tasks.inform_instrument_scientists() def run_instrument_deploy_main(self): - """ - Upgrade an instrument. Steps to do after ibex has been stopped but before it is restarted. + """Upgrade an instrument. Steps to do after ibex has been stopped but before it is restarted. Current the server can not be started or stopped in this python script. """ @@ -268,8 +258,7 @@ def run_instrument_deploy_main(self): self._python_tasks.remove_instrument_script_githooks() def run_instrument_deploy_pre_stop(self): - """ - Upgrade an instrument. Steps to do before ibex is stopped. + """Upgrade an instrument. Steps to do before ibex is stopped. Current the server can not be started or stopped in this python script. """ @@ -300,8 +289,7 @@ def run_instrument_deploy_pre_stop(self): self._server_tasks.save_motor_blocks_blockserver_to_file() def run_truncate_database(self): - """ - Backup and truncate databases only + """Backup and truncate databases only """ self._mysql_tasks.backup_database() self._mysql_tasks.truncate_database() @@ -313,16 +301,14 @@ def run_force_upgrade_mysql(self): self._mysql_tasks.install_mysql(force=True) def run_developer_update(self): - """ - Update all the developer tools to latest version + """Update all the developer tools to latest version """ self._mysql_tasks.install_mysql(force=False) self._system_tasks.check_java_installation() self._system_tasks.install_or_upgrade_git() def run_vhd_creation(self): - """ - Automated job which creates a set of VHDs containing all IBEX components. + """Automated job which creates a set of VHDs containing all IBEX components. Note: this will run under jenkins, don't add interactive tasks to this list. """ @@ -352,26 +338,22 @@ def run_vhd_creation(self): self._vhd_tasks.deploy_vhds() def mount_vhds(self): - """ - Task which actually mounts the VHDs (will be run as admin) + """Task which actually mounts the VHDs (will be run as admin) """ self._vhd_tasks.mount_vhds() def dismount_vhds(self): - """ - Task which actually dismounts the VHDs (will be run as admin) + """Task which actually dismounts the VHDs (will be run as admin) """ self._vhd_tasks.dismount_vhds() def request_dismount_vhds(self): - """ - Standalone task to request VHDs to be dismounted + """Standalone task to request VHDs to be dismounted """ self._vhd_tasks.request_dismount_vhds() def run_vhd_post_install(self): - """ - This job is run by the MDT build system when it has built a windows image and mounted the VHDS + """This job is run by the MDT build system when it has built a windows image and mounted the VHDS It will tidy up and remaining jobs that were not possible when the vdh was created e.g. register mysql service """ # self._server_tasks.update_icp(self.icp_in_labview_modules()) diff --git a/installation_and_upgrade/ibex_install_utils/kafka_utils.py b/installation_and_upgrade/ibex_install_utils/kafka_utils.py index 46d02ac..11c805b 100644 --- a/installation_and_upgrade/ibex_install_utils/kafka_utils.py +++ b/installation_and_upgrade/ibex_install_utils/kafka_utils.py @@ -9,8 +9,7 @@ def get_existing_topics(kafka_broker): def add_required_topics(kafka_broker, instrument): - """ - Adds required Kafka topics for the instrument + """Adds required Kafka topics for the instrument Args: kafka_broker: the broker to add the topics to diff --git a/installation_and_upgrade/ibex_install_utils/logger.py b/installation_and_upgrade/ibex_install_utils/logger.py index 2b3d97c..c897c64 100644 --- a/installation_and_upgrade/ibex_install_utils/logger.py +++ b/installation_and_upgrade/ibex_install_utils/logger.py @@ -4,8 +4,7 @@ class Logger: - """ - Logger class used to capture output and input to a log file. + """Logger class used to capture output and input to a log file. """ def __init__(self): diff --git a/installation_and_upgrade/ibex_install_utils/motor_params.py b/installation_and_upgrade/ibex_install_utils/motor_params.py index ddf45ec..137af5a 100644 --- a/installation_and_upgrade/ibex_install_utils/motor_params.py +++ b/installation_and_upgrade/ibex_install_utils/motor_params.py @@ -1,11 +1,11 @@ -""" -Script to extract information from motors to be consumed by the motion controls team. Exports data as CSV. To run, +"""Script to extract information from motors to be consumed by the motion controls team. Exports data as CSV. To run, load the script into a genie_python console and run as a standard user script. """ import csv -import sys import multiprocessing.dummy as multiprocessing +import sys + from genie_python import genie as g g.set_instrument(None, import_instrument_init=False) @@ -112,8 +112,7 @@ def pv_exists(pv): - """ - Gets whether a pv exists. + """Gets whether a pv exists. Args: pv (string): The PV to check. @@ -130,8 +129,7 @@ def pv_exists(pv): def get_params_for_one_axis(axis, data, g, progress, total): - """ - Gets all the interesting parameters for one axis + """Gets all the interesting parameters for one axis Args: axis (string): The PV of the axis. @@ -180,8 +178,7 @@ def update_progress_bar(progress, total, width=20): def get_params_and_save_to_file(file_reference, num_of_controllers=8): - """ - Gets all the motor parameters and saves them to an open file reference as a csv. + """Gets all the motor parameters and saves them to an open file reference as a csv. Args: file_reference (BinaryIO): The csv file to save the data to. @@ -232,8 +229,7 @@ def get_motor_number(item): def get_params_and_save(file_name, num_of_controllers=8): - """ - Gets all the motor parameters and saves them to a file by name as a csv. + """Gets all the motor parameters and saves them to a file by name as a csv. Args: file_name: name of the file to save to diff --git a/installation_and_upgrade/ibex_install_utils/progress_bar.py b/installation_and_upgrade/ibex_install_utils/progress_bar.py index 698ac2f..5ba5671 100644 --- a/installation_and_upgrade/ibex_install_utils/progress_bar.py +++ b/installation_and_upgrade/ibex_install_utils/progress_bar.py @@ -2,8 +2,7 @@ class ProgressBar: - """ - A simple progress bar that can be used to print/update progress in the terminal. + """A simple progress bar that can be used to print/update progress in the terminal. To use first set the total value. To show progress bar call print() and to change progress modify the value of progress accordingly. diff --git a/installation_and_upgrade/ibex_install_utils/run_process.py b/installation_and_upgrade/ibex_install_utils/run_process.py index 8f1b35f..2258618 100644 --- a/installation_and_upgrade/ibex_install_utils/run_process.py +++ b/installation_and_upgrade/ibex_install_utils/run_process.py @@ -1,16 +1,15 @@ -""" -Running processes infrastructure. +"""Running processes infrastructure. """ import os import subprocess -from typing import Union, List +from typing import List, Union + from ibex_install_utils.exceptions import ErrorInRun class RunProcess: - """ - Create a process runner to run a process. + """Create a process runner to run a process. """ def __init__( @@ -27,8 +26,7 @@ def __init__( capture_last_output=False, progress_metric=[], ): - """ - Create a process that needs running + """Create a process that needs running Args: working_dir: working directory of the process @@ -66,8 +64,7 @@ def __init__( self._full_path_to_process_file = os.path.join(executable_directory, executable_file) def run(self): - """ - Run the process + """Run the process Returns: Raises ErrorInRun: if there is a known problem with the run diff --git a/installation_and_upgrade/ibex_install_utils/software_dependency/__init__.py b/installation_and_upgrade/ibex_install_utils/software_dependency/__init__.py index 9bce4b2..06f2e66 100644 --- a/installation_and_upgrade/ibex_install_utils/software_dependency/__init__.py +++ b/installation_and_upgrade/ibex_install_utils/software_dependency/__init__.py @@ -1,40 +1,35 @@ import os import re - from abc import ABC, abstractmethod + from ibex_install_utils.tasks.common_paths import THIRD_PARTY_INSTALLERS_LATEST_DIR class SoftwareDependency(ABC): - """ - Represents a software dependency installed on the system. + """Represents a software dependency installed on the system. """ @abstractmethod def get_name(self) -> str: - """ - Return user friendly name of the software. + """Return user friendly name of the software. """ ... @abstractmethod def get_installed_version(self) -> str: - """ - Return the installed version of the software. + """Return the installed version of the software. """ ... @abstractmethod def get_file_pattern(self) -> str: - """ - Return the regex to match the files describing different versions of this software. + """Return the regex to match the files describing different versions of this software. """ ... @abstractmethod def get_version_of(self, path: str) -> str: - """ - Return the version of the software dependency described by the path. + """Return the version of the software dependency described by the path. Args: path: The path to the dir/file. @@ -42,14 +37,12 @@ def get_version_of(self, path: str) -> str: ... def get_search_dir(self) -> str: - """ - Return the path to the directory containing the available versions of the software. + """Return the path to the directory containing the available versions of the software. """ return THIRD_PARTY_INSTALLERS_LATEST_DIR def find_available(self) -> list[str]: - """ - Return a list of paths pointing to the available versions of this software on the system. + """Return a list of paths pointing to the available versions of this software on the system. """ # Get filenames in directory filenames = next(os.walk(self.get_search_dir()), (None, None, []))[2] # [] if no file @@ -60,8 +53,7 @@ def find_available(self) -> list[str]: return file_paths def find_latest(self) -> tuple[str, str]: - """ - Return a tuple of (dependency_file, version) + """Return a tuple of (dependency_file, version) where the dependency_file points to the location describing the latest version of the software and the version is the version of this dependency. """ @@ -82,8 +74,7 @@ def find_latest(self) -> tuple[str, str]: def is_higher(v1, v2): - """ - Returns True if v2 represents a higher version than v1. + """Returns True if v2 represents a higher version than v1. """ segments1 = v1.split(".") segments2 = v2.split(".") diff --git a/installation_and_upgrade/ibex_install_utils/software_dependency/mysql.py b/installation_and_upgrade/ibex_install_utils/software_dependency/mysql.py index de36dd7..d7f4002 100644 --- a/installation_and_upgrade/ibex_install_utils/software_dependency/mysql.py +++ b/installation_and_upgrade/ibex_install_utils/software_dependency/mysql.py @@ -2,10 +2,10 @@ import re import subprocess +from ibex_install_utils.file_utils import file_in_zip, get_version from ibex_install_utils.software_dependency import SoftwareDependency -from ibex_install_utils.version_check import VERSION_REGEX, get_major_minor_patch from ibex_install_utils.tasks.common_paths import APPS_BASE_DIR, INST_SHARE_AREA -from ibex_install_utils.file_utils import get_version, file_in_zip +from ibex_install_utils.version_check import VERSION_REGEX, get_major_minor_patch MYSQL8_INSTALL_DIR = os.path.join(APPS_BASE_DIR, "MySQL") diff --git a/installation_and_upgrade/ibex_install_utils/task.py b/installation_and_upgrade/ibex_install_utils/task.py index f9ce796..c47cc45 100644 --- a/installation_and_upgrade/ibex_install_utils/task.py +++ b/installation_and_upgrade/ibex_install_utils/task.py @@ -1,10 +1,9 @@ -""" -Task infrastructure. +"""Task infrastructure. """ import traceback -from ibex_install_utils.exceptions import UserStop, ErrorInTask +from ibex_install_utils.exceptions import ErrorInTask, UserStop from ibex_install_utils.user_prompt import UserPrompt @@ -34,8 +33,7 @@ def _run_task_to_completion(task_name, prompt, self_decorated_method, func, args def task(task_name, attribute_name="prompt"): - """ - Decorator for tasks to be performed for installs. + """Decorator for tasks to be performed for installs. Confirms a step is to be run (if needed). If there is a problem will ask the user what to do. Wraps the task in print statements so users can see when a task starts and ends. diff --git a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py index 16965cb..d1e44a3 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/__init__.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/__init__.py @@ -4,7 +4,7 @@ from ibex_install_utils.ca_utils import CaWrapper from ibex_install_utils.file_utils import FileUtils -from ibex_install_utils.tasks.common_paths import BACKUP_DIR, BACKUP_DATA_DIR +from ibex_install_utils.tasks.common_paths import BACKUP_DATA_DIR, BACKUP_DIR class BaseTasks: @@ -19,8 +19,8 @@ def __init__( ibex_version, file_utils=FileUtils(), ): - """ - Initializer. + """Initializer. + Args: user_prompt (ibex_install_utils.user_prompt.UserPrompt): a object to allow prompting of the user server_source_dir: directory to install ibex server from @@ -40,18 +40,16 @@ def __init__( @staticmethod def _get_machine_name(): - """ - Returns: - The current machine name + """Returns: + The current machine name """ return socket.gethostname() @staticmethod def _get_instrument_name(): - """ - Returns: - The name of the current instrument + """Returns: + The name of the current instrument """ return BaseTasks._get_machine_name().replace("NDX", "") @@ -65,8 +63,7 @@ def _generate_backup_dir_name(): @staticmethod def _get_backup_dir(): - """ - The backup directory contains the date of backup, if this script is + """The backup directory contains the date of backup, if this script is running over multiple days this will return the date this method was first called. Returns: The backup dir, will create it if needed (both old and dir). diff --git a/installation_and_upgrade/ibex_install_utils/tasks/backup_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/backup_tasks.py index d31e686..8e458e7 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/backup_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/backup_tasks.py @@ -1,24 +1,23 @@ import os import shutil -import shutil -from ibex_install_utils.user_prompt import UserPrompt -from ibex_install_utils.progress_bar import ProgressBar from ibex_install_utils.file_utils import FileUtils +from ibex_install_utils.progress_bar import ProgressBar from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import ( + AUTOSAVE, BACKUP_DATA_DIR, BACKUP_DIR, EPICS_PATH, - PYTHON_PATH, - PYTHON_3_PATH, EPICS_UTILS_PATH, GUI_PATH, - STAGE_DELETED, + PYTHON_3_PATH, + PYTHON_PATH, SETTINGS_DIR, - AUTOSAVE, + STAGE_DELETED, ) +from ibex_install_utils.user_prompt import UserPrompt COMMON_IGNORE_PATTERNS = [ "*.*dmp", @@ -50,8 +49,7 @@ class BackupTasks(BaseTasks): - """ - The tasks dealing with backing up current install, removing current install and moving old backups to the shares. + """The tasks dealing with backing up current install, removing current install and moving old backups to the shares. """ @@ -66,8 +64,7 @@ class BackupTasks(BaseTasks): @task("Backup old directories") def backup_old_directories(self): - """ - Backup old directories. + """Backup old directories. """ if os.path.exists(BACKUP_DATA_DIR): @@ -93,8 +90,7 @@ def backup_old_directories(self): @task("Verify backup") def backup_checker(self): - """ - Verify backup. This function checks if the backup has been sucessful by checking for a + """Verify backup. This function checks if the backup has been sucessful by checking for a VERSION.txt file within the backup folders for EPICS, PYTHON, GUI. """ @@ -113,8 +109,7 @@ def backup_checker(self): @task("Removing old version of IBEX") def remove_old_ibex(self): - """ - Removes older versions of IBEX server, client, genie_python and epics utils. + """Removes older versions of IBEX server, client, genie_python and epics utils. """ for path in ALL_INSTALL_DIRECTORIES: @@ -222,8 +217,7 @@ def copy_function(src, dst): # ? Moving other backups to stage deleted could be a task on its own def _move_old_backups_to_share(self): - """ - Move all but the newest backup to the shares. + """Move all but the newest backup to the shares. """ current_backups = [ diff --git a/installation_and_upgrade/ibex_install_utils/tasks/client_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/client_tasks.py index 7f3cc76..6394162 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/client_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/client_tasks.py @@ -10,15 +10,14 @@ class ClientTasks(BaseTasks): @task("Installing IBEX Client with builtin python") def install_ibex_client(self): - """ - Install the ibex client with builtin python. + """Install the ibex client with builtin python. """ self._install_set_version_of_ibex_client(self._client_source_dir) def _install_set_version_of_ibex_client(self, source_dir): - """ - Install a given version of the Ibex client. + """Install a given version of the Ibex client. + Args: source_dir: source directory for the client """ @@ -28,16 +27,14 @@ def _install_set_version_of_ibex_client(self, source_dir): @task("Starting IBEX gui") def start_ibex_gui(self): - """ - Start the IBEX GUI + """Start the IBEX GUI :return: """ subprocess.Popen([os.path.join(GUI_PATH, "ibex-client.exe")], cwd=GUI_PATH) @task("Client release tests") def perform_client_tests(self): - """ - Test that the client works + """Test that the client works """ self.prompt.prompt_and_raise_if_not_yes( "Check that the version displayed in the client is as expected after the deployment" diff --git a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py index 75fb2dc..37a77fd 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py @@ -1,10 +1,9 @@ +import re import subprocess from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import EPICS_PATH -from genie_python import genie as g -import re class GitTasks(BaseTasks): @@ -12,7 +11,7 @@ class GitTasks(BaseTasks): def show_git_status(self): subprocess.call(f"cd {EPICS_PATH} && git status", shell=True) - @task(f"Swap instrument git branch to release on CONTROL-SVCS") + @task("Swap instrument git branch to release on CONTROL-SVCS") def checkout_to_release_branch(self): version_pattern = r"^\d+\.\d+\.\d+$" if self._server_source_dir.endswith("32"): @@ -61,7 +60,7 @@ def checkout_to_release_branch(self): print("Pushed to the remote") except subprocess.CalledProcessError as e: print(f"Error checking out to new release branch and push: {e}") - print(f"Branch may previously exist either locally or remotely - intervention required") + print("Branch may previously exist either locally or remotely - intervention required") # something for the future in case creting new beranch fails - maybe one exists we want to use? diff --git a/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py index 26a16cb..14f8a57 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py @@ -8,18 +8,18 @@ from ibex_install_utils.admin_runner import AdminCommandBuilder from ibex_install_utils.exceptions import ErrorInRun from ibex_install_utils.run_process import RunProcess +from ibex_install_utils.software_dependency.mysql import MySQL from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import ( APPS_BASE_DIR, + EPICS_PATH, INST_SHARE_AREA, - VAR_DIR, STAGE_DELETED, - EPICS_PATH, + VAR_DIR, ) from ibex_install_utils.user_prompt import UserPrompt from ibex_install_utils.version_check import version_check -from ibex_install_utils.software_dependency.mysql import MySQL try: from subprocess import DETACHED_PROCESS @@ -60,13 +60,11 @@ class MysqlTasks(BaseTasks): - """ - Tasks relating to installing or maintaining an installation of MySQL on a machine. + """Tasks relating to installing or maintaining an installation of MySQL on a machine. """ def _get_mysql_dir(self): - """ - Returns the mysql 8 default install directory if it exists, else 5.7. + """Returns the mysql 8 default install directory if it exists, else 5.7. """ if os.path.exists(MYSQL8_INSTALL_DIR): @@ -88,8 +86,7 @@ def _get_mysql_backup_dir(self): @task("Truncate database") def truncate_database(self): - """ - Truncate the message log, sample and alarms tables + """Truncate the message log, sample and alarms tables """ try: mysql_bin_dir = self._get_mysql_dir() @@ -111,8 +108,7 @@ def truncate_database(self): ) def _configure_mysql(self): - """ - Copy mysql settings and run the MySQL configuration script + """Copy mysql settings and run the MySQL configuration script """ my_ini_file = os.path.join(EPICS_PATH, "systemsetup", "my.ini") try: @@ -282,8 +278,7 @@ def _setup_mysql8_service(self): admin_commands.run_all() def _install_latest_mysql8(self, clean_install): - """ - Install the latest mysql. If this is a clean install remove old data directories first and create a new + """Install the latest mysql. If this is a clean install remove old data directories first and create a new database Args: clean_install: True to destroy and recreate data directories @@ -312,8 +307,7 @@ def _install_latest_mysql8(self, clean_install): @task("Install latest MySQL for VHD deployment") def install_mysql_for_vhd(self): - """ - Installs MySQL for the VHD creation build. + """Installs MySQL for the VHD creation build. Ensure we start from a clean slate. We are creating VHDs so we can assume that no files should exist in C:\instrument\apps\mysql or c:\instrument\var\mysql and delete them if they do exist. This facilitates @@ -348,8 +342,7 @@ def _install_vcruntime140(self): @version_check(MySQL()) @task("Install latest MySQL") def install_mysql(self, force=False): - """ - Install mysql and the ibex database schemas + """Install mysql and the ibex database schemas Args: force: True delete old data and update """ @@ -391,8 +384,7 @@ def install_mysql(self, force=False): @task("Configure MySQL for VHD post install") def configure_mysql_for_vhd_post_install(self): - """ - configure mysql after vhd is deployed to an instrukent/mdt build + """Configure mysql after vhd is deployed to an instrukent/mdt build """ self._setup_mysql8_service() @@ -400,8 +392,7 @@ def configure_mysql_for_vhd_post_install(self): @task("Backup database") def backup_database(self): - """ - Backup the database + """Backup the database """ mysql_bin_dir = self._get_mysql_dir() result_file = os.path.join( @@ -448,8 +439,7 @@ def backup_database(self): ) def _backup_data(self): - """ - Backup the data for transfer. This dumps just the data not the schema. + """Backup the data for transfer. This dumps just the data not the schema. """ result_file = os.path.join( self._get_mysql_backup_dir(), @@ -488,8 +478,7 @@ def _backup_data(self): ) def _reload_backup_data(self): - """ - Reload backup the data + """Reload backup the data """ result_file = os.path.join( self._get_mysql_backup_dir(), diff --git a/installation_and_upgrade/ibex_install_utils/tasks/python_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/python_tasks.py index 368e160..1642058 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/python_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/python_tasks.py @@ -7,22 +7,19 @@ class PythonTasks(BaseTasks): - """ - Tasks relating to installing or maintaining an installation of genie_python. + """Tasks relating to installing or maintaining an installation of genie_python. """ @task("Installing Genie Python 3") def install_genie_python3(self): - """ - Install ibex server. + """Install ibex server. """ self._file_utils.mkdir_recursive(APPS_BASE_DIR) RunProcess(self._genie_python_3_source_dir, "genie_python_install.bat").run() @task("Update script generator script definitions") def update_script_definitions(self): - """ - Update (or at least ask the user to update) the script definitions used by the script generator. + """Update (or at least ask the user to update) the script definitions used by the script generator. """ if os.path.exists("C:\\ScriptGeneratorConfigs") or os.path.exists("C:\\ScriptDefinitions"): print( @@ -33,8 +30,7 @@ def update_script_definitions(self): @task("Remove instrument scripts githooks") def remove_instrument_script_githooks(self): - """ - Remove the githooks in the instrument scripts dierectory + """Remove the githooks in the instrument scripts dierectory """ hook_path = os.path.join(INSTRUMENT_BASE_DIR, "scripts", ".git", "hooks", "commit-msg") if os.path.exists(hook_path): diff --git a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py index 8e62882..a2cfd7a 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/server_tasks.py @@ -4,9 +4,7 @@ import pprint import shutil import subprocess -import sys import tempfile -import time import lxml.etree from ibex_install_utils.user_prompt import UserPrompt @@ -71,15 +69,13 @@ class ServerTasks(BaseTasks): - """ - Tasks relating to installing or maintaining an IBEX server and it's associated configuration files. + """Tasks relating to installing or maintaining an IBEX server and it's associated configuration files. """ @staticmethod def _get_config_path(): - """ - Returns: - The path to the instrument's configurations directory + """Returns: + The path to the instrument's configurations directory """ return os.path.join( @@ -88,8 +84,7 @@ def _get_config_path(): @task("Removing old settings file") def remove_settings(self): - """ - remove old settings + """Remove old settings Returns: """ @@ -97,12 +92,10 @@ def remove_settings(self): @task("Install settings") def install_settings(self): - """ - Install new settings from the current folder + """Install new settings from the current folder Returns: """ - self._file_utils.mkdir_recursive(SETTINGS_CONFIG_PATH) settings_path = os.path.join(SETTINGS_CONFIG_PATH, ServerTasks._get_machine_name()) @@ -127,8 +120,8 @@ def install_settings(self): @task("Installing IBEX Server") def install_ibex_server(self, use_old_galil=None): - """ - Install ibex server. + """Install ibex server. + Args: use_old_galil(bool): whether to restore old galil driver version """ @@ -140,8 +133,7 @@ def install_ibex_server(self, use_old_galil=None): @task("Set up configuration repository") def setup_config_repository(self): - """ - Creates the configuration repository, and swaps or creates a branch for the instrument. + """Creates the configuration repository, and swaps or creates a branch for the instrument. """ inst_name = BaseTasks._get_machine_name() @@ -256,8 +248,7 @@ def setup_config_repository(self): @task("Upgrading instrument configuration") def upgrade_instrument_configuration(self): - """ - Update the configuration on the instrument using its upgrade config script. + """Update the configuration on the instrument using its upgrade config script. """ manual_prompt = ( "Merge the master configurations branch into the instrument configuration. " @@ -301,8 +292,7 @@ def upgrade_instrument_configuration(self): @task("Install shared instrument scripts repository") def install_shared_scripts_repository(self): - """ - Install shared instrument scripts repository containing + """Install shared instrument scripts repository containing """ if os.path.isdir(INST_SCRIPTS_PATH): if ( @@ -325,8 +315,7 @@ def install_shared_scripts_repository(self): @task("Set up shared instrument scripts library") def update_shared_scripts_repository(self): - """ - Update the shared instrument scripts repository containing + """Update the shared instrument scripts repository containing """ try: repo = git.Repo(INST_SCRIPTS_PATH) @@ -339,8 +328,7 @@ def update_shared_scripts_repository(self): @task("Set up calibrations repository") def setup_calibrations_repository(self): - """ - Set up the calibration repository + """Set up the calibration repository """ if os.path.isdir(CALIBRATION_PATH): if ( @@ -364,8 +352,7 @@ def setup_calibrations_repository(self): @task("Updating calibrations repository") def update_calibrations_repository(self): - """ - Update the calibration repository + """Update the calibration repository """ try: repo = git.Repo(CALIBRATION_PATH) @@ -378,8 +365,7 @@ def update_calibrations_repository(self): @task("Server release tests") def perform_server_tests(self): - """ - Test that the server works + """Test that the server works """ server_release_tests_url = ( "https://github.com/ISISComputingGroup/ibex_developers_manual/wiki/" @@ -412,8 +398,7 @@ def perform_server_tests(self): @task("Install wiring tables") def install_wiring_tables(self): - """ - Prompt user to install wiring tables in the appropriate folder. + """Prompt user to install wiring tables in the appropriate folder. """ tables_dir = os.path.join( SETTINGS_CONFIG_PATH, @@ -425,8 +410,7 @@ def install_wiring_tables(self): @task("Configure motion setpoints") def configure_motion(self): - """ - Prompt user to configure Galils + """Prompt user to configure Galils """ url = ( "https://github.com/ISISComputingGroup/ibex_developers_manual/wiki/" @@ -446,8 +430,7 @@ def configure_motion(self): @contextmanager def timestamped_pv_backups_file(self, name, directory, extension="txt"): - """ - Context manager to create a timestamped file in the pv backups directory + """Context manager to create a timestamped file in the pv backups directory Args: name (str): path to the file @@ -470,11 +453,9 @@ def timestamped_pv_backups_file(self, name, directory, extension="txt"): @task("Backup motors, blocks and blockserver to csv files") def save_motor_blocks_blockserver_to_file(self): - """ - Saves the motor, blocks and blockserver to csv file. + """Saves the motor, blocks and blockserver to csv file. """ - print("Backing up: motor params pvs") self.save_motor_parameters_to_file() print("Finished backing up: motor params pvs") @@ -488,10 +469,8 @@ def save_motor_blocks_blockserver_to_file(self): print("Finished backing up: blockserver config pvs") def save_motor_parameters_to_file(self): + """Saves the motor parameters to csv file. """ - Saves the motor parameters to csv file. - """ - with self.timestamped_pv_backups_file( name="motors", directory="motors", extension="csv" ) as f: @@ -499,10 +478,8 @@ def save_motor_parameters_to_file(self): get_params_and_save_to_file(f) def save_blocks_to_file(self): + """Saves block parameters in a file. """ - Saves block parameters in a file. - """ - blocks = self._ca.get_blocks() if blocks is None: print("Blockserver unavailable - not archiving.") @@ -544,8 +521,7 @@ def block_caget(self, block, counter, data): data[counter] = f"{self._ca.cget(block)}\r\n" def save_blockserver_pv_to_file(self): - """ - Saves the blockserver PV to a file. + """Saves the blockserver PV to a file. """ pvs_to_save = [ ("all_component_details", "CS:BLOCKSERVER:ALL_COMPONENT_DETAILS"), @@ -588,8 +564,8 @@ def _pretty_print(data): @task("Update the ICP") def update_icp(self, icp_in_labview_modules, register_icp=True): - """ - Updates the IPC to the latest version. + """Updates the IPC to the latest version. + Args: icp_in_labview_modules (bool): true if the ICP is in labview modules register_icp (bool): whether to re-register the ISISICP program (requires admin rights; interactive only) @@ -711,8 +687,7 @@ def run_config_checker(self): RunProcess(tmpdir, python, prog_args=args).run() def select_galil_driver(self): - """ - Select galil driver to use. Return True if old driver in operation or should be used + """Select galil driver to use. Return True if old driver in operation or should be used """ # GALIL_OLD.txt file gets copied to the tmp dir by instrument_deploy.bat tmpdir = tempfile.gettempdir() @@ -746,8 +721,7 @@ def select_galil_driver(self): return not self.prompt.confirm_step("Use new Galil driver") def _swap_galil_driver(self, use_old): - """ - Swap galil back to old if needed + """Swap galil back to old if needed Args: use_old(bool): whether to restore old driver version """ diff --git a/installation_and_upgrade/ibex_install_utils/tasks/system_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/system_tasks.py index 107b53f..ec02d7e 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/system_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/system_tasks.py @@ -1,24 +1,23 @@ +import glob import os import shutil -import psutil -import glob import subprocess -from win32com.client import Dispatch +import psutil from ibex_install_utils.admin_runner import AdminCommandBuilder from ibex_install_utils.exceptions import UserStop from ibex_install_utils.kafka_utils import add_required_topics from ibex_install_utils.run_process import RunProcess +from ibex_install_utils.software_dependency.git import Git +from ibex_install_utils.software_dependency.java import Java from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import ( APPS_BASE_DIR, EPICS_PATH, - THIRD_PARTY_INSTALLERS_DIR, ) from ibex_install_utils.version_check import version_check -from ibex_install_utils.software_dependency.java import Java -from ibex_install_utils.software_dependency.git import Git +from win32com.client import Dispatch GIGABYTE = 1024**3 @@ -53,14 +52,12 @@ class SystemTasks(BaseTasks): - """ - Tasks relating to the system e.g. installed software other than core IBEX components, windows, firewalls, etc. + """Tasks relating to the system e.g. installed software other than core IBEX components, windows, firewalls, etc. """ @task("Record running LabVIEW VIs or any relevant looking other programs") def record_running_vis(self): - """ - Get user to record running vis + """Get user to record running vis """ self.prompt.prompt_and_raise_if_not_yes( "Write down any LabVIEW VIs/relevant looking programs which are running for use later?" @@ -68,8 +65,7 @@ def record_running_vis(self): @task("Upgrading Notepad++. Please follow system dialogs") def upgrade_notepad_pp(self): - """ - Install (start installation of) notepad ++ + """Install (start installation of) notepad ++ Returns: """ @@ -83,8 +79,7 @@ def upgrade_notepad_pp(self): @task("Removing training folder on desktop ...") def clean_up_desktop_ibex_training_folder(self): - """ - Remove training folder from the desktop + """Remove training folder from the desktop Returns: """ @@ -92,8 +87,7 @@ def clean_up_desktop_ibex_training_folder(self): @task("Remove SECI shortcuts") def remove_seci_shortcuts(self): - """ - Remove (or at least ask the user to remove) all Seci shortcuts + """Remove (or at least ask the user to remove) all Seci shortcuts """ for path in SECI_AUTOSTART_LOCATIONS: if os.path.exists(path): @@ -107,8 +101,7 @@ def remove_seci_shortcuts(self): @task("Remove Treesize shortcuts") def remove_treesize_shortcuts(self): - """ - Remove (or at least ask the user to remove) all Treesize shortcuts. + """Remove (or at least ask the user to remove) all Treesize shortcuts. For justification see https://github.com/ISISComputingGroup/IBEX/issues/4214 """ @@ -120,8 +113,7 @@ def remove_treesize_shortcuts(self): @task("Remove SECI 1 Path") def remove_seci_one(self): - """ - Removes SECI 1 + """Removes SECI 1 """ if os.path.exists(SECI_ONE_PATH): try: @@ -136,8 +128,7 @@ def remove_seci_one(self): @version_check(Java()) @task("Install java") def check_java_installation(self): - """ - Checks Java installation + """Checks Java installation """ installer, _ = Java().find_latest() @@ -159,8 +150,7 @@ def check_java_installation(self): @task("Configure COM ports") def configure_com_ports(self): - """ - Configure the COM ports + """Configure the COM ports """ self.prompt.prompt_and_raise_if_not_yes( "Using NPort Administrator (available under /Kits$/CompGroup/Utilities/), check that the COM ports " @@ -172,8 +162,7 @@ def configure_com_ports(self): @task("Reapply Hotfixes") def reapply_hotfixes(self): - """ - Reapply any hotfixes to the build. + """Reapply any hotfixes to the build. """ self.prompt.prompt_and_raise_if_not_yes( "Have you applied any hotfixes listed that are not fixed by the release, as on the instrument " @@ -182,8 +171,7 @@ def reapply_hotfixes(self): @task("Restart VIs") def restart_vis(self): - """ - Restart Vis which were running on upgrade start. + """Restart Vis which were running on upgrade start. """ self.prompt.prompt_and_raise_if_not_yes( "Please restart any VIs that were running at the start of the upgrade" @@ -191,8 +179,7 @@ def restart_vis(self): @task("Update release notes") def update_release_notes(self): - """ - Update the release notes. + """Update the release notes. """ self.prompt.prompt_and_raise_if_not_yes( "Have you updated the instrument release notes at https://github.com/ISISComputingGroup/IBEX/wiki?" @@ -200,8 +187,7 @@ def update_release_notes(self): @task("Update Instrument List") def update_instlist(self): - """ - Prompt user to add instrument to the list of known IBEX instruments + """Prompt user to add instrument to the list of known IBEX instruments """ self.prompt.prompt_and_raise_if_not_yes( "Add the host name of the instrument to the list saved in the CS:INSTLIST PV" @@ -209,15 +195,13 @@ def update_instlist(self): @task("Update kafka topics") def update_kafka_topics(self): - """ - Adds the required kafka topics to the cluster. + """Adds the required kafka topics to the cluster. """ add_required_topics("livedata.isis.cclrc.ac.uk:9092", self._get_instrument_name()) @task("Add Nagios checks") def add_nagios_checks(self): - """ - Prompt user to add nagios checks. + """Prompt user to add nagios checks. """ # For future reference, genie_python can send emails! self.prompt.prompt_and_raise_if_not_yes( @@ -227,8 +211,7 @@ def add_nagios_checks(self): @task("Inform instrument scientists") def inform_instrument_scientists(self): - """ - Inform instrument scientists that the machine has been upgraded. + """Inform instrument scientists that the machine has been upgraded. """ # For future reference, genie_python can send emails! ibex_version = ( @@ -247,8 +230,7 @@ def inform_instrument_scientists(self): @task("Apply changes in release notes") def apply_changes_noted_in_release_notes(self): - """ - Apply any changes noted in the release notes. + """Apply any changes noted in the release notes. """ # For future reference, genie_python can send emails! self.prompt.prompt_and_raise_if_not_yes( @@ -256,16 +238,14 @@ def apply_changes_noted_in_release_notes(self): ) def check_resources(self): - """ - Check the machine's resources meet minimum requirements. + """Check the machine's resources meet minimum requirements. """ self.check_virtual_memory() self._check_disk_usage() @task("Check virtual machine memory") def check_virtual_memory(self): - """ - Checks the machine's virtual memory meet minimum requirements. + """Checks the machine's virtual memory meet minimum requirements. """ ram = psutil.virtual_memory().total @@ -299,8 +279,7 @@ def check_virtual_memory(self): @task("Check there is {:.1e}B free disk space".format(FREE_DISK_MIN)) def _check_disk_usage(self): - """ - Checks the machine's free disk space meets minimum requirements. + """Checks the machine's free disk space meets minimum requirements. """ disk_space = psutil.disk_usage("/") @@ -313,15 +292,13 @@ def _check_disk_usage(self): @task("Put IBEX autostart script into startup for current user") def put_autostart_script_in_startup_area(self): - """ - Checks the startup location for all users for the autostart script and removes any instances. + """Checks the startup location for all users for the autostart script and removes any instances. Checks the startup location of the current user and removes the autostart script if it was copied. Creates a shortcut of the ibex server autostart script into the current user startup folder so that the IBEX server starts automatically on startup. """ - AUTOSTART_SCRIPT_NAME = "ibex_system_boot" # Check all users startup folder. @@ -352,8 +329,7 @@ def put_autostart_script_in_startup_area(self): @task("Restrict Internet Explorer") def restrict_ie(self): - """ - Restrict access of external websites to address a security vulnerability in Internet Explorer. + """Restrict access of external websites to address a security vulnerability in Internet Explorer. """ self.prompt.prompt_and_raise_if_not_yes( "Configure Internet Explorer to restrict access to the web except for select whitelisted sites:\n" @@ -366,8 +342,7 @@ def restrict_ie(self): @version_check(Git()) @task("Update Git") def install_or_upgrade_git(self): - """ - Install the latest git version + """Install the latest git version """ git_path = shutil.which("git") if os.path.exists(git_path): @@ -404,14 +379,12 @@ def install_or_upgrade_git(self): ) def confirm(self, message): - """ - Ask user to confirm correct script was chosen. + """Ask user to confirm correct script was chosen. """ self.prompt.prompt_and_raise_if_not_yes(message, default="Y") def _read_file(self, path, error_text): - """ - print a file contents to screen + """Print a file contents to screen """ data = "" try: @@ -422,8 +395,7 @@ def _read_file(self, path, error_text): return data def user_confirm_upgrade_type_on_machine(self, machine_type): - """ - Print information about the current upgrade and prompt the user + """Print information about the current upgrade and prompt the user Returns: Raises UserStop: when the user doesn't want to continue diff --git a/installation_and_upgrade/ibex_install_utils/tasks/vhd_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/vhd_tasks.py index 7d8a79a..77fa1ac 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/vhd_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/vhd_tasks.py @@ -8,10 +8,10 @@ from ibex_install_utils.task import task from ibex_install_utils.tasks import BaseTasks from ibex_install_utils.tasks.common_paths import ( - INSTRUMENT_BASE_DIR, - INST_SHARE_AREA, - EPICS_PATH, APPS_BASE_DIR, + EPICS_PATH, + INST_SHARE_AREA, + INSTRUMENT_BASE_DIR, SETTINGS_DIR, VAR_DIR, ) @@ -60,8 +60,7 @@ def __init__(self, name, source_filename, dest_filename, mount_point): class VHDTasks(BaseTasks): - """ - Tasks relating to creating or using VHDs. + """Tasks relating to creating or using VHDs. """ @task("Copy VHDs to local area") @@ -152,7 +151,6 @@ def mount_vhds(self): "powershell", r'-command "Hyper-V\Mount-VHD -path {vhd_file} -Passthru | Get-Disk | Get-Partition | Get-Volume | foreach {{ $_.DriveLetter }} | out-file -filepath {driveletter_file} -Encoding ASCII -NoNewline"'.format( vhd_file=os.path.join(LOCAL_VHD_DIR, vhd.source_filename), - name=vhd.name, driveletter_file=driveletter_file, ), ) @@ -179,7 +177,7 @@ def mount_vhds(self): admin_commands.add_command( "powershell", r'-command "&cmd /c mklink /J /D {mount_point} @(cat {driveletter_file})"'.format( - mount_point=vhd.mount_point, name=vhd.name, driveletter_file=driveletter_file + mount_point=vhd.mount_point, driveletter_file=driveletter_file ), ) @@ -259,8 +257,7 @@ def deploy_vhds(self): @task("Initialize var dir") def initialize_var_dir(self): - """ - Creates the folder structure for the C:\instrument\var directory. + """Creates the folder structure for the C:\instrument\var directory. """ # config_env creates all the necessary directories for us RunProcess(working_dir=EPICS_PATH, executable_file="config_env.bat").run() diff --git a/installation_and_upgrade/ibex_install_utils/tests/test_backup_tasks.py b/installation_and_upgrade/ibex_install_utils/tests/test_backup_tasks.py index 5a9626c..f6add53 100644 --- a/installation_and_upgrade/ibex_install_utils/tests/test_backup_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tests/test_backup_tasks.py @@ -1,9 +1,10 @@ import io import os -from unittest.mock import patch, Mock -from ibex_install_utils.tasks.backup_tasks import BackupTasks +from unittest.mock import Mock, patch + from ibex_install_utils.progress_bar import ProgressBar from ibex_install_utils.tasks import BaseTasks +from ibex_install_utils.tasks.backup_tasks import BackupTasks from ibex_install_utils.user_prompt import UserPrompt diff --git a/installation_and_upgrade/ibex_install_utils/tests/test_dummy.py b/installation_and_upgrade/ibex_install_utils/tests/test_dummy.py index ca84e22..603d733 100644 --- a/installation_and_upgrade/ibex_install_utils/tests/test_dummy.py +++ b/installation_and_upgrade/ibex_install_utils/tests/test_dummy.py @@ -3,7 +3,7 @@ # to run the file as a temp fix import io -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch class TestDummyTest: diff --git a/installation_and_upgrade/ibex_install_utils/tests/test_test.py b/installation_and_upgrade/ibex_install_utils/tests/test_test.py index a216fb4..ceb369e 100644 --- a/installation_and_upgrade/ibex_install_utils/tests/test_test.py +++ b/installation_and_upgrade/ibex_install_utils/tests/test_test.py @@ -4,6 +4,7 @@ import io from unittest.mock import patch + from ibex_install_utils.progress_bar import ProgressBar diff --git a/installation_and_upgrade/ibex_install_utils/tests/test_version_check.py b/installation_and_upgrade/ibex_install_utils/tests/test_version_check.py index ab91e53..d4c5c01 100644 --- a/installation_and_upgrade/ibex_install_utils/tests/test_version_check.py +++ b/installation_and_upgrade/ibex_install_utils/tests/test_version_check.py @@ -1,16 +1,15 @@ -import pytest import os from unittest.mock import Mock -from ibex_install_utils.version_check import * -from ibex_install_utils.software_dependency.java import Java -from ibex_install_utils.software_dependency.git import Git -from ibex_install_utils.software_dependency.mysql import MySQL + +import pytest from ibex_install_utils.software_dependency import is_higher +from ibex_install_utils.software_dependency.git import Git +from ibex_install_utils.software_dependency.java import Java +from ibex_install_utils.version_check import * def get_version_from_name(name): - """ - Return version number read out from the filename + """Return version number read out from the filename """ basename = os.path.basename(name) version = re.search(r"([0-9]+\.[0-9]+(\.[0-9]+)+)", basename).group(1) diff --git a/installation_and_upgrade/ibex_install_utils/user_prompt.py b/installation_and_upgrade/ibex_install_utils/user_prompt.py index af17eda..25fcf43 100644 --- a/installation_and_upgrade/ibex_install_utils/user_prompt.py +++ b/installation_and_upgrade/ibex_install_utils/user_prompt.py @@ -1,22 +1,21 @@ -""" -Classes to interact with the user +"""Classes to interact with the user """ -from .exceptions import UserStop import six.moves +from .exceptions import UserStop + class UserPrompt: - """ - A user prompt object to ask the user questions. + """A user prompt object to ask the user questions. """ # Sentinel value for allowing any answer to a prompt ANY = object() def __init__(self, automatic, confirm_steps): - """ - Initializer. + """Initializer. + Args: automatic (bool): should the prompt ignore the user and use default responses confirm_steps (bool): should the user confirm a step before running it; setting automatic overrides this @@ -27,8 +26,7 @@ def __init__(self, automatic, confirm_steps): def prompt( self, prompt_text, possibles, default, case_sensitive=False, show_automatic_answer=True ): - """ - Prompt the user for an answer and check that answer. If in auto mode just answer the default + """Prompt the user for an answer and check that answer. If in auto mode just answer the default Args: prompt_text: Text to prompt possibles: allowed answers @@ -59,8 +57,7 @@ def _get_user_answer(self, prompt_text, possibles, case_sensitive=False): print(f"Answer is not allowed can be one of ({possibles})") def confirm_step(self, step_text): - """ - Confirm that a step should be done if in confirm steps mode + """Confirm that a step should be done if in confirm steps mode Args: step_text: the text for the step @@ -72,8 +69,7 @@ def confirm_step(self, step_text): return self._get_user_answer(f"Do step '{step_text}'? : ", ("Y", "N")) == "Y" def prompt_and_raise_if_not_yes(self, message, default="N"): - """ - Prompt the user and raise and exception if they do not answer yes + """Prompt the user and raise and exception if they do not answer yes Default to Y in quiet mode Args: message: the message to prompt the user with diff --git a/installation_and_upgrade/ibex_install_utils/version_check.py b/installation_and_upgrade/ibex_install_utils/version_check.py index 86d5df2..36d1be3 100644 --- a/installation_and_upgrade/ibex_install_utils/version_check.py +++ b/installation_and_upgrade/ibex_install_utils/version_check.py @@ -1,8 +1,8 @@ -""" -Third party program version checker infrastructure. +"""Third party program version checker infrastructure. """ import re + from ibex_install_utils.software_dependency import SoftwareDependency VERSION_REGEX = r"\s([0-9]+\.[0-9]+(\.[0-9]+)+)" @@ -13,8 +13,7 @@ def version_check(software: SoftwareDependency): - """ - Decorator for tasks that check program version numbers. + """Decorator for tasks that check program version numbers. Identifies the current installed version of program by running the following code in the terminal. 'program --version' diff --git a/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/init_inst_name.py b/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/init_inst_name.py index 197ec36..953f409 100644 --- a/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/init_inst_name.py +++ b/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/init_inst_name.py @@ -1,4 +1,3 @@ -import inst def init(inst): diff --git a/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/inst/__init__.py b/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/inst/__init__.py index 551ecbf..c6d91df 100644 --- a/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/inst/__init__.py +++ b/installation_and_upgrade/resources/Settings/config/NDXOTHER/Python/inst/__init__.py @@ -1,7 +1,7 @@ __all__ = [] -import pkgutil import inspect +import pkgutil for loader, name, is_pkg in pkgutil.walk_packages(__path__): module = loader.find_module(name).load_module(name) diff --git a/installation_and_upgrade/vhd_scheduled_task.py b/installation_and_upgrade/vhd_scheduled_task.py index 61d3169..c0325ee 100644 --- a/installation_and_upgrade/vhd_scheduled_task.py +++ b/installation_and_upgrade/vhd_scheduled_task.py @@ -1,15 +1,13 @@ -""" -Script to install IBEX to various machines +"""Script to install IBEX to various machines """ import sys import traceback +from ibex_install_utils.exceptions import ErrorInTask, UserStop from ibex_install_utils.install_tasks import UpgradeInstrument -from ibex_install_utils.exceptions import UserStop, ErrorInTask from ibex_install_utils.user_prompt import UserPrompt - if __name__ == "__main__": try: prompt = UserPrompt(automatic=True, confirm_steps=False) @@ -19,7 +17,7 @@ except UserStop: print("User stopped upgrade") sys.exit(2) - except ErrorInTask as error_in_run_ex: + except ErrorInTask: traceback.print_exc() sys.exit(1) diff --git a/ioc_copier/ioc_copier.py b/ioc_copier/ioc_copier.py index f58a096..c8216ff 100644 --- a/ioc_copier/ioc_copier.py +++ b/ioc_copier/ioc_copier.py @@ -1,11 +1,10 @@ -""" -Script to handle duplicating IOCs +"""Script to handle duplicating IOCs """ -from shutil import copytree, ignore_patterns -import sys import os import re +import sys +from shutil import copytree, ignore_patterns global START_COPY global current_copy @@ -15,8 +14,7 @@ def rename_files(root_folder, rename, ioc): - """ - Function to handle renaming of files. + """Function to handle renaming of files. Parameters: root_folder - The root folder path for use in os.path.join. rename - the path of the file or folder to rename. @@ -50,13 +48,13 @@ def rename_files(root_folder, rename, ioc): def replace_text(text_lines, ioc, skip=None): - """ - Function to handle replacing of text within files. + """Function to handle replacing of text within files. Parameters: text_lines - the text from the file to process. start - the value to look for to remove. current - the value to replace start with. - return: + + Return: The new text to be placed in the file. """ if skip is None: @@ -68,11 +66,11 @@ def replace_text(text_lines, ioc, skip=None): def replace_line(ioc, line): - """ - Function to replace a single line in a file. + """Function to replace a single line in a file. param ioc: The name of the ioc. param line: The line of text to replace. - return: + + Return: Tne new line of text. """ global asub_record @@ -101,8 +99,7 @@ def replace_line(ioc, line): def help_check(): - """ - Function to handle printing help. + """Function to handle printing help. """ if "-h" in sys.argv: print("First Argument: ") @@ -128,8 +125,8 @@ def help_check(): def handle_arguments(): - """ - Function to handle arguments of ioc_copier.py. + """Function to handle arguments of ioc_copier.py. + Returns: ioc - The name of the ioc to run, folders to copy will be of the form, -IOC-01App and ioc-IOC-01. @@ -164,12 +161,12 @@ def handle_arguments(): def copy_folder(file_format, ioc_name): - """ - Function to handle copying folder before replacing text and names. + """Function to handle copying folder before replacing text and names. Parameters: file_format - The format to use for the folder name, either ending in app or starting with ioc. in the form of an fstring. ioc_name - name of the ioc. + Returns: The path of the new folder. """ @@ -190,8 +187,7 @@ def copy_folder(file_format, ioc_name): def generate_config(ioc): - """ - Generate the config if copying ioc 01 as it should just reference ioc 01s config rather than duplicating it. + """Generate the config if copying ioc 01 as it should just reference ioc 01s config rather than duplicating it. :param ioc: the ioc name :return: the text lines of the config. """ @@ -206,8 +202,7 @@ def generate_config(ioc): def remove_db_plus(text): - """ - delete DB += lines from a makefile + """Delete DB += lines from a makefile :param text: the line to check whether to comment :return: the line to delete if it is a DB+= line """ @@ -216,8 +211,7 @@ def remove_db_plus(text): def get_file_text(file, ioc, root): - """ - function to get the text to write to a file. + """Function to get the text to write to a file. :param file: The file to get the initial text from. :param ioc: The Ioc name. :param root: The root folder. @@ -246,8 +240,7 @@ def get_file_text(file, ioc, root): def write_file_text(file, root, text): - """ - function to write to a file. + """Function to write to a file. :param file: The file to write to. :param root: The root folder. :param text: The text to write to the file. @@ -259,8 +252,7 @@ def write_file_text(file, root, text): def file_walk(files, ioc, root): - """ - Function to walk through each file retrieved by os.walk and call necessary functions. + """Function to walk through each file retrieved by os.walk and call necessary functions. :param files: The list of files to walk through. :param ioc: The ioc name. :param root: The root folder. @@ -274,8 +266,7 @@ def file_walk(files, ioc, root): def folder_walk(ioc, root, sub_folder): - """ - Function to walk through folders and rename them. + """Function to walk through folders and rename them. :param ioc: The ioc name. :param root: :param sub_folder: @@ -286,8 +277,7 @@ def folder_walk(ioc, root, sub_folder): def copy_loop(initial_copy, max_copy, file_format, ioc): - """ - Main loop to handle copy and renaming of files + """Main loop to handle copy and renaming of files Parameters: start_copy - The folder to copy (avoid IOC1 as other IOCs usually reference this rather than copying it directly). @@ -313,8 +303,7 @@ def copy_loop(initial_copy, max_copy, file_format, ioc): def add_zero_padding(copy): - """ - Function to add zero padding to the copy number if nessecary. + """Function to add zero padding to the copy number if nessecary. :param copy: The copy number to add zero padding to. :return: The copy, with zero padding added if it has less than 2 digits. @@ -323,8 +312,7 @@ def add_zero_padding(copy): def check_valid_ioc_to_copy(ioc): - """ - Check that duplicating this IOC is valid + """Check that duplicating this IOC is valid :param ioc: The ioc name. """ if not os.path.exists(os.path.join("iocBoot", f"ioc{ioc}-IOC-01", "st-common.cmd")): diff --git a/ioc_copier/test_ioc_copier.py b/ioc_copier/test_ioc_copier.py index 02e520b..83264a1 100644 --- a/ioc_copier/test_ioc_copier.py +++ b/ioc_copier/test_ioc_copier.py @@ -1,9 +1,11 @@ -import ioc_copier -import unittest import os -from parameterized import parameterized +import unittest from unittest import mock +from parameterized import parameterized + +import ioc_copier + class TestIocCopier(unittest.TestCase): @mock.patch("builtins.print") diff --git a/release_scripts/branches.py b/release_scripts/branches.py index 23c9818..8052494 100644 --- a/release_scripts/branches.py +++ b/release_scripts/branches.py @@ -1,13 +1,13 @@ import argparse -import git import logging import os -import sys import re +import sys import xml.etree.ElementTree as ET from typing import Any, List -from git.repo import Repo +import git +from git.repo import Repo EPICS_REPO_URL = "https://github.com/ISISComputingGroup/EPICS.git" IBEX_REPO_URL = "https://github.com/ISISComputingGroup/ibex_gui.git" @@ -111,8 +111,7 @@ def __init__(self, repo: Repo = None): self.repo = repo def create(self, url: str, dir: str, branch_name: str, submodules: bool = False): - """ - Initializes a repository by using 'git clone'. + """Initializes a repository by using 'git clone'. Creates a release branch for the main repository and, optionally, all submodules. Uses an environment variable 'REMOTE' set in Jenkins to check if submodules should be updated with '--remote'. @@ -127,7 +126,6 @@ def create(self, url: str, dir: str, branch_name: str, submodules: bool = False) branch_name: The name of the release branch. submodules: If submodules should be updated and have release branches created. """ - self.branch_name = branch_name logging.info(f"Cloning '{url}' into '{dir}'.") @@ -189,12 +187,11 @@ def create(self, url: str, dir: str, branch_name: str, submodules: bool = False) submodule.module().create_head(branch_name).checkout() def commit(self, items: List[Any], msg: str): - """ - Commits a list of items. + """Commits a list of items. - Parameters: - items: The list of items to stage. - msg: The commit message. + Parameters: + items: The list of items to stage. + msg: The commit message. """ logging.info(f"Committing '{msg}' to '{self.repo.remote().url}'.") @@ -202,11 +199,10 @@ def commit(self, items: List[Any], msg: str): self.repo.index.commit(msg) def push(self, submodules: bool = False): - """ - Pushes changes to remote. + """Pushes changes to remote. - Parameters: - submodules: If changes to submodules should be pushed. + Parameters: + submodules: If changes to submodules should be pushed. """ logging.info(f"Pushing to repo '{self.repo.remote().url}'.") diff --git a/repo_tool/repository_manipulator/RepositoryManipulator.py b/repo_tool/repository_manipulator/RepositoryManipulator.py index 460f68a..8d4a8f2 100644 --- a/repo_tool/repository_manipulator/RepositoryManipulator.py +++ b/repo_tool/repository_manipulator/RepositoryManipulator.py @@ -1,5 +1,4 @@ -""" -This file is part of the ISIS IBEX application. +"""This file is part of the ISIS IBEX application. Copyright (C) 2012-2016 Science & Technology Facilities Council. All rights reserved. @@ -13,8 +12,9 @@ import re from datetime import datetime, timedelta + import pytz as pytz -from github3 import login, GitHubError +from github3 import GitHubError, login # template for the sprint milestones title SPRINT_MILESTONE_REGEX = "SPRINT_(\d+)_(\d+)_(\d+)" @@ -22,8 +22,7 @@ class UserError(Exception): - """ - Exception with a message that can be shown to the user + """Exception with a message that can be shown to the user """ def __init__(self, detail): @@ -35,16 +34,14 @@ def __str__(self): class RepositoryManipulator: - """ - GitHub Repository Manipulator class + """GitHub Repository Manipulator class Provides functions that can manipulate a set of repositories in GitHub Uses github3.py: https://github.com/sigmavirus24/github3.py """ def __init__(self, username, token, login_method=login, dry_run=False): - """ - Constructor + """Constructor :param username: username :param token: token for access to GitHub :param login_method: method to login to GitHub and provide a GitHub manipulation object @@ -60,8 +57,7 @@ def __init__(self, username, token, login_method=login, dry_run=False): @staticmethod def get_checked_date(date_string): - """ - Return a date dictionary from a text date + """Return a date dictionary from a text date :param date_string: date string YYYY-MM-DD :return: date dictionary """ @@ -74,8 +70,7 @@ def get_checked_date(date_string): raise UserError("Invalid date") def add_all_repos_for_owner(self, owner_name="ISISComputingGroup"): - """ - Add all the repositories for an owner to the list of repositories to manipulate; + """Add all the repositories for an owner to the list of repositories to manipulate; assume owner is an organisation but default to user if organisation doesn't exist :param owner_name: name of the owner; organisation/user :return: @@ -97,8 +92,7 @@ def add_all_repos_for_owner(self, owner_name="ISISComputingGroup"): raise UserError(f"Failed to get owner's repositories. {str(ex)}") def _add_to_repositories_to_use(self, iter_repos): - """ - Add the repositories from the generator to the list and sort by name + """Add the repositories from the generator to the list and sort by name :param iter_repos: generator for repositories :return: """ @@ -107,15 +101,13 @@ def _add_to_repositories_to_use(self, iter_repos): self._sort_repo_list() def _sort_repo_list(self): - """ - sort the repository list + """Sort the repository list :return: nothing """ self._repo_list.sort(key=lambda x: x.name) def use_repos_for_user(self, owner): - """ - Add all repositories from the currently logged in user to the list of repositories to manipulate + """Add all repositories from the currently logged in user to the list of repositories to manipulate :param owner: owner of the repos :return: """ @@ -125,8 +117,7 @@ def use_repos_for_user(self, owner): raise UserError(f"Failed to get user repositories. {str(ex)}") def add_repository_from(self, owner, names): - """ - Add repositories to those to be manipulated + """Add repositories to those to be manipulated :param owner: owner of the repository :param names: repository names :return: nothing @@ -142,8 +133,7 @@ def add_repository_from(self, owner, names): raise UserError(f"Failed to get named repositories. {str(ex)}") def update_milestones(self, date_from, date_to, close_old_milestones): - """ - Update milestones for all selected repositories + """Update milestones for all selected repositories :param date_to: date that the milestone runs to :param date_from: date that the milestone runs from :param close_old_milestones: close old sprint milestones, if possible @@ -172,8 +162,7 @@ def update_milestones(self, date_from, date_to, close_old_milestones): ) def _update_milestones_on_rep(self, due_on, new_milestone_title, repo, close_old_milestones): - """ - Create a new milestones for the repository if it does not exist already + """Create a new milestones for the repository if it does not exist already :param due_on: date that the milestone is due on :param new_milestone_title: title for the milestone :param repo: repository to which it should be added @@ -197,14 +186,12 @@ def _update_milestones_on_rep(self, due_on, new_milestone_title, repo, close_old raise UserError("Unknown error creating milestone on repository.") def _close_milestone_if_old(self, milestone, repo_name): - """ - Close a milestone if it is old + """Close a milestone if it is old :param milestone: milestone to close :param repo_name: repository that the milestone is from :return: :raises UserError: if there is a problem closing the milestone """ - match = re.match(SPRINT_MILESTONE_REGEX, milestone.title) if match is not None: year = int(match.group(1)) @@ -229,8 +216,7 @@ def _close_milestone_if_old(self, milestone, repo_name): ) def update_labels(self, ensure_labels): - """ - Update labels on all repositories + """Update labels on all repositories :param ensure_labels: labels to ensure are on the repository; list of (colour, name) tuples :type ensure_labels: list(tuple(string, string)) :return: @@ -246,8 +232,7 @@ def update_labels(self, ensure_labels): self._update_labels_on_rep(ensure_labels, repo) def _update_labels_on_rep(self, ensure_labels, repo): - """ - Update the labels on a single repository + """Update the labels on a single repository :param ensure_labels: labels to ensure are on the repository; list of (colour, name) tuples :param repo: repository top update :return: @@ -272,8 +257,7 @@ def _update_labels_on_rep(self, ensure_labels, repo): raise UserError("Validation filed on create/modify label") def list_repos(self): - """ - List the repositories + """List the repositories :return: """ print("Repository set") diff --git a/repo_tool/scripts/change_repos.py b/repo_tool/scripts/change_repos.py index 7117654..06c110d 100644 --- a/repo_tool/scripts/change_repos.py +++ b/repo_tool/scripts/change_repos.py @@ -1,7 +1,6 @@ #!/usr/bin/env python -""" -This file is part of the ISIS IBEX application. +"""This file is part of the ISIS IBEX application. Copyright (C) 2012-2016 Science & Technology Facilities Council. All rights reserved. @@ -14,16 +13,14 @@ """ import argparse -from getpass import getpass - import sys +from getpass import getpass from repo_tool.repository_manipulator.RepositoryManipulator import RepositoryManipulator, UserError def main(settings, token): - """ - Main function + """Main function :param settings: settings :param token: git hub token :return: error code @@ -69,8 +66,7 @@ def main(settings, token): def _get_repository_name_from_file(filename): - """ - Get the repository names from a file + """Get the repository names from a file :param filename: filename :return: unique items in file """ @@ -89,8 +85,7 @@ def _get_repository_name_from_file(filename): def _get_labels_from(filename): - """ - Get the labels from a file + """Get the labels from a file :param filename: file to get names from :return: label names and colours """ @@ -130,8 +125,7 @@ def _get_labels_from(filename): def _parse_command_line(): - """ - Parse the command line + """Parse the command line :return: settings for the run """ parser = argparse.ArgumentParser( @@ -243,8 +237,7 @@ def _parse_command_line(): def _get_user_token(): - """ - Get the users access token for github + """Get the users access token for github :return: token """ token = "" diff --git a/repo_tool/scripts/proposal_printing.py b/repo_tool/scripts/proposal_printing.py index 692be83..2f4fcdc 100644 --- a/repo_tool/scripts/proposal_printing.py +++ b/repo_tool/scripts/proposal_printing.py @@ -1,5 +1,4 @@ -""" -A Utility to check and provide a printable list of proposed and ready tickets ready for prioritisation. +"""A Utility to check and provide a printable list of proposed and ready tickets ready for prioritisation. To use this the python-docx package needs to be installed. There is limited error checking, and there are plenty of adaptations to make to this to improve it. This is fairly nasty code - but it is a first iteration to get some of the job done. @@ -8,9 +7,8 @@ Print it by going in print printer propeties -> Priniting Shortcuts -> Pages per sheet = 2 pages per sheet """ -import requests - import docx +import requests from six import moves # Globals - used in a couple of locations @@ -24,8 +22,7 @@ def get_web_content_as_json(url, auth): - """ - This will go and get the contents listed at a URL, and bring it back as a JSON object. + """This will go and get the contents listed at a URL, and bring it back as a JSON object. This needs a try...catch implemented Args: @@ -47,8 +44,7 @@ def get_web_content_as_json(url, auth): class IbexTicket: - """ - Give our use of labels, this is a class to deal with just those aspects needed for printing for prioritisation. + """Give our use of labels, this is a class to deal with just those aspects needed for printing for prioritisation. """ # Internal constants @@ -160,11 +156,11 @@ def priority_decision(self, from_file=False): def get_open_tickets(auth, label): - """ - Get all the open tickets + """Get all the open tickets Will have to page the readbacks Note that page 0 and 1 are the same page! A Page that is too big comes back with no data, just [] + Args: auth: autherentication @@ -203,8 +199,7 @@ def get_open_tickets(auth, label): def load_tickets_from_file(filename): - """ - Load tickets from file + """Load tickets from file Args: filename: @@ -220,8 +215,7 @@ def load_tickets_from_file(filename): def is_number(number): - """ - Check for a number, and swallow the error + """Check for a number, and swallow the error Args: number: number as string @@ -263,8 +257,7 @@ def add_info( def get_tickets_generate_doc(auth, from_file, label): - """ - Get the tickets and generate the document in word + """Get the tickets and generate the document in word Args: auth: wuthentication to use for github from_file: True to get data from a file; False otherwise diff --git a/repo_tool/test/test_login.py b/repo_tool/test/test_login.py index 6107963..e286971 100644 --- a/repo_tool/test/test_login.py +++ b/repo_tool/test/test_login.py @@ -1,11 +1,12 @@ import unittest - from datetime import datetime -from hamcrest import * + from github3 import GitHubError -from repo_tool.repository_manipulator.RepositoryManipulator import RepositoryManipulator, UserError +from hamcrest import * from mock import Mock +from repo_tool.repository_manipulator.RepositoryManipulator import RepositoryManipulator, UserError + class GitHubMock: username = None diff --git a/workflow_support_scripts/configs.py b/workflow_support_scripts/configs.py index b013d78..65ab545 100644 --- a/workflow_support_scripts/configs.py +++ b/workflow_support_scripts/configs.py @@ -1,5 +1,4 @@ -""" -Static variables used for calibration file conversion +"""Static variables used for calibration file conversion """ from datetime import date diff --git a/workflow_support_scripts/convert_curve_files.py b/workflow_support_scripts/convert_curve_files.py index 31e40c8..5959671 100644 --- a/workflow_support_scripts/convert_curve_files.py +++ b/workflow_support_scripts/convert_curve_files.py @@ -1,20 +1,17 @@ -""" -Convert .curve files into the correct format for ISIS +"""Convert .curve files into the correct format for ISIS """ import os - from io import open -from configs import FileTypes, ISISCalibration import utility +from configs import FileTypes, ISISCalibration dir_path = os.path.dirname(os.path.realpath(__file__)) def convert(original_file_name, output_folder, root) -> None: - """ - Converts .curve calibration files to .txt files, adds a header and writes them to the output file + """Converts .curve calibration files to .txt files, adds a header and writes them to the output file :param original_file_name (str): The name of the original file :param output_folder (str): The output folder :param root (str): The root name of the output file @@ -37,8 +34,7 @@ def convert(original_file_name, output_folder, root) -> None: def generate_header(output_file) -> None: - """ - Generate a header in ISIS Calibration File Format + """Generate a header in ISIS Calibration File Format :param output_file (file): The output file :return: None """ diff --git a/workflow_support_scripts/convert_temp_calib_files.py b/workflow_support_scripts/convert_temp_calib_files.py index 59e8905..aa6d05f 100644 --- a/workflow_support_scripts/convert_temp_calib_files.py +++ b/workflow_support_scripts/convert_temp_calib_files.py @@ -1,20 +1,17 @@ -""" -Convert .dat files into the correct format for ISIS +"""Convert .dat files into the correct format for ISIS """ import os - from io import open -from configs import FileTypes import utility +from configs import FileTypes dir_path = os.path.dirname(os.path.realpath(__file__)) def convert(original_file_name, output_folder, root, num_of_header_lines=1): - """ - Converts .dat calibration files to .txt files and writes them to the output file + """Converts .dat calibration files to .txt files and writes them to the output file :param original_file_name (str): The name of the file to be converted :param output_folder (str): The output folder :param root (str): The root of the file name diff --git a/workflow_support_scripts/file_converter.py b/workflow_support_scripts/file_converter.py index 3280940..b38bfde 100644 --- a/workflow_support_scripts/file_converter.py +++ b/workflow_support_scripts/file_converter.py @@ -1,18 +1,16 @@ -import os import argparse - -from configs import FileTypes +import os from typing import Tuple import convert_curve_files import convert_temp_calib_files +from configs import FileTypes dir_path = os.path.dirname(os.path.realpath(__file__)) def get_arguments() -> Tuple[str, str, int]: - """ - Get the input and output folders from the command line + """Get the input and output folders from the command line :return: input_folder, output_folder """ parser = argparse.ArgumentParser(description="Get the input and output folder") @@ -40,13 +38,11 @@ def get_arguments() -> Tuple[str, str, int]: class FileConverter: - """ - Convert RHFE sensor data into the correct format for ISIS + """Convert RHFE sensor data into the correct format for ISIS """ def __init__(self, input_dir: str, output_dir: str): - """ - Initialise the class + """Initialise the class :param input_dir (str): The input directory :param output_dir (str): The output directory """ @@ -54,16 +50,14 @@ def __init__(self, input_dir: str, output_dir: str): self.output_dir = output_dir def exit_if_input_dir_not_exist(self) -> None: - """ - Check the input directory exists and exit if it does not + """Check the input directory exists and exit if it does not :return None """ if not self.check_dir_exists("input"): exit(f"Input folder {self.input_dir} does not exist") def check_dir_exists(self, input_output_dir) -> bool: - """ - Get the input directory if it exists + """Get the input directory if it exists :return input_dir (str): The input directory """ exists = False @@ -77,8 +71,7 @@ def check_dir_exists(self, input_output_dir) -> bool: @staticmethod def file_overwrite_check() -> bool: - """ - Check if the output file exists and if it should be overwritten + """Check if the output file exists and if it should be overwritten :return overwrite (bool): If the file should be overwritten """ overwrite = False @@ -87,8 +80,7 @@ def file_overwrite_check() -> bool: return overwrite def set_up_output_dir(self) -> None: - """ - Set up the output directory + """Set up the output directory :return None """ # Check if the path exists @@ -116,8 +108,7 @@ def set_up_output_dir(self) -> None: @staticmethod def convert_file(root, original_file_name) -> None: - """ - Convert RHFE sensor data into the correct format for ISIS + """Convert RHFE sensor data into the correct format for ISIS :param original_file_name: The original file name :return None """ @@ -131,8 +122,7 @@ def convert_file(root, original_file_name) -> None: ) def convert_all_files(self) -> None: - """ - Convert all files in the input directory + """Convert all files in the input directory :return None """ for root, _, files in os.walk(self.input_dir): diff --git a/workflow_support_scripts/utility.py b/workflow_support_scripts/utility.py index bb17ba4..e2cbdb9 100644 --- a/workflow_support_scripts/utility.py +++ b/workflow_support_scripts/utility.py @@ -1,11 +1,9 @@ -""" -Utility functions to convert files to correct format for ISIS +"""Utility functions to convert files to correct format for ISIS """ def strip_header(lines_to_strip, original_file) -> None: - """ - Strip unnecessary header information from the original file + """Strip unnecessary header information from the original file :param lines_to_strip (int): The number of lines to strip from the original file :param original_file (file): The file to strip header from :return: None @@ -15,8 +13,7 @@ def strip_header(lines_to_strip, original_file) -> None: def format_output_file(original_file, output_file, first_column, second_column) -> None: - """ - Format the output file according to ISIS Calibration File Format + """Format the output file according to ISIS Calibration File Format :param original_file (file): The file to convert from :param output_file (file): The file to convert to :param first_column (int): The first column in the output_file @@ -32,8 +29,7 @@ def format_output_file(original_file, output_file, first_column, second_column) def format_output_file_name(original_file_name, original_file_type, output_file_type) -> str: - """ - Format the name of the output file + """Format the name of the output file :param original_file_name (str): The name of the original file :param original_file_type (str): The original file's type :param output_file_type (str): The output file's type