From c8733b4c762a9fc5f303f1fd085f6d50a49fbd56 Mon Sep 17 00:00:00 2001 From: Adrian Chifor Date: Tue, 17 Apr 2018 13:44:58 +0100 Subject: [PATCH 1/4] Save and check kapitan version used to compile to keep things consistent --- .gitignore | 1 + kapitan/targets.py | 4 +++- kapitan/utils.py | 56 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2b1b633dd..ad083cb2b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ kapitan.egg-info/ tags .python-version +.kapitan diff --git a/kapitan/targets.py b/kapitan/targets.py index ee0967055..0e8efa600 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -35,7 +35,7 @@ import time from kapitan.resources import search_imports, resource_callbacks, inventory, inventory_reclass -from kapitan.utils import jsonnet_file, prune_empty, render_jinja2, PrettyDumper +from kapitan.utils import jsonnet_file, prune_empty, render_jinja2, PrettyDumper, enforce_version from kapitan.secrets import secret_gpg_raw_read, secret_token_from_tag, secret_token_attributes from kapitan.secrets import SECRET_TOKEN_TAG_PATTERN, secret_gpg_read from kapitan.errors import KapitanError, CompileError @@ -53,6 +53,8 @@ def compile_targets(inventory_path, search_path, output_path, parallel, targets, target_objs = load_target_inventory(inventory_path, targets) + enforce_version() + pool = multiprocessing.Pool(parallel) # append "compiled" to output_path so we can safely overwrite it compile_path = os.path.join(output_path, "compiled") diff --git a/kapitan/utils.py b/kapitan/utils.py index 8a0eaccd4..b760e742e 100644 --- a/kapitan/utils.py +++ b/kapitan/utils.py @@ -22,18 +22,32 @@ from hashlib import sha256 import logging import os +import sys import stat import collections import jinja2 import _jsonnet as jsonnet import yaml +from distutils.version import StrictVersion +from kapitan.version import VERSION from kapitan.errors import CompileError logger = logging.getLogger(__name__) +class termcolor: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + def normalise_join_path(dirname, path): "Join dirname with path and return in normalised form" logger.debug(os.path.normpath(os.path.join(dirname, path))) @@ -49,10 +63,12 @@ def jinja2_sha256_hex_filter(string): "Returns hex digest for string" return sha256(string.encode("UTF-8")).hexdigest() + def jinja2_yaml_filter(obj): "Returns yaml for object" return yaml.safe_dump(obj, default_flow_style=False) + def render_jinja2_file(name, context): "Render jinja2 file name with context" path, filename = os.path.split(name) @@ -95,9 +111,10 @@ def render_jinja2(path, context): # get subpath and filename, strip any leading/trailing / name = render_path[len(os.path.commonprefix([root, path])):].strip('/') try: - rendered[name] = {"content": render_jinja2_file(render_path, context), - "mode": file_mode(render_path) - } + rendered[name] = { + "content": render_jinja2_file(render_path, context), + "mode": file_mode(render_path) + } except Exception as e: logger.error("Jinja2 error: failed to render %s: %s", render_path, str(e)) raise CompileError(e) @@ -217,3 +234,36 @@ def get_directory_hash(directory): raise return hash.hexdigest() + + +def enforce_version(): + ''' + Enforces that the last version of kapitan used is at least smaller or equal to current version. + If the last version of kapitan used is bigger, it will give instructions on how to upgrade and exit(1). + ''' + if os.path.exists('.kapitan'): + with open('.kapitan', 'r') as f: + dot_kapitan = yaml.safe_load(f) + # If 'version is not saved' or 'saved version is smaller or equal than current version' + if not dot_kapitan['version'] or StrictVersion(dot_kapitan['version']) <= StrictVersion(VERSION): + save_version() + # If 'saved version is bigger than current version' + else: + print(f'{termcolor.WARNING}Current version: {VERSION}') + print(f'Last used version (in .kapitan): {dot_kapitan["version"]}{termcolor.ENDC}\n') + print(f'Please upgrade kapitan to at least "{dot_kapitan["version"]}" in order to keep results consistent:\n') + print('Docker: docker pull deepmind/kapitan') + print('Pip (user): pip3 install --user --upgrade git+https://github.com/deepmind/kapitan.git --process-dependency-links') + print('Pip (system): sudo pip3 install --upgrade git+https://github.com/deepmind/kapitan.git --process-dependency-links') + sys.exit(1) + else: + save_version() + + +def save_version(): + ''' + Saves the current kapitan version to a local .kapitan file + ''' + with open('.kapitan', 'w') as f: + dot_kapitan = {'version': VERSION} + yaml.safe_dump(dot_kapitan, stream=f, default_flow_style=False) From d0ee1209eff120cfc07f832d2782ee855d8b0d3d Mon Sep 17 00:00:00 2001 From: Adrian Chifor Date: Tue, 17 Apr 2018 13:55:23 +0100 Subject: [PATCH 2/4] Flag to ignore version check --- kapitan/cli.py | 5 ++++- kapitan/targets.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 112898cb0..0c80bcdb9 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -83,6 +83,9 @@ def main(): action='store_true', default=False) compile_parser.add_argument('--inventory-path', default='./inventory', help='set inventory path, default is "./inventory"') + compile_parser.add_argument('--ignore-version-check', + help='ignore the last kapitan version used to compile (from .kapitan)', + action='store_true', default=False) inventory_parser = subparser.add_parser('inventory', help='show inventory') inventory_parser.add_argument('--target-name', '-t', default='', @@ -171,7 +174,7 @@ def main(): gpg_obj = secret_gpg_backend() compile_targets(args.inventory_path, search_path, args.output_path, - args.parallelism, args.targets, + args.parallelism, args.targets, args.ignore_version_check, prune=(not args.no_prune), secrets_path=args.secrets_path, secrets_reveal=args.reveal, gpg_obj=gpg_obj) diff --git a/kapitan/targets.py b/kapitan/targets.py index 0e8efa600..1850efef1 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) -def compile_targets(inventory_path, search_path, output_path, parallel, targets, **kwargs): +def compile_targets(inventory_path, search_path, output_path, parallel, targets, ignore_version_check, **kwargs): """ Searches and loads target files, and runs compile_target_file() on a multiprocessing pool with parallel number of processes. @@ -53,7 +53,8 @@ def compile_targets(inventory_path, search_path, output_path, parallel, targets, target_objs = load_target_inventory(inventory_path, targets) - enforce_version() + if not ignore_version_check: + enforce_version() pool = multiprocessing.Pool(parallel) # append "compiled" to output_path so we can safely overwrite it From 068ea70aa9ed73ef65eb22df4af2f98773b1c1a7 Mon Sep 17 00:00:00 2001 From: Adrian Chifor Date: Tue, 17 Apr 2018 13:55:33 +0100 Subject: [PATCH 3/4] Version increment --- kapitan/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/version.py b/kapitan/version.py index e732207fa..5cb0b1ef3 100644 --- a/kapitan/version.py +++ b/kapitan/version.py @@ -17,7 +17,7 @@ "Project description variables" PROJECT_NAME = 'kapitan' -VERSION = '0.14.0' +VERSION = '0.15.0' DESCRIPTION = ('Generic templated configuration management for Kubernetes,' 'Terraform and other things') AUTHOR = 'Ricardo Amaro' From 4bf403384c0db2f4ca13c27055e839cf8f28d12d Mon Sep 17 00:00:00 2001 From: Adrian Chifor Date: Tue, 17 Apr 2018 15:05:03 +0100 Subject: [PATCH 4/4] Improved design a bit --- kapitan/cli.py | 10 ++++++++-- kapitan/targets.py | 7 ++----- kapitan/utils.py | 11 +++-------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 0c80bcdb9..1e1a68699 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -26,7 +26,7 @@ import traceback import yaml -from kapitan.utils import jsonnet_file, PrettyDumper, flatten_dict, searchvar, deep_get +from kapitan.utils import jsonnet_file, PrettyDumper, flatten_dict, searchvar, deep_get, check_version, save_version from kapitan.targets import compile_targets from kapitan.resources import search_imports, resource_callbacks, inventory_reclass from kapitan.version import PROJECT_NAME, DESCRIPTION, VERSION @@ -173,11 +173,17 @@ def main(): search_path = os.path.abspath(args.search_path) gpg_obj = secret_gpg_backend() + if not args.ignore_version_check: + check_version() + compile_targets(args.inventory_path, search_path, args.output_path, - args.parallelism, args.targets, args.ignore_version_check, + args.parallelism, args.targets, prune=(not args.no_prune), secrets_path=args.secrets_path, secrets_reveal=args.reveal, gpg_obj=gpg_obj) + if not args.ignore_version_check: + save_version() + elif cmd == 'inventory': if args.pattern and args.target_name == '': parser.error("--pattern requires --target_name") diff --git a/kapitan/targets.py b/kapitan/targets.py index 1850efef1..ee0967055 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -35,14 +35,14 @@ import time from kapitan.resources import search_imports, resource_callbacks, inventory, inventory_reclass -from kapitan.utils import jsonnet_file, prune_empty, render_jinja2, PrettyDumper, enforce_version +from kapitan.utils import jsonnet_file, prune_empty, render_jinja2, PrettyDumper from kapitan.secrets import secret_gpg_raw_read, secret_token_from_tag, secret_token_attributes from kapitan.secrets import SECRET_TOKEN_TAG_PATTERN, secret_gpg_read from kapitan.errors import KapitanError, CompileError logger = logging.getLogger(__name__) -def compile_targets(inventory_path, search_path, output_path, parallel, targets, ignore_version_check, **kwargs): +def compile_targets(inventory_path, search_path, output_path, parallel, targets, **kwargs): """ Searches and loads target files, and runs compile_target_file() on a multiprocessing pool with parallel number of processes. @@ -53,9 +53,6 @@ def compile_targets(inventory_path, search_path, output_path, parallel, targets, target_objs = load_target_inventory(inventory_path, targets) - if not ignore_version_check: - enforce_version() - pool = multiprocessing.Pool(parallel) # append "compiled" to output_path so we can safely overwrite it compile_path = os.path.join(output_path, "compiled") diff --git a/kapitan/utils.py b/kapitan/utils.py index b760e742e..96e7d0338 100644 --- a/kapitan/utils.py +++ b/kapitan/utils.py @@ -236,19 +236,16 @@ def get_directory_hash(directory): return hash.hexdigest() -def enforce_version(): +def check_version(): ''' - Enforces that the last version of kapitan used is at least smaller or equal to current version. + Checks that the last version of kapitan used is at least smaller or equal to current version. If the last version of kapitan used is bigger, it will give instructions on how to upgrade and exit(1). ''' if os.path.exists('.kapitan'): with open('.kapitan', 'r') as f: dot_kapitan = yaml.safe_load(f) - # If 'version is not saved' or 'saved version is smaller or equal than current version' - if not dot_kapitan['version'] or StrictVersion(dot_kapitan['version']) <= StrictVersion(VERSION): - save_version() # If 'saved version is bigger than current version' - else: + if dot_kapitan['version'] and StrictVersion(dot_kapitan['version']) > StrictVersion(VERSION): print(f'{termcolor.WARNING}Current version: {VERSION}') print(f'Last used version (in .kapitan): {dot_kapitan["version"]}{termcolor.ENDC}\n') print(f'Please upgrade kapitan to at least "{dot_kapitan["version"]}" in order to keep results consistent:\n') @@ -256,8 +253,6 @@ def enforce_version(): print('Pip (user): pip3 install --user --upgrade git+https://github.com/deepmind/kapitan.git --process-dependency-links') print('Pip (system): sudo pip3 install --upgrade git+https://github.com/deepmind/kapitan.git --process-dependency-links') sys.exit(1) - else: - save_version() def save_version():