diff --git a/senza/cli.py b/senza/cli.py index ec237240..428a4f40 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -19,13 +19,14 @@ import senza.stups.taupage as taupage import requests import yaml +import senza.respawn as respawn from botocore.exceptions import ClientError from clickclick import (Action, FloatRange, OutputFormat, choice, error, fatal_error, info, ok) from clickclick.console import print_table -from spotinst.components import elastigroup_api +from .spotinst.components import elastigroup_api from .arguments import (GLOBAL_OPTIONS, json_output_option, output_option, parameter_file_option, region_option, stacktrace_visible_option, watch_option, @@ -42,8 +43,7 @@ from .manaus.exceptions import VPCError from .manaus.route53 import Route53, Route53Record from .manaus.utils import extract_client_error_code -from .patch import patch_auto_scaling_group -from .respawn import get_auto_scaling_group, respawn_auto_scaling_group +from .patch import patch_auto_scaling_group, patch_elastigroup from .stups.piu import Piu from .subcommands.config import cmd_config from .subcommands.root import cli @@ -1476,7 +1476,7 @@ def get_auto_scaling_groups(stack_refs, region): def patch(stack_ref, region, image, instance_type, user_data): '''Patch specific properties of existing stack. - Currently only supports patching ASG launch configurations.''' + Currently supports patching ASG launch configurations and ElastiGroup groups.''' stack_refs = get_stack_refs(stack_ref) region = get_region(region) @@ -1499,25 +1499,55 @@ def patch(stack_ref, region, image, instance_type, user_data): asg = BotoClientProxy('autoscaling', region) - for asg_name in get_auto_scaling_groups(stack_refs, region): - with Action('Patching Auto Scaling Group {}..'.format(asg_name)) as act: - result = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) - groups = result['AutoScalingGroups'] - for group in groups: - if not patch_auto_scaling_group(group, region, properties): - act.ok('NO CHANGES') + stacks = get_stacks(stack_refs, region) + for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): + if group['type'] == ELASTIGROUP_TYPE: + patch_spotinst_elastigroup(properties, group['resource_id'], region, group['stack_name']) + elif group['type'] == AUTO_SCALING_GROUP_TYPE: + patch_aws_asg(properties, region, asg, group['resource_id']) + + +def patch_aws_asg(properties, region, asg, asg_name): + ''' + Patch an AWS Auto Scaling Group + ''' + with Action('Patching Auto Scaling Group {}..'.format(asg_name)) as act: + result = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + groups = result['AutoScalingGroups'] + for group in groups: + if not patch_auto_scaling_group(group, region, properties): + act.ok('NO CHANGES') + + +def patch_spotinst_elastigroup(properties, elastigroup_id, region, stack_name): + ''' + Patch specific properties of an existing ElastiGroup + ''' + + spotinst_account_data = elastigroup_api.get_spotinst_account_data(region, stack_name) + + with Action('Patching ElastiGroup {} (ID: {})..'.format(stack_name, elastigroup_id)) as act: + groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) + + for group in groups: + if not patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data): + act.ok('NO CHANGES') @cli.command('respawn-instances') @click.argument('stack_ref', nargs=-1) @click.option('--inplace', - is_flag=True, help='Perform inplace update, do not scale out') + is_flag=True, help='Perform inplace update, do not scale out. Ignored for ElastiGroups.') @click.option('-f', '--force', is_flag=True, - help='Force respawn even if Launch Configuration is unchanged') + help='Force respawn even if Launch Configuration is unchanged. Ignored for ElastiGroups.') +@click.option('--batch_size_percentage', + metavar='PERCENTAGE', + help='Percentage (int value) of the ElastiGroup cluster that is respawned in each step.' + ' Valid only for ElastiGroups. The default value for this of 20.') @region_option @stacktrace_visible_option -def respawn_instances(stack_ref, inplace, force, region): +def respawn_instances(stack_ref, inplace, force, batch_size_percentage, region): '''Replace all EC2 instances in Auto Scaling Group(s) Performs a rolling update to prevent downtimes.''' @@ -1526,8 +1556,12 @@ def respawn_instances(stack_ref, inplace, force, region): region = get_region(region) check_credentials(region) - for asg_name in get_auto_scaling_groups(stack_refs, region): - respawn_auto_scaling_group(asg_name, region, inplace=inplace, force=force) + stacks = get_stacks(stack_refs, region) + for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): + if group['type'] == AUTO_SCALING_GROUP_TYPE: + respawn.respawn_auto_scaling_group(group['resource_id'], region, inplace=inplace, force=force) + elif group['type'] == ELASTIGROUP_TYPE: + respawn.respawn_elastigroup(group['resource_id'], group['stack_name'], region, batch_size_percentage) @cli.command() @@ -1545,14 +1579,13 @@ def scale(stack_ref, region, desired_capacity, force): region = get_region(region) check_credentials(region) - asg = BotoClientProxy('autoscaling', region) - stacks = get_stacks(stack_refs, region) stack_count = len(stacks) if not force and stack_count > 1: confirm_str = 'Number of stacks to be scaled - {}. Do you want to continue?'.format(stack_count) click.confirm(confirm_str, abort=True) + asg = BotoClientProxy('autoscaling', region) for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): if group['type'] == AUTO_SCALING_GROUP_TYPE: scale_auto_scaling_group(asg, group['resource_id'], desired_capacity) @@ -1564,36 +1597,30 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): ''' Commands to scale an ElastiGroup ''' - cf = boto3.client('cloudformation', region) - template = cf.get_template(StackName=stack_name)['TemplateBody'] + spotinst_account_data = elastigroup_api.get_spotinst_account_data(region, stack_name) - spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] - spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) - group = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) - capacity = group['capacity'] + for group in groups: + capacity = group['capacity'] - with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( - stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: - if capacity['target'] == desired_capacity: - act.ok('NO CHANGES') - else: - minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] - maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] + with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( + stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: + if capacity['target'] == desired_capacity: + act.ok('NO CHANGES') + else: + minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] + maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - elastigroup_api.update_capacity(minimum, - maximum, - desired_capacity, - elastigroup_id, - spotinst_account_id, - spotinst_token) + elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, + spotinst_account_data) def scale_auto_scaling_group(asg, asg_name, desired_capacity): ''' Commands to scale an AWS Auto Scaling Group ''' - group = get_auto_scaling_group(asg, asg_name) + group = respawn.get_auto_scaling_group(asg, asg_name) current_capacity = group['DesiredCapacity'] with Action('Scaling {} from {} to {} instances..'.format( asg_name, current_capacity, desired_capacity)) as act: diff --git a/senza/patch.py b/senza/patch.py index 920dc1d1..3989f29f 100644 --- a/senza/patch.py +++ b/senza/patch.py @@ -1,9 +1,11 @@ import codecs +import base64 import datetime import yaml +from .spotinst.components import elastigroup_api from .exceptions import InvalidUserDataType from .manaus.boto_proxy import BotoClientProxy @@ -29,6 +31,19 @@ ]) +def should_patch_user_data(new_val, old_val): + ''' + Validate if User Data should be patched. + ''' + current_user_data = yaml.safe_load(old_val) + if isinstance(new_val, dict): + return True + elif isinstance(current_user_data, dict): + raise InvalidUserDataType(type(current_user_data), + type(new_val)) + return False + + def patch_user_data(old: str, new: dict): first_line, sep, data = old.partition('\n') data = yaml.safe_load(data) @@ -60,12 +75,8 @@ def patch_auto_scaling_group(group: dict, region: str, properties: dict): for key, val in properties.items(): if key == 'UserData': - current_user_data = yaml.safe_load(kwargs['UserData']) - if isinstance(val, dict): + if should_patch_user_data(val, kwargs['UserData']): kwargs[key] = patch_user_data(kwargs[key], val) - elif isinstance(current_user_data, dict): - raise InvalidUserDataType(type(current_user_data), - type(val)) else: kwargs[key] = val asg.create_launch_configuration(**kwargs) @@ -73,3 +84,37 @@ def patch_auto_scaling_group(group: dict, region: str, properties: dict): LaunchConfigurationName=kwargs['LaunchConfigurationName']) changed = True return changed + + +def patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data): + ''' + Patch specific properties of an existing ElastiGroup + ''' + changed = False + properties_to_patch = {} + + group_user_data = group['compute']['launchSpecification']['userData'] + current_user_data = codecs.decode(group_user_data.encode('utf-8'), 'base64').decode('utf-8') + + current_properties = { + 'ImageId': group['compute']['launchSpecification']['imageId'], + 'InstanceType': group['compute']['instanceTypes']['ondemand'], + 'UserData': current_user_data + } + + for key, val in properties.items(): + if key in current_properties: + if key == 'UserData': + if should_patch_user_data(val, current_properties[key]): + patched_user_data = patch_user_data(current_properties[key], val) + encoded_user_data = base64.urlsafe_b64encode(patched_user_data.encode('utf-8')).decode('utf-8') + properties_to_patch[key] = encoded_user_data + else: + if current_properties[key] != val: + properties_to_patch[key] = val + + if len(properties_to_patch) > 0: + elastigroup_api.patch_elastigroup(properties_to_patch, elastigroup_id, spotinst_account_data) + changed = True + + return changed diff --git a/senza/respawn.py b/senza/respawn.py index 3aeaff3f..1debab17 100644 --- a/senza/respawn.py +++ b/senza/respawn.py @@ -6,10 +6,15 @@ from clickclick import Action, info from .manaus.boto_proxy import BotoClientProxy +from .spotinst.components import elastigroup_api SCALING_PROCESSES_TO_SUSPEND = ['AZRebalance', 'AlarmNotification', 'ScheduledActions'] RUNNING_LIFECYCLE_STATES = set(['Pending', 'InService', 'Rebooting']) +ELASTIGROUP_TERMINATED_DEPLOY_STATUS = ['stopped', 'failed'] + +DEFAULT_BATCH_SIZE = 20 + def get_auto_scaling_group(asg, asg_name: str): '''Get boto3 Auto Scaling Group by name or raise exception''' @@ -151,3 +156,37 @@ def respawn_auto_scaling_group(asg_name: str, region: str, inplace: bool=False, inplace) else: info('Nothing to do') + + +def respawn_elastigroup(elastigroup_id: str, stack_name: str, region: str, batch_size: int): + ''' + Respawn all instances in the ElastiGroup. + ''' + + if batch_size is None or batch_size < 1: + batch_size = DEFAULT_BATCH_SIZE + + spotinst_account = elastigroup_api.get_spotinst_account_data(region, stack_name) + + info('Redeploying the cluster for ElastiGroup {} (ID {})'.format(stack_name, elastigroup_id)) + + deploy_output = elastigroup_api.deploy(batch_size=batch_size, grace_period=600, elastigroup_id=elastigroup_id, + spotinst_account_data=spotinst_account) + + deploy_count = len(deploy_output) + deploys_finished = 0 + with Action('Waiting for deploy to complete. Total of {} deploys'.format(deploy_count)) as act: + while True: + for deploy in deploy_output: + deploy_status = elastigroup_api.deploy_status(deploy['id'], elastigroup_id, spotinst_account) + for ds in deploy_status: + if ds['id'] == deploy['id']: + if ds['progress']['value'] >= 100\ + or ds['status'].lower() in ELASTIGROUP_TERMINATED_DEPLOY_STATUS: + deploys_finished += 1 + info('Deploy {} finished with status {}'.format(ds['id'], ds['status'])) + + if deploys_finished == deploy_count: + break + time.sleep(2) + act.progress() diff --git a/spotinst/__init__.py b/senza/spotinst/__init__.py similarity index 100% rename from spotinst/__init__.py rename to senza/spotinst/__init__.py diff --git a/spotinst/components/__init__.py b/senza/spotinst/components/__init__.py similarity index 100% rename from spotinst/components/__init__.py rename to senza/spotinst/components/__init__.py diff --git a/spotinst/components/elastigroup.py b/senza/spotinst/components/elastigroup.py similarity index 99% rename from spotinst/components/elastigroup.py rename to senza/spotinst/components/elastigroup.py index 6043d975..7d39d6b8 100644 --- a/spotinst/components/elastigroup.py +++ b/senza/spotinst/components/elastigroup.py @@ -15,7 +15,7 @@ from senza.components.taupage_auto_scaling_group import check_application_id, check_application_version, \ check_docker_image_exists, generate_user_data from senza.utils import ensure_keys -from spotinst import MissingSpotinstAccount +from senza.spotinst import MissingSpotinstAccount SPOTINST_LAMBDA_FORMATION_ARN = 'arn:aws:lambda:{}:178579023202:function:spotinst-cloudformation' SPOTINST_API_URL = 'https://api.spotinst.io' diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py new file mode 100644 index 00000000..95cffa82 --- /dev/null +++ b/senza/spotinst/components/elastigroup_api.py @@ -0,0 +1,174 @@ +''' +Wrapper methods for ElastiGroup's API +''' +import requests +import json +import boto3 + + +SPOTINST_API_URL = 'https://api.spotinst.io' + +DEPLOY_STRATEGY_RESTART = 'RESTART_SERVER' +DEPLOY_STRATEGY_REPLACE = 'REPLACE_SERVER' + + +class SpotInstAccountData: + ''' + Data required to access SpotInst API + ''' + def __init__(self, account_id, access_token): + self.account_id = account_id + self.access_token = access_token + + +def get_spotinst_account_data(region, stack_name): + ''' + Extracts required parameters required to access SpotInst API + ''' + cf = boto3.client('cloudformation', region) + template = cf.get_template(StackName=stack_name)['TemplateBody'] + + spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] + spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + + return SpotInstAccountData(spotinst_account_id, spotinst_token) + + +def update_elastigroup(body, elastigroup_id, spotinst_account_data): + ''' + Performs the update ElastiGroup API call. + + Note: Although this should only return one element in the list, + it still returns the entire list to prevent some silent decision making + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.put( + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups + + +def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_data): + ''' + Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. + Returns the updated description of the ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + ''' + + new_capacity = { + 'target': target, + 'minimum': minimum, + 'maximum': maximum + } + + body = {'group': {'capacity': new_capacity}} + + return update_elastigroup(body, elastigroup_id, spotinst_account_data) + + +def get_elastigroup(elastigroup_id, spotinst_account_data): + ''' + Returns a list containing the description of an ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + + Note: Although this should only return one element in the list, + it still returns the entire list to prevent some silent decision making + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.get( + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups + + +def patch_elastigroup(properties, elastigroup_id, spotinst_account_data): + ''' + Patch specific properties of the ElastiGroup. + ''' + compute = {} + if 'InstanceType' in properties: + compute['instanceTypes'] = { + 'ondemand': properties['InstanceType'], + } + + if 'ImageId' in properties: + compute.setdefault('launchSpecification', {})['imageId'] = properties['ImageId'] + + if 'UserData' in properties: + compute.setdefault('launchSpecification', {})['userData'] = properties['UserData'] + + body = {'group': {'compute': compute}} + return update_elastigroup(body, elastigroup_id, spotinst_account_data) + + +def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, + elastigroup_id=None, spotinst_account_data=None): + ''' + Triggers Blue/Green Deployment that replaces the existing instances in the Elastigroup + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/deploy/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + body = { + 'batchSizePercentage': batch_size, + 'gracePeriod': grace_period, + 'strategy': { + 'action': strategy + } + } + + response = requests.put( + '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + data = response.json() + deploys = data.get("response", {}).get("items", []) + + return deploys + + +def deploy_status(deploy_id, elastigroup_id, spotinst_account_data): + ''' + Obtains the current status of a deployment. + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/deploy-status/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.get( + '{}/aws/ec2/group/{}/roll/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, deploy_id, + spotinst_account_data.account_id), + headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + deploys = data.get("response", {}).get("items", []) + + return deploys diff --git a/spotinst/components/elastigroup_api.py b/spotinst/components/elastigroup_api.py deleted file mode 100644 index 063a251d..00000000 --- a/spotinst/components/elastigroup_api.py +++ /dev/null @@ -1,60 +0,0 @@ -''' -Wrapper methods for ElastiGroup's API -''' -import requests -import json - - -SPOTINST_API_URL = 'https://api.spotinst.io' - - -def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. - Returns the updated description of the ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - new_capacity = { - 'target': target, - 'minimum': minimum, - 'maximum': maximum - } - - body = {'group': {'capacity': new_capacity}} - response = requests.put( - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), - headers=headers, timeout=10, data=json.dumps(body)) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] - - -def get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Returns the description of an ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - response = requests.get( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - headers=headers, timeout=5) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 2120a4e3..903a97e3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1606,7 +1606,8 @@ def test_patch(monkeypatch): 'CreationTime': '2016-06-14'}]} boto3.describe_stack_resources.return_value = {'StackResources': [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg'}]} + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} group = {'AutoScalingGroupName': 'myasg'} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} image = MagicMock() @@ -1631,21 +1632,53 @@ def patch_auto_scaling_group(group, region, properties): def test_respawn(monkeypatch): boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) - monkeypatch.setattr('senza.cli.get_auto_scaling_groups', lambda *args: 'myasg') - monkeypatch.setattr('senza.cli.respawn_auto_scaling_group', lambda *args, **kwargs: None) + boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', + 'CreationTime': '2016-06-14'}]} + boto3.describe_stack_resources.return_value = {'StackResources': [ + { + 'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1' + }]} + monkeypatch.setattr('senza.respawn.respawn_auto_scaling_group', lambda *args, **kwargs: None) runner = CliRunner() runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], catch_exceptions=False) -def test_scale(monkeypatch): +def test_respawn_elastigroup(monkeypatch): boto3 = MagicMock() + monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', 'CreationTime': '2016-06-14'}]} + + elastigroup_id = 'myelasti' boto3.describe_stack_resources.return_value = {'StackResources': - [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg', + [{'ResourceType': 'Custom::elastigroup', + 'PhysicalResourceId': elastigroup_id, 'StackName': 'myapp-1'}]} + + test = {'success': False} + + def verification(*args): + test['success'] = True + + monkeypatch.setattr('senza.respawn.respawn_elastigroup', verification) + runner = CliRunner() + runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], + catch_exceptions=False) + + assert test['success'] + + +def test_scale(monkeypatch): + boto3 = MagicMock() + boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', + 'CreationTime': '2016-06-14'}]} + boto3.describe_stack_resources.return_value = {'StackResources': [ + {'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} # NOTE: we are using invalid MinSize (< capacity) here to get one more line covered ;-) group = {'AutoScalingGroupName': 'myasg', 'DesiredCapacity': 1, 'MinSize': 3, 'MaxSize': 1} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} @@ -1685,31 +1718,31 @@ def test_scale_elastigroup(monkeypatch): } } - group = { + group = [{ 'capacity': { 'minimum': 1, 'maximum': 2, 'target': 1, 'unit': 'instance' } - } + }] get_elastigroup = MagicMock() get_elastigroup.return_value = group - update = { + update = [{ 'capacity': { 'minimum': 1, 'maximum': 3, 'target': 3, 'unit': 'instance' } - } + }] update_capacity = MagicMock() update_capacity.return_value = update monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) - monkeypatch.setattr('spotinst.components.elastigroup_api.get_elastigroup', get_elastigroup) - monkeypatch.setattr('spotinst.components.elastigroup_api.update_capacity', update_capacity) + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.get_elastigroup', get_elastigroup) + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.update_capacity', update_capacity) runner = CliRunner() result = runner.invoke(cli, ['scale', 'myapp', '1', '3', '--region=aa-fakeregion-1'], diff --git a/tests/test_elastigroup.py b/tests/test_elastigroup.py index 865eb394..c70857b2 100644 --- a/tests/test_elastigroup.py +++ b/tests/test_elastigroup.py @@ -3,13 +3,12 @@ import responses from mock import MagicMock -from senza.definitions import AccountArguments -from spotinst import MissingSpotinstAccount -from spotinst.components.elastigroup import component_elastigroup, ELASTIGROUP_DEFAULT_PRODUCT, \ - ELASTIGROUP_DEFAULT_STRATEGY, resolve_account_id, SPOTINST_API_URL, extract_block_mappings, \ - extract_auto_scaling_rules, ensure_instance_monitoring, ensure_default_strategy, extract_autoscaling_capacity, \ - ensure_default_product, fill_standard_tags, extract_subnets, extract_load_balancer_name, extract_public_ips, \ - extract_image_id, extract_security_group_ids, extract_instance_types, extract_instance_profile +from senza.spotinst import MissingSpotinstAccount +from senza.spotinst.components.elastigroup import (component_elastigroup, ELASTIGROUP_DEFAULT_PRODUCT, + ELASTIGROUP_DEFAULT_STRATEGY, resolve_account_id, SPOTINST_API_URL, extract_block_mappings, + extract_auto_scaling_rules, ensure_instance_monitoring, ensure_default_strategy, extract_autoscaling_capacity, + ensure_default_product, fill_standard_tags, extract_subnets, extract_load_balancer_name, extract_public_ips, + extract_image_id, extract_security_group_ids, extract_instance_types, extract_instance_profile) def test_component_elastigroup_defaults(monkeypatch): @@ -37,7 +36,7 @@ def test_component_elastigroup_defaults(monkeypatch): mock_resolve_account_id = MagicMock() mock_resolve_account_id.return_value = 'act-12345abcdef' - monkeypatch.setattr('spotinst.components.elastigroup.resolve_account_id', mock_resolve_account_id) + monkeypatch.setattr('senza.spotinst.components.elastigroup.resolve_account_id', mock_resolve_account_id) mock_account_info = MagicMock() mock_account_info.Region = "reg1" diff --git a/tests/test_elastigroup_api.py b/tests/test_elastigroup_api.py index 39f92a48..b429e45b 100644 --- a/tests/test_elastigroup_api.py +++ b/tests/test_elastigroup_api.py @@ -1,5 +1,6 @@ import responses -from spotinst.components.elastigroup_api import update_capacity, get_elastigroup, SPOTINST_API_URL +from senza.spotinst.components.elastigroup_api import update_capacity, get_elastigroup, patch_elastigroup, deploy, \ + deploy_status, SPOTINST_API_URL, SpotInstAccountData def test_update_capacity(monkeypatch): @@ -19,15 +20,15 @@ def test_update_capacity(monkeypatch): } elastigroup_id = 'sig-xfy' - spotinst_account_id = 'act-zwk' - spotinst_token = 'fake-token' + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: rsps.add(rsps.PUT, - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), status=200, json=update) - update_response = update_capacity(1, 3, 3, elastigroup_id, spotinst_account_id, spotinst_token) + update_response = update_capacity(1, 3, 3, elastigroup_id, spotinst_account_data)[0] assert update_response['id'] == elastigroup_id assert update_response['name'] == 'my-app-1' assert update_response['capacity']['minimum'] == 1 @@ -47,13 +48,123 @@ def test_get_elastigroup(monkeypatch): } elastigroup_id = 'sig-xfy' - spotinst_account_id = 'act-zwk' - spotinst_token = 'fake-token' + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: - rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + rsps.add(rsps.GET, '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), status=200, json=group) - group = get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) + group = get_elastigroup(elastigroup_id, spotinst_account_data)[0] assert group['id'] == elastigroup_id assert group['name'] == 'my-app-1' + + +def test_patch_elastigroup(monkeypatch): + patch = { + 'ImageId': 'image-foo', + 'InstanceType': 'm1.micro', + 'UserData': 'user-data-value' + } + + update_response = { + 'response': { + 'items': [{ + 'compute': { + 'instanceTypes': { + 'ondemand': 'm1.micro', + 'spot': [ + 'm1.micro' + ] + }, + 'launchSpecification': { + 'imageId': 'image-foo', + 'userData': 'user-data-value' + } + } + }] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + rsps.add(rsps.PUT, '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), + status=200, + json=update_response) + + patch_response = patch_elastigroup(patch, elastigroup_id, spotinst_account_data)[0] + assert patch_response['compute']['launchSpecification']['imageId'] == 'image-foo' + assert patch_response['compute']['instanceTypes']['ondemand'] == 'm1.micro' + assert patch_response['compute']['launchSpecification']['userData'] == 'user-data-value' + + +def test_deploy(monkeypatch): + response_json = { + "response": { + "items": [ + { + 'id': 'deploy-id', + 'status': 'STARTING', + 'currentBatch': 1, + 'numOfBatches': 1, + 'progress': { + 'unit': 'percentage', + 'value': 0 + } + } + ] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + rsps.add(rsps.PUT, '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), + status=200, + json=response_json) + + deploy_response = deploy(batch_size=35, grace_period=50, elastigroup_id=elastigroup_id, + spotinst_account_data=spotinst_account_data)[0] + + assert deploy_response['id'] == 'deploy-id' + assert deploy_response['status'] == 'STARTING' + assert deploy_response['numOfBatches'] == 1 + + +def test_deploy_status(monkeypatch): + deploy_id = 'deploy-id-x' + response_json = { + "response": { + "items": [ + { + 'id': deploy_id, + 'status': 'STARTING', + 'currentBatch': 13, + 'numOfBatches': 20, + 'progress': { + 'unit': 'percentage', + 'value': 65 + } + } + ] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + rsps.add(rsps.GET, '{}/aws/ec2/group/{}/roll/{}?accountId={}'.format(SPOTINST_API_URL, + elastigroup_id, + deploy_id, + spotinst_account_data.account_id), + status=200, + json=response_json) + + deploy_status_response = deploy_status(deploy_id, elastigroup_id, spotinst_account_data)[0] + + assert deploy_status_response['id'] == deploy_id + assert deploy_status_response['numOfBatches'] == 20 + assert deploy_status_response['progress']['value'] == 65 + diff --git a/tests/test_patch.py b/tests/test_patch.py index 61711b24..2408ab79 100644 --- a/tests/test_patch.py +++ b/tests/test_patch.py @@ -2,9 +2,12 @@ from unittest.mock import MagicMock import pytest +import base64 from senza.exceptions import InvalidUserDataType -from senza.patch import patch_auto_scaling_group +from senza.patch import patch_auto_scaling_group, patch_elastigroup +from senza.spotinst.components.elastigroup_api import SpotInstAccountData + def test_patch_auto_scaling_group(monkeypatch): @@ -29,6 +32,36 @@ def create_lc(**kwargs): assert new_lc['UserData'] == 'myuserdata' +def test_patch_elastigroup(monkeypatch): + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + new_lc = {} + + def create_lc(properties_to_patch, *args): + new_lc.update(properties_to_patch) + + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.patch_elastigroup', create_lc) + + properties = {'ImageId': 'mynewimage', 'InstanceType': 'mynewinstancetyoe', 'UserData': {'source': 'newsource'}} + group = {'compute': { + 'launchSpecification': { + 'userData': codecs.encode(b'#firstline\nsource: oldsource', 'base64').decode('utf-8'), + 'imageId': 'myoldimage' + }, + 'instanceTypes': { + 'ondemand': 'myoldinstancetyoe' + } + } + } + changed = patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data) + + assert changed + assert new_lc['ImageId'] == 'mynewimage' + assert new_lc['UserData'] == base64.urlsafe_b64encode('#firstline\nsource: newsource\n'.encode('utf-8')).decode('utf-8') + assert new_lc['InstanceType'] == 'mynewinstancetyoe' + + def test_patch_auto_scaling_group_taupage_config(monkeypatch): lc = {'ImageId': 'originalimage', 'LaunchConfigurationName': 'originallc', @@ -52,7 +85,6 @@ def create_lc(**kwargs): assert new_lc['UserData'] == '#firstline\nsource: newsource\n' - def test_patch_user_data_wrong_type(monkeypatch): lc = {'ImageId': 'originalimage', 'LaunchConfigurationName': 'originallc', @@ -76,3 +108,25 @@ def create_lc(**kwargs): assert str(exc_info.value) == ('Current user data is a map but provided ' 'user data is a string.') + + +def test_patch_user_data_wrong_type_elastigroup(monkeypatch): + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + properties = {'UserData': "it's a string"} + group = {'compute': { + 'launchSpecification': { + 'userData': codecs.encode(b'#firstline\nsource: oldsource', 'base64').decode('utf-8'), + 'imageId': 'myoldimage' + }, + 'instanceTypes': { + 'ondemand': 'myoldinstancetyoe' + } + } + } + with pytest.raises(InvalidUserDataType) as exc_info: + patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data) + + assert str(exc_info.value) == ('Current user data is a map but provided ' + 'user data is a string.') \ No newline at end of file diff --git a/tests/test_respawn.py b/tests/test_respawn.py index c20cb3c5..2ad8f286 100644 --- a/tests/test_respawn.py +++ b/tests/test_respawn.py @@ -1,6 +1,8 @@ from unittest.mock import MagicMock -from senza.respawn import respawn_auto_scaling_group +from senza.respawn import respawn_auto_scaling_group, respawn_elastigroup +from senza.spotinst.components.elastigroup_api import SpotInstAccountData + def test_respawn_auto_scaling_group(monkeypatch): @@ -23,6 +25,7 @@ def terminate_instance(InstanceId, **kwargs): elb = MagicMock() elb.describe_instance_health.return_value = {'InstanceStates': instance_states} services = {'autoscaling': asg, 'elb': elb} + def client(service, region): assert region == 'myregion' return services[service] @@ -60,3 +63,48 @@ def client(service, *args): monkeypatch.setattr('time.sleep', lambda s: s) respawn_auto_scaling_group('myasg', 'myregion') + +def test_respawn_elastigroup(monkeypatch): + elastigroup_id = 'sig-xfy' + stack_name = 'my-app-stack' + region = 'my-region' + batch_size = 35 + + spotinst_account = SpotInstAccountData('act-zwk', 'fake-token') + spotinst_account_mock = MagicMock() + spotinst_account_mock.return_value = spotinst_account + + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.get_spotinst_account_data', spotinst_account_mock) + + deploy_output = [{ + 'id': 'deploy-1' + }] + deploy_output_mock = MagicMock() + deploy_output_mock.return_value = deploy_output + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.deploy', deploy_output_mock) + + execution_data = { + 'percentage': 0, + 'runs': 0, + 'status': 'starting' + } + + def deploy_status(*args): + execution_data['runs'] += 1 + execution_data['percentage'] += 50 + if execution_data['percentage'] == 100: + execution_data['status'] = 'finished' + else: + execution_data['status'] = 'in_progress' + return [{ + 'id': args[0], + 'status': execution_data['status'], + 'progress': { + 'value': execution_data['percentage'] + } + }] + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.deploy_status', deploy_status) + respawn_elastigroup(elastigroup_id, stack_name, region, batch_size) + + assert execution_data['runs'] == 2 + assert execution_data['percentage'] == 100