From 98f6a4b4e8663686e435bf2355c5272c578ca441 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Wed, 18 Jul 2018 21:34:11 +0200 Subject: [PATCH 01/10] poc changes --- senza/cli.py | 99 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index 12026792..d337aeae 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -118,6 +118,12 @@ SENZA_KMS_PREFIX = 'senza:kms:' +ELASTIGROUP_TYPE = 'Custom::elastigroup' + +AUTO_SCALING_GROUP_TYPE = 'AWS::AutoScaling::AutoScalingGroup' + +VALID_AUTO_SCALING_GROUPS = [AUTO_SCALING_GROUP_TYPE, ELASTIGROUP_TYPE] + def filter_output_columns(output_columns, filter_columns): """ @@ -1426,8 +1432,24 @@ def dump(stack_ref, region, output): print_json(cfjson, output) +def get_auto_scaling_groups_and_elasti_groups(stack_refs, region): + cf = BotoClientProxy('cloudformation', region) + # TODO :: this calls get_stacks again. make the stacks a parameter instead + for stack in get_stacks(stack_refs, region): + resources = cf.describe_stack_resources(StackName=stack.StackName)['StackResources'] + + for resource in resources: + if resource['ResourceType'] in VALID_AUTO_SCALING_GROUPS: + # TODO :: uhhhhh.... A yield... + # TODO :: a) return a tuple/dict with type and asg_name and whatever else is needed + # TODO :: b) return a full dict with all the information. Maybe this yield is premature optimization. + # (I mean.... to prevent storing the whole list, we store (stack_refs, region, cf, output of get_stacks, resources and resource + yield {'type': resource['ResourceType'], 'resource_id': resource['PhysicalResourceId'], 'stack_name': resource['StackName']} + + def get_auto_scaling_groups(stack_refs, region): cf = BotoClientProxy('cloudformation', region) + # TODO :: this calls get_stacks again. make the stacks a parameter instead for stack in get_stacks(stack_refs, region): resources = cf.describe_stack_resources(StackName=stack.StackName)['StackResources'] @@ -1528,22 +1550,67 @@ def scale(stack_ref, region, desired_capacity, force): confirm_str = 'Number of stacks to be scaled - {}. Do you want to continue?'.format(stack_count) click.confirm(confirm_str, abort=True) - for asg_name in get_auto_scaling_groups(stack_refs, region): - group = get_auto_scaling_group(asg, asg_name) - current_capacity = group['DesiredCapacity'] - with Action('Scaling {} from {} to {} instances..'.format( - asg_name, current_capacity, desired_capacity)) as act: - if current_capacity == desired_capacity: - act.ok('NO CHANGES') - else: - kwargs = {} - if desired_capacity < group['MinSize']: - kwargs['MinSize'] = desired_capacity - if desired_capacity > group['MaxSize']: - kwargs['MaxSize'] = desired_capacity - asg.update_auto_scaling_group(AutoScalingGroupName=asg_name, - DesiredCapacity=desired_capacity, - **kwargs) + for group in get_auto_scaling_groups_and_elasti_groups(stack_refs, region): + if group['type'] == AUTO_SCALING_GROUP_TYPE: + scale_auto_scaling_group(asg, group['resource_id'], desired_capacity) + else: + if group['type'] == ELASTIGROUP_TYPE: + scale_elastigroup(group['resource_id'], group['stack_name'], desired_capacity, region) + + +def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): + + cf = boto3.client('cloudformation', region) + template = cf.get_template(StackName=stack_name)['TemplateBody'] + + spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] + spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + + headers = { + "Authorization": "Bearer {}".format(spotinst_token), + "Content-Type": "application/json" + } + + # TODO :: know the targets for min, max and current + # https://api.spotinst.io/aws/ec2/group/:GROUP_ID?accountId= + response = requests.get('https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + capacity = groups[0]['capacity'] + + # TODO :: if not equal, then we call the spotinst api to update elastigroup + # TODO :: pay attention to readjust the min and max like it's done for ASG + # https://api.spotinst.com/elastigroup/amazon-web-services/update/ + new_capacity = { + 'target': desired_capacity, + 'minimum': desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'], + 'maximum': desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] + } + + body = {'group': {'capacity': new_capacity}} + response = requests.put( + 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + + +def scale_auto_scaling_group(asg, asg_name, desired_capacity): + group = get_auto_scaling_group(asg, asg_name) + current_capacity = group['DesiredCapacity'] + with Action('Scaling {} from {} to {} instances..'.format( + asg_name, current_capacity, desired_capacity)) as act: + if current_capacity == desired_capacity: + act.ok('NO CHANGES') + else: + kwargs = {} + if desired_capacity < group['MinSize']: + kwargs['MinSize'] = desired_capacity + if desired_capacity > group['MaxSize']: + kwargs['MaxSize'] = desired_capacity + asg.update_auto_scaling_group(AutoScalingGroupName=asg_name, + DesiredCapacity=desired_capacity, + **kwargs) def failure_event(event: dict): From 97fbd41da9349660a1baed4717f8dab36527fef9 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Thu, 19 Jul 2018 11:08:18 +0200 Subject: [PATCH 02/10] Added validation test --- senza/cli.py | 35 ++++++++++----------- tests/test_cli.py | 79 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 92 insertions(+), 22 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index d337aeae..c3168e58 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -1553,9 +1553,8 @@ def scale(stack_ref, region, desired_capacity, force): for group in get_auto_scaling_groups_and_elasti_groups(stack_refs, region): if group['type'] == AUTO_SCALING_GROUP_TYPE: scale_auto_scaling_group(asg, group['resource_id'], desired_capacity) - else: - if group['type'] == ELASTIGROUP_TYPE: - scale_elastigroup(group['resource_id'], group['stack_name'], desired_capacity, region) + elif group['type'] == ELASTIGROUP_TYPE: + scale_elastigroup(group['resource_id'], group['stack_name'], desired_capacity, region) def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): @@ -1571,28 +1570,28 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): "Content-Type": "application/json" } - # TODO :: know the targets for min, max and current - # https://api.spotinst.io/aws/ec2/group/:GROUP_ID?accountId= response = requests.get('https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), headers=headers, timeout=5) response.raise_for_status() data = response.json() groups = data.get("response", {}).get("items", []) capacity = groups[0]['capacity'] - # TODO :: if not equal, then we call the spotinst api to update elastigroup - # TODO :: pay attention to readjust the min and max like it's done for ASG - # https://api.spotinst.com/elastigroup/amazon-web-services/update/ - new_capacity = { - 'target': desired_capacity, - 'minimum': desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'], - 'maximum': desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - } + with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( + stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: + if capacity['target'] == desired_capacity: + act.ok('NO CHANGES') + else: + new_capacity = { + 'target': desired_capacity, + 'minimum': desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'], + 'maximum': desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] + } - body = {'group': {'capacity': new_capacity}} - response = requests.put( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - headers=headers, timeout=10, data=json.dumps(body)) - response.raise_for_status() + body = {'group': {'capacity': new_capacity}} + response = requests.put( + 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() def scale_auto_scaling_group(asg, asg_name, desired_capacity): diff --git a/tests/test_cli.py b/tests/test_cli.py index e4f04943..4372faf6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,7 +5,6 @@ from contextlib import contextmanager from typing import List, Dict -from typing import Optional from unittest.mock import MagicMock, mock_open import botocore.exceptions @@ -13,6 +12,7 @@ import senza.traffic import yaml import base64 +import responses from click.testing import CliRunner from senza.aws import SenzaStackSummary from senza.cli import (KeyValParamType, StackReference, @@ -1645,7 +1645,8 @@ def test_scale(monkeypatch): 'CreationTime': '2016-06-14'}]} boto3.describe_stack_resources.return_value = {'StackResources': [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg'}]} + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} # NOTE: we are using invalid MinSize (< capacity) here to get one more line covered ;-) group = {'AutoScalingGroupName': 'myasg', 'DesiredCapacity': 1, 'MinSize': 3, 'MaxSize': 1} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} @@ -1655,6 +1656,76 @@ def test_scale(monkeypatch): catch_exceptions=False) assert 'Scaling myasg from 1 to 2 instances' in result.output + +def test_scale_elastigroup(monkeypatch): + spotinst_account_id = 'fakeactid' + elastigroup_id = 'myelasti' + boto3 = MagicMock() + boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', + 'CreationTime': '2016-06-14'}]} + boto3.describe_stack_resources.return_value = {'StackResources': + [{'ResourceType': 'Custom::elastigroup', + 'PhysicalResourceId': elastigroup_id, + 'StackName': 'myapp-1'}]} + boto3.get_template.return_value = { + 'TemplateBody': { + 'Mappings': { + 'Senza': { + 'Info': { + 'SpotinstAccessToken': 'faketoken' + } + } + }, + 'Resources': { + 'AppServerConfig': { + 'Properties': { + 'accountId': spotinst_account_id + } + } + } + } + } + + group = { + 'response': { + 'items': [{ + 'capacity': { + 'minimum': 1, + 'maximum': 2, + 'target': 1, + 'unit': 'instance' + } + }] + } + } + update = { + 'response': { + 'items': [{ + 'capacity': { + 'minimum': 1, + 'maximum': 3, + 'target': 3, + 'unit': 'instance' + } + }] + } + } + with responses.RequestsMock() as rsps: + rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + status=200, + json=group) + + rsps.add(rsps.PUT, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + status=200, + json=update) + + monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) + runner = CliRunner() + result = runner.invoke(cli, ['scale', 'myapp', '1', '3', '--region=aa-fakeregion-1'], + catch_exceptions=False) + assert 'Scaling ElastiGroup myapp-1 (ID: myelasti) from 1 to 3 instances' in result.output + + def test_scale_with_confirm(monkeypatch): boto3 = MagicMock() boto3.list_stacks.return_value = {'StackSummaries': [ @@ -1683,7 +1754,8 @@ def test_scale_with_force_confirm(monkeypatch): ]} boto3.describe_stack_resources.return_value = {'StackResources': [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg'}]} + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} group = {'AutoScalingGroupName': 'myasg', 'DesiredCapacity': 1, 'MinSize': 3, 'MaxSize': 1} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) @@ -1693,7 +1765,6 @@ def test_scale_with_force_confirm(monkeypatch): assert 'Scaling myasg from 1 to 2 instances' in result.output - def test_wait(monkeypatch): cf = MagicMock() stack1 = {'StackName': 'test-1', From 30f9c7e5d8a43a3e597ba58a6cfb1a59200ae5ac Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Thu, 19 Jul 2018 14:59:21 +0200 Subject: [PATCH 03/10] Code refactoring --- senza/aws.py | 7 ++- senza/cli.py | 48 ++++++++------------- spotinst/components/elastigroup_api.py | 57 +++++++++++++++++++++++++ tests/test_cli.py | 57 +++++++++++-------------- tests/test_elastigroup_api.py | 59 ++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 66 deletions(-) create mode 100644 spotinst/components/elastigroup_api.py create mode 100644 tests/test_elastigroup_api.py diff --git a/senza/aws.py b/senza/aws.py index a2be35de..037ad055 100644 --- a/senza/aws.py +++ b/senza/aws.py @@ -301,10 +301,13 @@ def get_stacks(stack_refs: list, # stack names that were already yielded to avoid yielding old deleted # stacks whose name was reused stacks_yielded = set() + output_stacks = [] for stack in stacks: if stack['StackName'] not in stacks_yielded or not unique_only: stacks_yielded.add(stack['StackName']) - yield SenzaStackSummary(stack) + output_stacks.append(SenzaStackSummary(stack)) + + return output_stacks def matches_any(cf_stack_name: str, stack_refs: list): @@ -461,7 +464,7 @@ def all_stacks_in_final_state(related_stacks_refs: list, region: str, timeout: O while not all_in_final_state and wait_timeout > datetime.datetime.utcnow(): # assume all stacks are ready all_in_final_state = True - related_stacks = list(get_stacks(related_stacks_refs, region)) + related_stacks = get_stacks(related_stacks_refs, region) if not related_stacks: error("Stack not found!") diff --git a/senza/cli.py b/senza/cli.py index c3168e58..99a68c6a 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -25,6 +25,7 @@ fatal_error, info, ok) from clickclick.console import print_table +import spotinst from .arguments import (GLOBAL_OPTIONS, json_output_option, output_option, parameter_file_option, region_option, stacktrace_visible_option, watch_option, @@ -1432,24 +1433,23 @@ def dump(stack_ref, region, output): print_json(cfjson, output) -def get_auto_scaling_groups_and_elasti_groups(stack_refs, region): +def get_auto_scaling_groups_and_elasti_groups(stacks, region): + ''' + Returns data for both AWS Auto Scaling Groups and ElastiGroups + + Note: This method will eventually replace get_auto_scaling_groups when the remaining commands support ElastiGroups + ''' cf = BotoClientProxy('cloudformation', region) - # TODO :: this calls get_stacks again. make the stacks a parameter instead - for stack in get_stacks(stack_refs, region): + for stack in stacks: resources = cf.describe_stack_resources(StackName=stack.StackName)['StackResources'] for resource in resources: if resource['ResourceType'] in VALID_AUTO_SCALING_GROUPS: - # TODO :: uhhhhh.... A yield... - # TODO :: a) return a tuple/dict with type and asg_name and whatever else is needed - # TODO :: b) return a full dict with all the information. Maybe this yield is premature optimization. - # (I mean.... to prevent storing the whole list, we store (stack_refs, region, cf, output of get_stacks, resources and resource yield {'type': resource['ResourceType'], 'resource_id': resource['PhysicalResourceId'], 'stack_name': resource['StackName']} def get_auto_scaling_groups(stack_refs, region): cf = BotoClientProxy('cloudformation', region) - # TODO :: this calls get_stacks again. make the stacks a parameter instead for stack in get_stacks(stack_refs, region): resources = cf.describe_stack_resources(StackName=stack.StackName)['StackResources'] @@ -1545,12 +1545,13 @@ def scale(stack_ref, region, desired_capacity, force): asg = BotoClientProxy('autoscaling', region) - stack_count = len(list(get_stacks(stack_refs, region))) + stacks = get_stacks(stack_refs, region) + stack_count = len(stacks) if not force and stack_count > 1: confirm_str = 'Number of stacks to be scaled - {}. Do you want to continue?'.format(stack_count) click.confirm(confirm_str, abort=True) - for group in get_auto_scaling_groups_and_elasti_groups(stack_refs, region): + for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): if group['type'] == AUTO_SCALING_GROUP_TYPE: scale_auto_scaling_group(asg, group['resource_id'], desired_capacity) elif group['type'] == ELASTIGROUP_TYPE: @@ -1565,33 +1566,18 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - response = requests.get('https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), headers=headers, timeout=5) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - capacity = groups[0]['capacity'] + group = spotinst.components.elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) + capacity = group['capacity'] with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: if capacity['target'] == desired_capacity: act.ok('NO CHANGES') else: - new_capacity = { - 'target': desired_capacity, - 'minimum': desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'], - 'maximum': desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - } + minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] + maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - body = {'group': {'capacity': new_capacity}} - response = requests.put( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - headers=headers, timeout=10, data=json.dumps(body)) - response.raise_for_status() + spotinst.components.elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, spotinst_account_id, spotinst_token) def scale_auto_scaling_group(asg, asg_name, desired_capacity): @@ -1652,7 +1638,7 @@ def wait(stack_ref, region, deletion, timeout, interval): stacks_in_progress = set() successful_actions = set() - stacks_found = list(get_stacks(stack_refs, region, all=True, unique_only=True)) + stacks_found = get_stacks(stack_refs, region, all=True, unique_only=True) if not stacks_found: raise click.UsageError('No matching stack for "{}" found'.format(' '.join(stack_ref))) diff --git a/spotinst/components/elastigroup_api.py b/spotinst/components/elastigroup_api.py new file mode 100644 index 00000000..8db0ffb5 --- /dev/null +++ b/spotinst/components/elastigroup_api.py @@ -0,0 +1,57 @@ +import requests +import json + + +SPOTINST_API_URL = 'https://api.spotinst.io' + + +def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_id, spotinst_token): + ''' + Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. + Returns the updated description of the ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_token), + "Content-Type": "application/json" + } + + new_capacity = { + 'target': target, + 'minimum': minimum, + 'maximum': maximum + } + + body = {'group': {'capacity': new_capacity}} + response = requests.put( + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups[0] + + +def get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token): + ''' + Returns the description of an ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_token), + "Content-Type": "application/json" + } + + response = requests.get( + 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 4372faf6..2120a4e3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,7 +12,6 @@ import senza.traffic import yaml import base64 -import responses from click.testing import CliRunner from senza.aws import SenzaStackSummary from senza.cli import (KeyValParamType, StackReference, @@ -1687,43 +1686,35 @@ def test_scale_elastigroup(monkeypatch): } group = { - 'response': { - 'items': [{ - 'capacity': { - 'minimum': 1, - 'maximum': 2, - 'target': 1, - 'unit': 'instance' - } - }] + 'capacity': { + 'minimum': 1, + 'maximum': 2, + 'target': 1, + 'unit': 'instance' } } + get_elastigroup = MagicMock() + get_elastigroup.return_value = group + update = { - 'response': { - 'items': [{ - 'capacity': { - 'minimum': 1, - 'maximum': 3, - 'target': 3, - 'unit': 'instance' - } - }] + 'capacity': { + 'minimum': 1, + 'maximum': 3, + 'target': 3, + 'unit': 'instance' } } - with responses.RequestsMock() as rsps: - rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - status=200, - json=group) - - rsps.add(rsps.PUT, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - status=200, - json=update) - - monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) - runner = CliRunner() - result = runner.invoke(cli, ['scale', 'myapp', '1', '3', '--region=aa-fakeregion-1'], - catch_exceptions=False) - assert 'Scaling ElastiGroup myapp-1 (ID: myelasti) from 1 to 3 instances' in result.output + update_capacity = MagicMock() + update_capacity.return_value = update + + monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) + monkeypatch.setattr('spotinst.components.elastigroup_api.get_elastigroup', get_elastigroup) + monkeypatch.setattr('spotinst.components.elastigroup_api.update_capacity', update_capacity) + + runner = CliRunner() + result = runner.invoke(cli, ['scale', 'myapp', '1', '3', '--region=aa-fakeregion-1'], + catch_exceptions=False) + assert 'Scaling ElastiGroup myapp-1 (ID: myelasti) from 1 to 3 instances' in result.output def test_scale_with_confirm(monkeypatch): diff --git a/tests/test_elastigroup_api.py b/tests/test_elastigroup_api.py new file mode 100644 index 00000000..39f92a48 --- /dev/null +++ b/tests/test_elastigroup_api.py @@ -0,0 +1,59 @@ +import responses +from spotinst.components.elastigroup_api import update_capacity, get_elastigroup, SPOTINST_API_URL + + +def test_update_capacity(monkeypatch): + update = { + 'response': { + 'items': [{ + 'id': 'sig-xfy', + 'name': 'my-app-1', + 'capacity': { + 'minimum': 1, + 'maximum': 3, + 'target': 3, + 'unit': 'instance' + } + }] + } + } + + elastigroup_id = 'sig-xfy' + spotinst_account_id = 'act-zwk' + spotinst_token = 'fake-token' + with responses.RequestsMock() as rsps: + rsps.add(rsps.PUT, + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), + status=200, + json=update) + + update_response = update_capacity(1, 3, 3, elastigroup_id, spotinst_account_id, spotinst_token) + assert update_response['id'] == elastigroup_id + assert update_response['name'] == 'my-app-1' + assert update_response['capacity']['minimum'] == 1 + assert update_response['capacity']['maximum'] == 3 + assert update_response['capacity']['target'] == 3 + + +def test_get_elastigroup(monkeypatch): + group = { + 'response': { + 'items': [{ + 'id': 'sig-xfy', + 'name': 'my-app-1', + 'region': 'eu-central-1', + }] + } + } + + elastigroup_id = 'sig-xfy' + spotinst_account_id = 'act-zwk' + spotinst_token = 'fake-token' + with responses.RequestsMock() as rsps: + rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + status=200, + json=group) + + group = get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) + assert group['id'] == elastigroup_id + assert group['name'] == 'my-app-1' From 329d74541b90e9752e0a71136a6aca8bfd8aa232 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Thu, 19 Jul 2018 15:09:17 +0200 Subject: [PATCH 04/10] Codacy issues --- senza/cli.py | 18 +++++++++++++++--- spotinst/components/elastigroup_api.py | 3 +++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index 99a68c6a..589fcf9a 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -1445,7 +1445,9 @@ def get_auto_scaling_groups_and_elasti_groups(stacks, region): for resource in resources: if resource['ResourceType'] in VALID_AUTO_SCALING_GROUPS: - yield {'type': resource['ResourceType'], 'resource_id': resource['PhysicalResourceId'], 'stack_name': resource['StackName']} + yield {'type': resource['ResourceType'], + 'resource_id': resource['PhysicalResourceId'], + 'stack_name': resource['StackName']} def get_auto_scaling_groups(stack_refs, region): @@ -1559,7 +1561,9 @@ def scale(stack_ref, region, desired_capacity, force): def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): - + ''' + Commands to scale an ElastiGroup + ''' cf = boto3.client('cloudformation', region) template = cf.get_template(StackName=stack_name)['TemplateBody'] @@ -1577,10 +1581,18 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - spotinst.components.elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, spotinst_account_id, spotinst_token) + spotinst.components.elastigroup_api.update_capacity(minimum, + maximum, + desired_capacity, + elastigroup_id, + spotinst_account_id, + spotinst_token) def scale_auto_scaling_group(asg, asg_name, desired_capacity): + ''' + Commands to scale an AWS Auto Scaling Group + ''' group = get_auto_scaling_group(asg, asg_name) current_capacity = group['DesiredCapacity'] with Action('Scaling {} from {} to {} instances..'.format( diff --git a/spotinst/components/elastigroup_api.py b/spotinst/components/elastigroup_api.py index 8db0ffb5..063a251d 100644 --- a/spotinst/components/elastigroup_api.py +++ b/spotinst/components/elastigroup_api.py @@ -1,3 +1,6 @@ +''' +Wrapper methods for ElastiGroup's API +''' import requests import json From 90b912a60fa40f21526ebb1f97d0ae6aa546d064 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Thu, 19 Jul 2018 16:08:22 +0200 Subject: [PATCH 05/10] solved import issue --- senza/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index 589fcf9a..ec237240 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -25,7 +25,7 @@ fatal_error, info, ok) from clickclick.console import print_table -import spotinst +from spotinst.components import elastigroup_api from .arguments import (GLOBAL_OPTIONS, json_output_option, output_option, parameter_file_option, region_option, stacktrace_visible_option, watch_option, @@ -1570,7 +1570,7 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] - group = spotinst.components.elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) + group = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) capacity = group['capacity'] with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( @@ -1581,12 +1581,12 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - spotinst.components.elastigroup_api.update_capacity(minimum, - maximum, - desired_capacity, - elastigroup_id, - spotinst_account_id, - spotinst_token) + elastigroup_api.update_capacity(minimum, + maximum, + desired_capacity, + elastigroup_id, + spotinst_account_id, + spotinst_token) def scale_auto_scaling_group(asg, asg_name, desired_capacity): From 6f3b09a80e206fa214739f37d118534bdd5da900 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Mon, 23 Jul 2018 17:54:08 +0200 Subject: [PATCH 06/10] refactor and tests --- senza/cli.py | 95 ++++++++++------ senza/patch.py | 55 +++++++++- {spotinst => senza/spotinst}/__init__.py | 0 .../spotinst}/components/__init__.py | 0 .../spotinst}/components/elastigroup.py | 2 +- senza/spotinst/components/elastigroup_api.py | 103 ++++++++++++++++++ spotinst/components/elastigroup_api.py | 60 ---------- tests/test_cli.py | 15 +-- tests/test_elastigroup.py | 14 +-- tests/test_elastigroup_api.py | 54 +++++++-- tests/test_patch.py | 58 +++++++++- 11 files changed, 333 insertions(+), 123 deletions(-) rename {spotinst => senza/spotinst}/__init__.py (100%) rename {spotinst => senza/spotinst}/components/__init__.py (100%) rename {spotinst => senza/spotinst}/components/elastigroup.py (99%) create mode 100644 senza/spotinst/components/elastigroup_api.py delete mode 100644 spotinst/components/elastigroup_api.py diff --git a/senza/cli.py b/senza/cli.py index ec237240..d1f2fb2c 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -25,7 +25,7 @@ fatal_error, info, ok) from clickclick.console import print_table -from spotinst.components import elastigroup_api +from .spotinst.components import elastigroup_api from .arguments import (GLOBAL_OPTIONS, json_output_option, output_option, parameter_file_option, region_option, stacktrace_visible_option, watch_option, @@ -42,7 +42,7 @@ from .manaus.exceptions import VPCError from .manaus.route53 import Route53, Route53Record from .manaus.utils import extract_client_error_code -from .patch import patch_auto_scaling_group +from .patch import patch_auto_scaling_group, patch_elastigroup from .respawn import get_auto_scaling_group, respawn_auto_scaling_group from .stups.piu import Piu from .subcommands.config import cmd_config @@ -1476,7 +1476,7 @@ def get_auto_scaling_groups(stack_refs, region): def patch(stack_ref, region, image, instance_type, user_data): '''Patch specific properties of existing stack. - Currently only supports patching ASG launch configurations.''' + Currently supports patching ASG launch configurations and ElastiGroup groups.''' stack_refs = get_stack_refs(stack_ref) region = get_region(region) @@ -1499,13 +1499,52 @@ def patch(stack_ref, region, image, instance_type, user_data): asg = BotoClientProxy('autoscaling', region) - for asg_name in get_auto_scaling_groups(stack_refs, region): - with Action('Patching Auto Scaling Group {}..'.format(asg_name)) as act: - result = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) - groups = result['AutoScalingGroups'] - for group in groups: - if not patch_auto_scaling_group(group, region, properties): - act.ok('NO CHANGES') + stacks = get_stacks(stack_refs, region) + for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): + if group['type'] == ELASTIGROUP_TYPE: + patch_spotinst_elastigroup(properties, group['resource_id'], region, group['stack_name']) + elif group['type'] == AUTO_SCALING_GROUP_TYPE: + patch_aws_asg(properties, region, asg, group['resource_id']) + + +def patch_aws_asg(properties, region, asg, asg_name): + ''' + Patch an AWS Auto Scaling Group + ''' + with Action('Patching Auto Scaling Group {}..'.format(asg_name)) as act: + result = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + groups = result['AutoScalingGroups'] + for group in groups: + if not patch_auto_scaling_group(group, region, properties): + act.ok('NO CHANGES') + + +def get_spotinst_account_data(region, stack_name): + ''' + Extracts required parameters required to access SpotInst API + ''' + cf = boto3.client('cloudformation', region) + template = cf.get_template(StackName=stack_name)['TemplateBody'] + + spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] + spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + + return elastigroup_api.SpotInstAccountData(spotinst_account_id, spotinst_token) + + +def patch_spotinst_elastigroup(properties, elastigroup_id, region, stack_name): + ''' + Patch specific properties of an existing ElastiGroup + ''' + + spotinst_account_data = get_spotinst_account_data(region, stack_name) + + with Action('Patching ElastiGroup {} (ID: {})..'.format(stack_name, elastigroup_id)) as act: + groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) + + for group in groups: + if not patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data): + act.ok('NO CHANGES') @cli.command('respawn-instances') @@ -1545,14 +1584,13 @@ def scale(stack_ref, region, desired_capacity, force): region = get_region(region) check_credentials(region) - asg = BotoClientProxy('autoscaling', region) - stacks = get_stacks(stack_refs, region) stack_count = len(stacks) if not force and stack_count > 1: confirm_str = 'Number of stacks to be scaled - {}. Do you want to continue?'.format(stack_count) click.confirm(confirm_str, abort=True) + asg = BotoClientProxy('autoscaling', region) for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): if group['type'] == AUTO_SCALING_GROUP_TYPE: scale_auto_scaling_group(asg, group['resource_id'], desired_capacity) @@ -1564,29 +1602,22 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): ''' Commands to scale an ElastiGroup ''' - cf = boto3.client('cloudformation', region) - template = cf.get_template(StackName=stack_name)['TemplateBody'] + spotinst_account_data = get_spotinst_account_data(region, stack_name) - spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] - spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) - group = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) - capacity = group['capacity'] + for group in groups: + capacity = group['capacity'] - with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( - stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: - if capacity['target'] == desired_capacity: - act.ok('NO CHANGES') - else: - minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] - maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - - elastigroup_api.update_capacity(minimum, - maximum, - desired_capacity, - elastigroup_id, - spotinst_account_id, - spotinst_token) + with Action('Scaling ElastiGroup {} (ID: {}) from {} to {} instances..'.format( + stack_name, elastigroup_id, capacity['target'], desired_capacity)) as act: + if capacity['target'] == desired_capacity: + act.ok('NO CHANGES') + else: + minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] + maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] + + elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, spotinst_account_data) def scale_auto_scaling_group(asg, asg_name, desired_capacity): diff --git a/senza/patch.py b/senza/patch.py index 920dc1d1..fc1bb15b 100644 --- a/senza/patch.py +++ b/senza/patch.py @@ -1,9 +1,11 @@ import codecs +import base64 import datetime import yaml +from .spotinst.components import elastigroup_api from .exceptions import InvalidUserDataType from .manaus.boto_proxy import BotoClientProxy @@ -29,6 +31,19 @@ ]) +def should_patch_user_data(new_val, old_val): + ''' + Validate if User Data should be patched. + ''' + current_user_data = yaml.safe_load(old_val) + if isinstance(new_val, dict): + return True + elif isinstance(current_user_data, dict): + raise InvalidUserDataType(type(current_user_data), + type(new_val)) + return False + + def patch_user_data(old: str, new: dict): first_line, sep, data = old.partition('\n') data = yaml.safe_load(data) @@ -60,12 +75,8 @@ def patch_auto_scaling_group(group: dict, region: str, properties: dict): for key, val in properties.items(): if key == 'UserData': - current_user_data = yaml.safe_load(kwargs['UserData']) - if isinstance(val, dict): + if should_patch_user_data(val, kwargs['UserData']): kwargs[key] = patch_user_data(kwargs[key], val) - elif isinstance(current_user_data, dict): - raise InvalidUserDataType(type(current_user_data), - type(val)) else: kwargs[key] = val asg.create_launch_configuration(**kwargs) @@ -73,3 +84,37 @@ def patch_auto_scaling_group(group: dict, region: str, properties: dict): LaunchConfigurationName=kwargs['LaunchConfigurationName']) changed = True return changed + + +def patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data): + ''' + Patch specific properties of an existing ElastiGroup + ''' + changed = False + properties_to_patch = {} + + group_user_data = group['compute']['launchSpecification']['userData'] + current_user_data = codecs.decode(group_user_data.encode('utf-8'), 'base64').decode('utf-8') + + current_properties = { + 'ImageId': group['compute']['launchSpecification']['imageId'], + 'InstanceType': group['compute']['instanceTypes']['ondemand'], + 'UserData': current_user_data + } + + for key, val in properties.items(): + if key in current_properties: + if key == 'UserData': + if should_patch_user_data(val, current_properties[key]): + patched_user_data = patch_user_data(current_properties[key], val) + encrypted_user_data = base64.urlsafe_b64encode(patched_user_data.encode('utf-8')).decode('utf-8') + properties_to_patch[key] = encrypted_user_data + else: + if current_properties[key] != val: + properties_to_patch[key] = val + + if len(properties_to_patch) > 0: + elastigroup_api.patch_elastigroup(properties_to_patch, elastigroup_id, spotinst_account_data) + changed = True + + return changed diff --git a/spotinst/__init__.py b/senza/spotinst/__init__.py similarity index 100% rename from spotinst/__init__.py rename to senza/spotinst/__init__.py diff --git a/spotinst/components/__init__.py b/senza/spotinst/components/__init__.py similarity index 100% rename from spotinst/components/__init__.py rename to senza/spotinst/components/__init__.py diff --git a/spotinst/components/elastigroup.py b/senza/spotinst/components/elastigroup.py similarity index 99% rename from spotinst/components/elastigroup.py rename to senza/spotinst/components/elastigroup.py index 8280ed4a..798dca1b 100644 --- a/spotinst/components/elastigroup.py +++ b/senza/spotinst/components/elastigroup.py @@ -16,7 +16,7 @@ from senza.components.taupage_auto_scaling_group import check_application_id, check_application_version, \ check_docker_image_exists, generate_user_data from senza.utils import ensure_keys -from spotinst import MissingSpotinstAccount +from senza.spotinst import MissingSpotinstAccount SPOTINST_LAMBDA_FORMATION_ARN = 'arn:aws:lambda:{}:178579023202:function:spotinst-cloudformation' SPOTINST_API_URL = 'https://api.spotinst.io' diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py new file mode 100644 index 00000000..80f42781 --- /dev/null +++ b/senza/spotinst/components/elastigroup_api.py @@ -0,0 +1,103 @@ +''' +Wrapper methods for ElastiGroup's API +''' +import requests +import json + + +SPOTINST_API_URL = 'https://api.spotinst.io' + + +class SpotInstAccountData: + ''' + Data required to access SpotInst API + ''' + def __init__(self, account_id, access_token): + self.account_id = account_id + self.access_token = access_token + + +def update_elastigroup(body, elastigroup_id, spotinst_account_data): + ''' + Performs the update ElastiGroup API call. + + Note: Although this should only return one element in the list, + it still returns the entire list to prevent some silent decision making + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.put( + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups + + +def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_data): + ''' + Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. + Returns the updated description of the ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + ''' + + new_capacity = { + 'target': target, + 'minimum': minimum, + 'maximum': maximum + } + + body = {'group': {'capacity': new_capacity}} + + return update_elastigroup(body, elastigroup_id, spotinst_account_data) + + +def get_elastigroup(elastigroup_id, spotinst_account_data): + ''' + Returns a list containing the description of an ElastiGroup as a dict. + Exceptions will be thrown for HTTP errors. + + Note: Although this should only return one element in the list, + it still returns the entire list to prevent some silent decision making + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.get( + 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), + headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + groups = data.get("response", {}).get("items", []) + + return groups + + +def patch_elastigroup(properties, elastigroup_id, spotinst_account_data): + ''' + Patch specific properties of the ElastiGroup. + ''' + body = {'compute': {}} + if 'InstanceType' in properties: + body['compute']['instanceTypes'] = { + 'ondemand': properties['InstanceType'], + } + + if 'ImageId' in properties: + body['compute'].setdefault('launchSpecification', {})['imageId'] = properties['ImageId'] + + if 'userData' in properties: + body['compute'].setdefault('launchSpecification', {})['userData'] = properties['UserData'] + + return update_elastigroup(body, elastigroup_id, spotinst_account_data) diff --git a/spotinst/components/elastigroup_api.py b/spotinst/components/elastigroup_api.py deleted file mode 100644 index 063a251d..00000000 --- a/spotinst/components/elastigroup_api.py +++ /dev/null @@ -1,60 +0,0 @@ -''' -Wrapper methods for ElastiGroup's API -''' -import requests -import json - - -SPOTINST_API_URL = 'https://api.spotinst.io' - - -def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. - Returns the updated description of the ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - new_capacity = { - 'target': target, - 'minimum': minimum, - 'maximum': maximum - } - - body = {'group': {'capacity': new_capacity}} - response = requests.put( - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), - headers=headers, timeout=10, data=json.dumps(body)) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] - - -def get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Returns the description of an ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - response = requests.get( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - headers=headers, timeout=5) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 2120a4e3..4ac925b5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1606,7 +1606,8 @@ def test_patch(monkeypatch): 'CreationTime': '2016-06-14'}]} boto3.describe_stack_resources.return_value = {'StackResources': [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg'}]} + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} group = {'AutoScalingGroupName': 'myasg'} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} image = MagicMock() @@ -1685,31 +1686,31 @@ def test_scale_elastigroup(monkeypatch): } } - group = { + group = [{ 'capacity': { 'minimum': 1, 'maximum': 2, 'target': 1, 'unit': 'instance' } - } + }] get_elastigroup = MagicMock() get_elastigroup.return_value = group - update = { + update = [{ 'capacity': { 'minimum': 1, 'maximum': 3, 'target': 3, 'unit': 'instance' } - } + }] update_capacity = MagicMock() update_capacity.return_value = update monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) - monkeypatch.setattr('spotinst.components.elastigroup_api.get_elastigroup', get_elastigroup) - monkeypatch.setattr('spotinst.components.elastigroup_api.update_capacity', update_capacity) + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.get_elastigroup', get_elastigroup) + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.update_capacity', update_capacity) runner = CliRunner() result = runner.invoke(cli, ['scale', 'myapp', '1', '3', '--region=aa-fakeregion-1'], diff --git a/tests/test_elastigroup.py b/tests/test_elastigroup.py index 022c0a83..d5e3c0d7 100644 --- a/tests/test_elastigroup.py +++ b/tests/test_elastigroup.py @@ -4,12 +4,12 @@ from mock import MagicMock from senza.definitions import AccountArguments -from spotinst import MissingSpotinstAccount -from spotinst.components.elastigroup import component_elastigroup, ELASTIGROUP_DEFAULT_PRODUCT, \ - ELASTIGROUP_DEFAULT_STRATEGY, resolve_account_id, SPOTINST_API_URL, extract_block_mappings, \ - extract_auto_scaling_rules, ensure_instance_monitoring, ensure_default_strategy, extract_autoscaling_capacity, \ - ensure_default_product, fill_standard_tags, extract_subnets, extract_load_balancer_name, extract_public_ips, \ - extract_image_id, extract_security_group_ids, extract_instance_types, extract_instance_profile +from senza.spotinst import MissingSpotinstAccount +from senza.spotinst.components.elastigroup import (component_elastigroup, ELASTIGROUP_DEFAULT_PRODUCT, + ELASTIGROUP_DEFAULT_STRATEGY, resolve_account_id, SPOTINST_API_URL, extract_block_mappings, + extract_auto_scaling_rules, ensure_instance_monitoring, ensure_default_strategy, extract_autoscaling_capacity, + ensure_default_product, fill_standard_tags, extract_subnets, extract_load_balancer_name, extract_public_ips, + extract_image_id, extract_security_group_ids, extract_instance_types, extract_instance_profile) def test_component_elastigroup_defaults(monkeypatch): @@ -37,7 +37,7 @@ def test_component_elastigroup_defaults(monkeypatch): resolve_account_id = MagicMock() resolve_account_id.return_value = 'act-12345abcdef' - monkeypatch.setattr('spotinst.components.elastigroup.resolve_account_id', resolve_account_id) + monkeypatch.setattr('senza.spotinst.components.elastigroup.resolve_account_id', resolve_account_id) result = component_elastigroup(definition, configuration, args, info, False, AccountArguments('reg1')) diff --git a/tests/test_elastigroup_api.py b/tests/test_elastigroup_api.py index 39f92a48..91283a0f 100644 --- a/tests/test_elastigroup_api.py +++ b/tests/test_elastigroup_api.py @@ -1,5 +1,5 @@ import responses -from spotinst.components.elastigroup_api import update_capacity, get_elastigroup, SPOTINST_API_URL +from senza.spotinst.components.elastigroup_api import update_capacity, get_elastigroup, patch_elastigroup, SPOTINST_API_URL, SpotInstAccountData def test_update_capacity(monkeypatch): @@ -19,15 +19,14 @@ def test_update_capacity(monkeypatch): } elastigroup_id = 'sig-xfy' - spotinst_account_id = 'act-zwk' - spotinst_token = 'fake-token' + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: rsps.add(rsps.PUT, - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), status=200, json=update) - update_response = update_capacity(1, 3, 3, elastigroup_id, spotinst_account_id, spotinst_token) + update_response = update_capacity(1, 3, 3, elastigroup_id, spotinst_account_data)[0] assert update_response['id'] == elastigroup_id assert update_response['name'] == 'my-app-1' assert update_response['capacity']['minimum'] == 1 @@ -47,13 +46,50 @@ def test_get_elastigroup(monkeypatch): } elastigroup_id = 'sig-xfy' - spotinst_account_id = 'act-zwk' - spotinst_token = 'fake-token' + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: - rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), + rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), status=200, json=group) - group = get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token) + group = get_elastigroup(elastigroup_id, spotinst_account_data)[0] assert group['id'] == elastigroup_id assert group['name'] == 'my-app-1' + + +def test_patch_elastigroup(monkeypatch): + patch = { + 'ImageId': 'image-foo', + 'InstanceType': 'm1.micro', + 'UserData': 'user-data-value' + } + + update_response = { + 'response': { + 'items': [{ + 'compute': { + 'instanceTypes': { + 'ondemand': 'm1.micro', + 'spot': [ + 'm1.micro' + ] + }, + 'launchSpecification': { + 'imageId': 'image-foo', + 'userData': 'user-data-value' + } + } + }] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + rsps.add(rsps.PUT, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), + status=200, + json=update_response) + + patch_response = patch_elastigroup(patch, elastigroup_id, spotinst_account_data)[0] + assert patch_response['compute']['launchSpecification']['imageId'] == 'image-foo' + assert patch_response['compute']['instanceTypes']['ondemand'] == 'm1.micro' + assert patch_response['compute']['launchSpecification']['userData'] == 'user-data-value' diff --git a/tests/test_patch.py b/tests/test_patch.py index 61711b24..2408ab79 100644 --- a/tests/test_patch.py +++ b/tests/test_patch.py @@ -2,9 +2,12 @@ from unittest.mock import MagicMock import pytest +import base64 from senza.exceptions import InvalidUserDataType -from senza.patch import patch_auto_scaling_group +from senza.patch import patch_auto_scaling_group, patch_elastigroup +from senza.spotinst.components.elastigroup_api import SpotInstAccountData + def test_patch_auto_scaling_group(monkeypatch): @@ -29,6 +32,36 @@ def create_lc(**kwargs): assert new_lc['UserData'] == 'myuserdata' +def test_patch_elastigroup(monkeypatch): + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + new_lc = {} + + def create_lc(properties_to_patch, *args): + new_lc.update(properties_to_patch) + + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.patch_elastigroup', create_lc) + + properties = {'ImageId': 'mynewimage', 'InstanceType': 'mynewinstancetyoe', 'UserData': {'source': 'newsource'}} + group = {'compute': { + 'launchSpecification': { + 'userData': codecs.encode(b'#firstline\nsource: oldsource', 'base64').decode('utf-8'), + 'imageId': 'myoldimage' + }, + 'instanceTypes': { + 'ondemand': 'myoldinstancetyoe' + } + } + } + changed = patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data) + + assert changed + assert new_lc['ImageId'] == 'mynewimage' + assert new_lc['UserData'] == base64.urlsafe_b64encode('#firstline\nsource: newsource\n'.encode('utf-8')).decode('utf-8') + assert new_lc['InstanceType'] == 'mynewinstancetyoe' + + def test_patch_auto_scaling_group_taupage_config(monkeypatch): lc = {'ImageId': 'originalimage', 'LaunchConfigurationName': 'originallc', @@ -52,7 +85,6 @@ def create_lc(**kwargs): assert new_lc['UserData'] == '#firstline\nsource: newsource\n' - def test_patch_user_data_wrong_type(monkeypatch): lc = {'ImageId': 'originalimage', 'LaunchConfigurationName': 'originallc', @@ -76,3 +108,25 @@ def create_lc(**kwargs): assert str(exc_info.value) == ('Current user data is a map but provided ' 'user data is a string.') + + +def test_patch_user_data_wrong_type_elastigroup(monkeypatch): + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + properties = {'UserData': "it's a string"} + group = {'compute': { + 'launchSpecification': { + 'userData': codecs.encode(b'#firstline\nsource: oldsource', 'base64').decode('utf-8'), + 'imageId': 'myoldimage' + }, + 'instanceTypes': { + 'ondemand': 'myoldinstancetyoe' + } + } + } + with pytest.raises(InvalidUserDataType) as exc_info: + patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data) + + assert str(exc_info.value) == ('Current user data is a map but provided ' + 'user data is a string.') \ No newline at end of file From 1479c22a4ca45b405f20edc94ec3c2934b176fe2 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Wed, 25 Jul 2018 16:14:05 +0200 Subject: [PATCH 07/10] fixes --- senza/patch.py | 4 +- senza/spotinst/components/elastigroup_api.py | 11 ++-- spotinst/components/elastigroup_api.py | 60 -------------------- tests/test_elastigroup.py | 3 +- 4 files changed, 9 insertions(+), 69 deletions(-) delete mode 100644 spotinst/components/elastigroup_api.py diff --git a/senza/patch.py b/senza/patch.py index fc1bb15b..3989f29f 100644 --- a/senza/patch.py +++ b/senza/patch.py @@ -107,8 +107,8 @@ def patch_elastigroup(group, properties, elastigroup_id, spotinst_account_data): if key == 'UserData': if should_patch_user_data(val, current_properties[key]): patched_user_data = patch_user_data(current_properties[key], val) - encrypted_user_data = base64.urlsafe_b64encode(patched_user_data.encode('utf-8')).decode('utf-8') - properties_to_patch[key] = encrypted_user_data + encoded_user_data = base64.urlsafe_b64encode(patched_user_data.encode('utf-8')).decode('utf-8') + properties_to_patch[key] = encoded_user_data else: if current_properties[key] != val: properties_to_patch[key] = val diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py index 80f42781..5ef726b8 100644 --- a/senza/spotinst/components/elastigroup_api.py +++ b/senza/spotinst/components/elastigroup_api.py @@ -88,16 +88,17 @@ def patch_elastigroup(properties, elastigroup_id, spotinst_account_data): ''' Patch specific properties of the ElastiGroup. ''' - body = {'compute': {}} + compute = {} if 'InstanceType' in properties: - body['compute']['instanceTypes'] = { + compute['instanceTypes'] = { 'ondemand': properties['InstanceType'], } if 'ImageId' in properties: - body['compute'].setdefault('launchSpecification', {})['imageId'] = properties['ImageId'] + compute.setdefault('launchSpecification', {})['imageId'] = properties['ImageId'] - if 'userData' in properties: - body['compute'].setdefault('launchSpecification', {})['userData'] = properties['UserData'] + if 'UserData' in properties: + compute.setdefault('launchSpecification', {})['userData'] = properties['UserData'] + body = {'group': {'compute': compute}} return update_elastigroup(body, elastigroup_id, spotinst_account_data) diff --git a/spotinst/components/elastigroup_api.py b/spotinst/components/elastigroup_api.py deleted file mode 100644 index 063a251d..00000000 --- a/spotinst/components/elastigroup_api.py +++ /dev/null @@ -1,60 +0,0 @@ -''' -Wrapper methods for ElastiGroup's API -''' -import requests -import json - - -SPOTINST_API_URL = 'https://api.spotinst.io' - - -def update_capacity(minimum, maximum, target, elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Updates the capacity (number of instances) for an ElastiGroup by calling the SpotInst API. - Returns the updated description of the ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/update/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - new_capacity = { - 'target': target, - 'minimum': minimum, - 'maximum': maximum - } - - body = {'group': {'capacity': new_capacity}} - response = requests.put( - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_id), - headers=headers, timeout=10, data=json.dumps(body)) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] - - -def get_elastigroup(elastigroup_id, spotinst_account_id, spotinst_token): - ''' - Returns the description of an ElastiGroup as a dict. - Exceptions will be thrown for HTTP errors. - - For more details see https://api.spotinst.com/elastigroup/amazon-web-services/list-group/ - ''' - headers = { - "Authorization": "Bearer {}".format(spotinst_token), - "Content-Type": "application/json" - } - - response = requests.get( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_id), - headers=headers, timeout=5) - response.raise_for_status() - data = response.json() - groups = data.get("response", {}).get("items", []) - - return groups[0] diff --git a/tests/test_elastigroup.py b/tests/test_elastigroup.py index fb738265..c70857b2 100644 --- a/tests/test_elastigroup.py +++ b/tests/test_elastigroup.py @@ -3,7 +3,6 @@ import responses from mock import MagicMock -from senza.definitions import AccountArguments from senza.spotinst import MissingSpotinstAccount from senza.spotinst.components.elastigroup import (component_elastigroup, ELASTIGROUP_DEFAULT_PRODUCT, ELASTIGROUP_DEFAULT_STRATEGY, resolve_account_id, SPOTINST_API_URL, extract_block_mappings, @@ -37,7 +36,7 @@ def test_component_elastigroup_defaults(monkeypatch): mock_resolve_account_id = MagicMock() mock_resolve_account_id.return_value = 'act-12345abcdef' - monkeypatch.setattr('spotinst.components.elastigroup.resolve_account_id', mock_resolve_account_id) + monkeypatch.setattr('senza.spotinst.components.elastigroup.resolve_account_id', mock_resolve_account_id) mock_account_info = MagicMock() mock_account_info.Region = "reg1" From 909764e98aa3111120246ea936ec35f0d45b9eaa Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Fri, 27 Jul 2018 22:26:55 +0200 Subject: [PATCH 08/10] changes to cli and elastigroup_api --- senza/cli.py | 23 +++++++--- senza/respawn.py | 17 ++++++++ senza/spotinst/components/elastigroup_api.py | 34 ++++++++++++++- tests/test_cli.py | 34 ++++++++++++--- tests/test_elastigroup_api.py | 46 ++++++++++++++++++-- 5 files changed, 137 insertions(+), 17 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index d1f2fb2c..55c2a49e 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -43,7 +43,7 @@ from .manaus.route53 import Route53, Route53Record from .manaus.utils import extract_client_error_code from .patch import patch_auto_scaling_group, patch_elastigroup -from .respawn import get_auto_scaling_group, respawn_auto_scaling_group +from .respawn import get_auto_scaling_group, respawn_auto_scaling_group, respawn_elastigroup from .stups.piu import Piu from .subcommands.config import cmd_config from .subcommands.root import cli @@ -1550,13 +1550,17 @@ def patch_spotinst_elastigroup(properties, elastigroup_id, region, stack_name): @cli.command('respawn-instances') @click.argument('stack_ref', nargs=-1) @click.option('--inplace', - is_flag=True, help='Perform inplace update, do not scale out') + is_flag=True, help='Perform inplace update, do not scale out. Ignored for ElastiGroups.') @click.option('-f', '--force', is_flag=True, - help='Force respawn even if Launch Configuration is unchanged') + help='Force respawn even if Launch Configuration is unchanged. Ignored for ElastiGroups.') +@click.option('--batch_size_percentage', + metavar='PERCENTAGE', + help='Percentage (int value) of the ElastiGroup cluster that is respawned in each step.' + ' Valid only for ElastiGroups. The default value for this of 20.') @region_option @stacktrace_visible_option -def respawn_instances(stack_ref, inplace, force, region): +def respawn_instances(stack_ref, inplace, force, batch_size_percentage, region): '''Replace all EC2 instances in Auto Scaling Group(s) Performs a rolling update to prevent downtimes.''' @@ -1565,8 +1569,12 @@ def respawn_instances(stack_ref, inplace, force, region): region = get_region(region) check_credentials(region) - for asg_name in get_auto_scaling_groups(stack_refs, region): - respawn_auto_scaling_group(asg_name, region, inplace=inplace, force=force) + stacks = get_stacks(stack_refs, region) + for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): + if group['type'] == AUTO_SCALING_GROUP_TYPE: + respawn_auto_scaling_group(group['resource_id'], region, inplace=inplace, force=force) + elif group['type'] == AUTO_SCALING_GROUP_TYPE: + respawn_elastigroup(batch_size=batch_size_percentage) @cli.command() @@ -1617,7 +1625,8 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): minimum = desired_capacity if capacity['minimum'] > desired_capacity else capacity['minimum'] maximum = desired_capacity if capacity['maximum'] < desired_capacity else capacity['maximum'] - elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, spotinst_account_data) + elastigroup_api.update_capacity(minimum, maximum, desired_capacity, elastigroup_id, + spotinst_account_data) def scale_auto_scaling_group(asg, asg_name, desired_capacity): diff --git a/senza/respawn.py b/senza/respawn.py index 3aeaff3f..0ad99cac 100644 --- a/senza/respawn.py +++ b/senza/respawn.py @@ -6,10 +6,13 @@ from clickclick import Action, info from .manaus.boto_proxy import BotoClientProxy +from .spotinst.components import elastigroup_api SCALING_PROCESSES_TO_SUSPEND = ['AZRebalance', 'AlarmNotification', 'ScheduledActions'] RUNNING_LIFECYCLE_STATES = set(['Pending', 'InService', 'Rebooting']) +DEFAULT_BATCH_SIZE = 20 + def get_auto_scaling_group(asg, asg_name: str): '''Get boto3 Auto Scaling Group by name or raise exception''' @@ -151,3 +154,17 @@ def respawn_auto_scaling_group(asg_name: str, region: str, inplace: bool=False, inplace) else: info('Nothing to do') + + +def respawn_elastigroup(batch_size: int): + ''' + Respawn all instances in the ElastiGroup. + ''' + + if batch_size is None or batch_size < 1: + batch_size = DEFAULT_BATCH_SIZE + + # TODO : call deploy with proper parameters + elastigroup_api.deploy(batch_size=batch_size) + + pass \ No newline at end of file diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py index 5ef726b8..5bb527a7 100644 --- a/senza/spotinst/components/elastigroup_api.py +++ b/senza/spotinst/components/elastigroup_api.py @@ -7,6 +7,9 @@ SPOTINST_API_URL = 'https://api.spotinst.io' +DEPLOY_STRATEGY_RESTART = 'RESTART_SERVER' +DEPLOY_STRATEGY_REPLACE = 'REPLACE_SERVER' + class SpotInstAccountData: ''' @@ -75,7 +78,7 @@ def get_elastigroup(elastigroup_id, spotinst_account_data): } response = requests.get( - 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), headers=headers, timeout=5) response.raise_for_status() data = response.json() @@ -102,3 +105,32 @@ def patch_elastigroup(properties, elastigroup_id, spotinst_account_data): body = {'group': {'compute': compute}} return update_elastigroup(body, elastigroup_id, spotinst_account_data) + + +def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, elastigroup_id=None, spotinst_account_data=None): + ''' + Triggers Blue/Green Deployment that replaces the existing instances in the Elastigroup + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/deploy/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + body = { + 'batchSizePercentage': batch_size, + 'gracePeriod': grace_period, + 'strategy': { + 'action': strategy + } + } + + response = requests.put( + '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + headers=headers, timeout=10, data=json.dumps(body)) + response.raise_for_status() + data = response.json() + deploys = data.get("response", {}).get("items", []) + + return deploys diff --git a/tests/test_cli.py b/tests/test_cli.py index 4ac925b5..2eee3c75 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1632,21 +1632,45 @@ def patch_auto_scaling_group(group, region, properties): def test_respawn(monkeypatch): boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) - monkeypatch.setattr('senza.cli.get_auto_scaling_groups', lambda *args: 'myasg') - monkeypatch.setattr('senza.cli.respawn_auto_scaling_group', lambda *args, **kwargs: None) + boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', + 'CreationTime': '2016-06-14'}]} + boto3.describe_stack_resources.return_value = {'StackResources': [ + { + 'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1' + }]} + monkeypatch.setattr('senza.respawn.respawn_auto_scaling_group', lambda *args, **kwargs: None) runner = CliRunner() runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], catch_exceptions=False) -def test_scale(monkeypatch): +def test_respawn_elastigroup(monkeypatch): boto3 = MagicMock() + monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', 'CreationTime': '2016-06-14'}]} + + elastigroup_id = 'myelasti' boto3.describe_stack_resources.return_value = {'StackResources': - [{'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', - 'PhysicalResourceId': 'myasg', + [{'ResourceType': 'Custom::elastigroup', + 'PhysicalResourceId': elastigroup_id, 'StackName': 'myapp-1'}]} + monkeypatch.setattr('senza.respawn.respawn_elastigroup', lambda *args, **kwargs: None) + runner = CliRunner() + runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], + catch_exceptions=False) + + +def test_scale(monkeypatch): + boto3 = MagicMock() + boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', + 'CreationTime': '2016-06-14'}]} + boto3.describe_stack_resources.return_value = {'StackResources': [ + {'ResourceType': 'AWS::AutoScaling::AutoScalingGroup', + 'PhysicalResourceId': 'myasg', + 'StackName': 'myapp-1'}]} # NOTE: we are using invalid MinSize (< capacity) here to get one more line covered ;-) group = {'AutoScalingGroupName': 'myasg', 'DesiredCapacity': 1, 'MinSize': 3, 'MaxSize': 1} boto3.describe_auto_scaling_groups.return_value = {'AutoScalingGroups': [group]} diff --git a/tests/test_elastigroup_api.py b/tests/test_elastigroup_api.py index 91283a0f..0678aa5e 100644 --- a/tests/test_elastigroup_api.py +++ b/tests/test_elastigroup_api.py @@ -1,5 +1,6 @@ import responses -from senza.spotinst.components.elastigroup_api import update_capacity, get_elastigroup, patch_elastigroup, SPOTINST_API_URL, SpotInstAccountData +from senza.spotinst.components.elastigroup_api import update_capacity, get_elastigroup, patch_elastigroup, deploy, \ + SPOTINST_API_URL, SpotInstAccountData def test_update_capacity(monkeypatch): @@ -22,7 +23,8 @@ def test_update_capacity(monkeypatch): spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: rsps.add(rsps.PUT, - '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), status=200, json=update) @@ -48,7 +50,8 @@ def test_get_elastigroup(monkeypatch): elastigroup_id = 'sig-xfy' spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') with responses.RequestsMock() as rsps: - rsps.add(rsps.GET, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), + rsps.add(rsps.GET, '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), status=200, json=group) @@ -85,7 +88,8 @@ def test_patch_elastigroup(monkeypatch): with responses.RequestsMock() as rsps: spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') elastigroup_id = 'sig-xfy' - rsps.add(rsps.PUT, 'https://api.spotinst.io/aws/ec2/group/{}?accountId={}'.format(elastigroup_id, spotinst_account_data.account_id), + rsps.add(rsps.PUT, '{}/aws/ec2/group/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), status=200, json=update_response) @@ -93,3 +97,37 @@ def test_patch_elastigroup(monkeypatch): assert patch_response['compute']['launchSpecification']['imageId'] == 'image-foo' assert patch_response['compute']['instanceTypes']['ondemand'] == 'm1.micro' assert patch_response['compute']['launchSpecification']['userData'] == 'user-data-value' + + +def test_deploy(monkeypatch): + response_json = { + "response": { + "items": [ + { + 'id': 'deploy-id', + 'status': 'STARTING', + 'currentBatch': 1, + 'numOfBatches': 1, + 'progress': { + 'unit': 'percentage', + 'value': 0 + } + } + ] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + rsps.add(rsps.PUT, '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), + status=200, + json=response_json) + + deploy_response = deploy(batch_size=35, grace_period=50, elastigroup_id=elastigroup_id, + spotinst_account_data=spotinst_account_data)[0] + + assert deploy_response['id'] == 'deploy-id' + assert deploy_response['status'] == 'STARTING' + assert deploy_response['numOfBatches'] == 1 From e479ccf3760d28993dde605cf639460050d0fa7b Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Tue, 31 Jul 2018 12:06:09 +0200 Subject: [PATCH 09/10] refactoring, tests and missing respawn functionality --- senza/cli.py | 27 +++-------- senza/respawn.py | 30 ++++++++++-- senza/spotinst/components/elastigroup_api.py | 36 ++++++++++++++ tests/test_cli.py | 13 ++++- tests/test_elastigroup_api.py | 39 ++++++++++++++- tests/test_respawn.py | 50 +++++++++++++++++++- 6 files changed, 168 insertions(+), 27 deletions(-) diff --git a/senza/cli.py b/senza/cli.py index 55c2a49e..428a4f40 100755 --- a/senza/cli.py +++ b/senza/cli.py @@ -19,6 +19,7 @@ import senza.stups.taupage as taupage import requests import yaml +import senza.respawn as respawn from botocore.exceptions import ClientError from clickclick import (Action, FloatRange, OutputFormat, choice, error, @@ -43,7 +44,6 @@ from .manaus.route53 import Route53, Route53Record from .manaus.utils import extract_client_error_code from .patch import patch_auto_scaling_group, patch_elastigroup -from .respawn import get_auto_scaling_group, respawn_auto_scaling_group, respawn_elastigroup from .stups.piu import Piu from .subcommands.config import cmd_config from .subcommands.root import cli @@ -1519,25 +1519,12 @@ def patch_aws_asg(properties, region, asg, asg_name): act.ok('NO CHANGES') -def get_spotinst_account_data(region, stack_name): - ''' - Extracts required parameters required to access SpotInst API - ''' - cf = boto3.client('cloudformation', region) - template = cf.get_template(StackName=stack_name)['TemplateBody'] - - spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] - spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] - - return elastigroup_api.SpotInstAccountData(spotinst_account_id, spotinst_token) - - def patch_spotinst_elastigroup(properties, elastigroup_id, region, stack_name): ''' Patch specific properties of an existing ElastiGroup ''' - spotinst_account_data = get_spotinst_account_data(region, stack_name) + spotinst_account_data = elastigroup_api.get_spotinst_account_data(region, stack_name) with Action('Patching ElastiGroup {} (ID: {})..'.format(stack_name, elastigroup_id)) as act: groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) @@ -1572,9 +1559,9 @@ def respawn_instances(stack_ref, inplace, force, batch_size_percentage, region): stacks = get_stacks(stack_refs, region) for group in get_auto_scaling_groups_and_elasti_groups(stacks, region): if group['type'] == AUTO_SCALING_GROUP_TYPE: - respawn_auto_scaling_group(group['resource_id'], region, inplace=inplace, force=force) - elif group['type'] == AUTO_SCALING_GROUP_TYPE: - respawn_elastigroup(batch_size=batch_size_percentage) + respawn.respawn_auto_scaling_group(group['resource_id'], region, inplace=inplace, force=force) + elif group['type'] == ELASTIGROUP_TYPE: + respawn.respawn_elastigroup(group['resource_id'], group['stack_name'], region, batch_size_percentage) @cli.command() @@ -1610,7 +1597,7 @@ def scale_elastigroup(elastigroup_id, stack_name, desired_capacity, region): ''' Commands to scale an ElastiGroup ''' - spotinst_account_data = get_spotinst_account_data(region, stack_name) + spotinst_account_data = elastigroup_api.get_spotinst_account_data(region, stack_name) groups = elastigroup_api.get_elastigroup(elastigroup_id, spotinst_account_data) @@ -1633,7 +1620,7 @@ def scale_auto_scaling_group(asg, asg_name, desired_capacity): ''' Commands to scale an AWS Auto Scaling Group ''' - group = get_auto_scaling_group(asg, asg_name) + group = respawn.get_auto_scaling_group(asg, asg_name) current_capacity = group['DesiredCapacity'] with Action('Scaling {} from {} to {} instances..'.format( asg_name, current_capacity, desired_capacity)) as act: diff --git a/senza/respawn.py b/senza/respawn.py index 0ad99cac..1debab17 100644 --- a/senza/respawn.py +++ b/senza/respawn.py @@ -11,6 +11,8 @@ SCALING_PROCESSES_TO_SUSPEND = ['AZRebalance', 'AlarmNotification', 'ScheduledActions'] RUNNING_LIFECYCLE_STATES = set(['Pending', 'InService', 'Rebooting']) +ELASTIGROUP_TERMINATED_DEPLOY_STATUS = ['stopped', 'failed'] + DEFAULT_BATCH_SIZE = 20 @@ -156,7 +158,7 @@ def respawn_auto_scaling_group(asg_name: str, region: str, inplace: bool=False, info('Nothing to do') -def respawn_elastigroup(batch_size: int): +def respawn_elastigroup(elastigroup_id: str, stack_name: str, region: str, batch_size: int): ''' Respawn all instances in the ElastiGroup. ''' @@ -164,7 +166,27 @@ def respawn_elastigroup(batch_size: int): if batch_size is None or batch_size < 1: batch_size = DEFAULT_BATCH_SIZE - # TODO : call deploy with proper parameters - elastigroup_api.deploy(batch_size=batch_size) + spotinst_account = elastigroup_api.get_spotinst_account_data(region, stack_name) + + info('Redeploying the cluster for ElastiGroup {} (ID {})'.format(stack_name, elastigroup_id)) - pass \ No newline at end of file + deploy_output = elastigroup_api.deploy(batch_size=batch_size, grace_period=600, elastigroup_id=elastigroup_id, + spotinst_account_data=spotinst_account) + + deploy_count = len(deploy_output) + deploys_finished = 0 + with Action('Waiting for deploy to complete. Total of {} deploys'.format(deploy_count)) as act: + while True: + for deploy in deploy_output: + deploy_status = elastigroup_api.deploy_status(deploy['id'], elastigroup_id, spotinst_account) + for ds in deploy_status: + if ds['id'] == deploy['id']: + if ds['progress']['value'] >= 100\ + or ds['status'].lower() in ELASTIGROUP_TERMINATED_DEPLOY_STATUS: + deploys_finished += 1 + info('Deploy {} finished with status {}'.format(ds['id'], ds['status'])) + + if deploys_finished == deploy_count: + break + time.sleep(2) + act.progress() diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py index 5bb527a7..ba6148b8 100644 --- a/senza/spotinst/components/elastigroup_api.py +++ b/senza/spotinst/components/elastigroup_api.py @@ -3,6 +3,7 @@ ''' import requests import json +import boto3 SPOTINST_API_URL = 'https://api.spotinst.io' @@ -20,6 +21,19 @@ def __init__(self, account_id, access_token): self.access_token = access_token +def get_spotinst_account_data(region, stack_name): + ''' + Extracts required parameters required to access SpotInst API + ''' + cf = boto3.client('cloudformation', region) + template = cf.get_template(StackName=stack_name)['TemplateBody'] + + spotinst_token = template['Mappings']['Senza']['Info']['SpotinstAccessToken'] + spotinst_account_id = template['Resources']['AppServerConfig']['Properties']['accountId'] + + return SpotInstAccountData(spotinst_account_id, spotinst_token) + + def update_elastigroup(body, elastigroup_id, spotinst_account_data): ''' Performs the update ElastiGroup API call. @@ -134,3 +148,25 @@ def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, el deploys = data.get("response", {}).get("items", []) return deploys + + +def deploy_status(deploy_id, elastigroup_id, spotinst_account_data): + ''' + Obtains the current status of a deployment. + + For more details see https://api.spotinst.com/elastigroup/amazon-web-services/deploy-status/ + ''' + headers = { + "Authorization": "Bearer {}".format(spotinst_account_data.access_token), + "Content-Type": "application/json" + } + + response = requests.get( + '{}/aws/ec2/group/{}/roll/{}?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, deploy_id, + spotinst_account_data.account_id), + headers=headers, timeout=5) + response.raise_for_status() + data = response.json() + deploys = data.get("response", {}).get("items", []) + + return deploys diff --git a/tests/test_cli.py b/tests/test_cli.py index 2eee3c75..b2119ed1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1630,6 +1630,7 @@ def patch_auto_scaling_group(group, region, properties): def test_respawn(monkeypatch): + #TODO : broken test boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', @@ -1640,6 +1641,7 @@ def test_respawn(monkeypatch): 'PhysicalResourceId': 'myasg', 'StackName': 'myapp-1' }]} + # TODO : this mocking is not working monkeypatch.setattr('senza.respawn.respawn_auto_scaling_group', lambda *args, **kwargs: None) runner = CliRunner() runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], @@ -1647,6 +1649,7 @@ def test_respawn(monkeypatch): def test_respawn_elastigroup(monkeypatch): + # TODO : broken test boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', @@ -1657,11 +1660,19 @@ def test_respawn_elastigroup(monkeypatch): [{'ResourceType': 'Custom::elastigroup', 'PhysicalResourceId': elastigroup_id, 'StackName': 'myapp-1'}]} - monkeypatch.setattr('senza.respawn.respawn_elastigroup', lambda *args, **kwargs: None) + + test = {'success': False} + + def verification(*args): + test['success'] = True + + monkeypatch.setattr('senza.respawn.respawn_elastigroup', verification) runner = CliRunner() runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], catch_exceptions=False) + assert test['success'] + def test_scale(monkeypatch): boto3 = MagicMock() diff --git a/tests/test_elastigroup_api.py b/tests/test_elastigroup_api.py index 0678aa5e..b429e45b 100644 --- a/tests/test_elastigroup_api.py +++ b/tests/test_elastigroup_api.py @@ -1,6 +1,6 @@ import responses from senza.spotinst.components.elastigroup_api import update_capacity, get_elastigroup, patch_elastigroup, deploy, \ - SPOTINST_API_URL, SpotInstAccountData + deploy_status, SPOTINST_API_URL, SpotInstAccountData def test_update_capacity(monkeypatch): @@ -131,3 +131,40 @@ def test_deploy(monkeypatch): assert deploy_response['id'] == 'deploy-id' assert deploy_response['status'] == 'STARTING' assert deploy_response['numOfBatches'] == 1 + + +def test_deploy_status(monkeypatch): + deploy_id = 'deploy-id-x' + response_json = { + "response": { + "items": [ + { + 'id': deploy_id, + 'status': 'STARTING', + 'currentBatch': 13, + 'numOfBatches': 20, + 'progress': { + 'unit': 'percentage', + 'value': 65 + } + } + ] + } + } + with responses.RequestsMock() as rsps: + spotinst_account_data = SpotInstAccountData('act-zwk', 'fake-token') + elastigroup_id = 'sig-xfy' + + rsps.add(rsps.GET, '{}/aws/ec2/group/{}/roll/{}?accountId={}'.format(SPOTINST_API_URL, + elastigroup_id, + deploy_id, + spotinst_account_data.account_id), + status=200, + json=response_json) + + deploy_status_response = deploy_status(deploy_id, elastigroup_id, spotinst_account_data)[0] + + assert deploy_status_response['id'] == deploy_id + assert deploy_status_response['numOfBatches'] == 20 + assert deploy_status_response['progress']['value'] == 65 + diff --git a/tests/test_respawn.py b/tests/test_respawn.py index c20cb3c5..2ad8f286 100644 --- a/tests/test_respawn.py +++ b/tests/test_respawn.py @@ -1,6 +1,8 @@ from unittest.mock import MagicMock -from senza.respawn import respawn_auto_scaling_group +from senza.respawn import respawn_auto_scaling_group, respawn_elastigroup +from senza.spotinst.components.elastigroup_api import SpotInstAccountData + def test_respawn_auto_scaling_group(monkeypatch): @@ -23,6 +25,7 @@ def terminate_instance(InstanceId, **kwargs): elb = MagicMock() elb.describe_instance_health.return_value = {'InstanceStates': instance_states} services = {'autoscaling': asg, 'elb': elb} + def client(service, region): assert region == 'myregion' return services[service] @@ -60,3 +63,48 @@ def client(service, *args): monkeypatch.setattr('time.sleep', lambda s: s) respawn_auto_scaling_group('myasg', 'myregion') + +def test_respawn_elastigroup(monkeypatch): + elastigroup_id = 'sig-xfy' + stack_name = 'my-app-stack' + region = 'my-region' + batch_size = 35 + + spotinst_account = SpotInstAccountData('act-zwk', 'fake-token') + spotinst_account_mock = MagicMock() + spotinst_account_mock.return_value = spotinst_account + + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.get_spotinst_account_data', spotinst_account_mock) + + deploy_output = [{ + 'id': 'deploy-1' + }] + deploy_output_mock = MagicMock() + deploy_output_mock.return_value = deploy_output + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.deploy', deploy_output_mock) + + execution_data = { + 'percentage': 0, + 'runs': 0, + 'status': 'starting' + } + + def deploy_status(*args): + execution_data['runs'] += 1 + execution_data['percentage'] += 50 + if execution_data['percentage'] == 100: + execution_data['status'] = 'finished' + else: + execution_data['status'] = 'in_progress' + return [{ + 'id': args[0], + 'status': execution_data['status'], + 'progress': { + 'value': execution_data['percentage'] + } + }] + monkeypatch.setattr('senza.spotinst.components.elastigroup_api.deploy_status', deploy_status) + respawn_elastigroup(elastigroup_id, stack_name, region, batch_size) + + assert execution_data['runs'] == 2 + assert execution_data['percentage'] == 100 From 545e9227af35ef98ae8a30f159be9fcd68839895 Mon Sep 17 00:00:00 2001 From: Pedro Alves Date: Tue, 31 Jul 2018 12:14:22 +0200 Subject: [PATCH 10/10] codacy review issues --- senza/spotinst/components/elastigroup_api.py | 6 ++++-- tests/test_cli.py | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/senza/spotinst/components/elastigroup_api.py b/senza/spotinst/components/elastigroup_api.py index ba6148b8..95cffa82 100644 --- a/senza/spotinst/components/elastigroup_api.py +++ b/senza/spotinst/components/elastigroup_api.py @@ -121,7 +121,8 @@ def patch_elastigroup(properties, elastigroup_id, spotinst_account_data): return update_elastigroup(body, elastigroup_id, spotinst_account_data) -def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, elastigroup_id=None, spotinst_account_data=None): +def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, + elastigroup_id=None, spotinst_account_data=None): ''' Triggers Blue/Green Deployment that replaces the existing instances in the Elastigroup @@ -141,7 +142,8 @@ def deploy(batch_size=20, grace_period=300, strategy=DEPLOY_STRATEGY_REPLACE, el } response = requests.put( - '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, spotinst_account_data.account_id), + '{}/aws/ec2/group/{}/roll?accountId={}'.format(SPOTINST_API_URL, elastigroup_id, + spotinst_account_data.account_id), headers=headers, timeout=10, data=json.dumps(body)) response.raise_for_status() data = response.json() diff --git a/tests/test_cli.py b/tests/test_cli.py index b2119ed1..903a97e3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1630,7 +1630,6 @@ def patch_auto_scaling_group(group, region, properties): def test_respawn(monkeypatch): - #TODO : broken test boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1', @@ -1641,7 +1640,6 @@ def test_respawn(monkeypatch): 'PhysicalResourceId': 'myasg', 'StackName': 'myapp-1' }]} - # TODO : this mocking is not working monkeypatch.setattr('senza.respawn.respawn_auto_scaling_group', lambda *args, **kwargs: None) runner = CliRunner() runner.invoke(cli, ['respawn', 'myapp', '1', '--region=aa-fakeregion-1'], @@ -1649,7 +1647,6 @@ def test_respawn(monkeypatch): def test_respawn_elastigroup(monkeypatch): - # TODO : broken test boto3 = MagicMock() monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) boto3.list_stacks.return_value = {'StackSummaries': [{'StackName': 'myapp-1',