Merge pull request #448 from cloudify-cosmo/blueprint-changes-for-3

ecs fixes :)

CHANGELOG.txt

@@ -2,6 +2,7 @@
     - RD-4434 Nics for Spot Fleet Requests
     - RD-2009 EC2 base properties improvement
     - Fix for EFS
+    - Fix for ECS
 3.0.1:
     - RD-3458 fix iam blueprint & iam status
 3.0.0:

cloudify_aws/autoscaling/resources/autoscaling_group.py

@@ -17,11 +17,10 @@
     AWS Autoscaling Group interface
 
 """
-# Standard imports
-from re import sub
 
 # Third party imports
-from cloudify.exceptions import OperationRetry
+from cloudify.exceptions import OperationRetry, NonRecoverableError
+
 from cloudify_aws.common import decorators, utils
 from cloudify_aws.autoscaling import AutoscalingBase
 
@@ -148,26 +147,7 @@ def create(ctx, iface, resource_config, params, **_):
                 INSTANCE_TYPE)
         params[INSTANCE_ID] = instance_id
 
-    subnet_list_string = params.get(SUBNET_LIST)
-    subnet_list = \
-        sub("[^\w]", " ", subnet_list_string).split() if \
-        subnet_list_string else []
-
-    subnet_list = \
-        utils.add_resources_from_rels(
-            ctx.instance,
-            SUBNET_TYPE,
-            subnet_list)
-    subnet_list = \
-        utils.add_resources_from_rels(
-            ctx.instance,
-            SUBNET_TYPE_DEPRECATED,
-            subnet_list)
-
-    if subnet_list:
-        # Remove any duplicate items from subnet list
-        subnet_list = list(set(subnet_list))
-        params[SUBNET_LIST] = ', '.join(subnet_list)
+    get_subnet_list(ctx.instance, params)
 
     # Actually create the resource
     if not iface.resource_id:
@@ -181,9 +161,7 @@ def create(ctx, iface, resource_config, params, **_):
 
 
 @decorators.aws_resource(AutoscalingGroup, RESOURCE_TYPE)
-def stop(iface,
-         resource_config,
-         **_):
+def stop(iface, resource_config, **_):
     """Stops all instances associated with Autoscaling group."""
 
     autoscaling_group = iface.properties
@@ -232,3 +210,28 @@ def delete(iface, resource_config, **_):
              [instance.get(INSTANCE_ID) for instance in instances]})
 
     iface.delete(params)
+
+
+def get_subnet_list(ctx_instance, params):
+    subnet_list = params.get(SUBNET_LIST)
+    subnet_list = subnet_list or []
+    if not isinstance(subnet_list, list):
+        raise NonRecoverableError(
+            'The provided {} is not a list, '
+            'please reformat. Provided value: {}'.format(
+                SUBNET_LIST, subnet_list))
+
+    subnet_list = \
+        utils.add_resources_from_rels(
+            ctx_instance,
+            SUBNET_TYPE,
+            subnet_list)
+    subnet_list = \
+        utils.add_resources_from_rels(
+            ctx_instance,
+            SUBNET_TYPE_DEPRECATED,
+            subnet_list)
+    if subnet_list:
+        # Remove any duplicate items from subnet list
+        subnet_list = list(set(subnet_list))
+        params[SUBNET_LIST] = ', '.join(subnet_list)

cloudify_aws/autoscaling/resources/launch_configuration.py

@@ -168,4 +168,4 @@ def delete(iface, resource_config, **_):
     params = dict() if not resource_config else resource_config.copy()
     if RESOURCE_NAME not in params:
         params.update({RESOURCE_NAME: iface.resource_id})
-    iface.delete(params)
+    utils.handle_response(iface, 'delete', params, ['not found'])

cloudify_aws/common/decorators.py

@@ -425,12 +425,19 @@ def _aws_resource(function,
     aws_config = ctx.instance.runtime_properties.get('aws_config')
     aws_config_kwargs = kwargs.get('aws_config')
 
+    resource_name = None
+    if 'cloudify.nodes.aws.elb.LoadBalancer' in ctx.node.type_hierarchy:
+        resource_name = ctx.node.properties.get(
+            'resource_config', {}).get('Name')
+
     # Attribute needed for AWS resource class
     class_decl_attr = {
         'ctx_node': ctx.node,
         'logger': ctx.logger,
         'resource_id': utils.get_resource_id(
-            node=ctx.node, instance=ctx.instance),
+            node=ctx.node,
+            instance=ctx.instance,
+            resource_name=resource_name),
     }
 
     _put_aws_config_in_class_decl(

cloudify_aws/common/utils.py

@@ -622,9 +622,11 @@ def handle_response(iface,
     :param raisable: The exception to raise if substrings are found.
     :return:
     """
+    exit_substrings = exit_substrings or []
     if isinstance(exit_substrings, text_type):
         exit_substrings = [exit_substrings]
 
+    raise_substrings = raise_substrings or []
     if isinstance(raise_substrings, text_type):
         raise_substrings = [raise_substrings]
 

cloudify_aws/ec2/resources/routetable.py

@@ -19,6 +19,7 @@
 from time import sleep
 
 # Boto
+from botocore.exceptions import ClientError
 from cloudify.exceptions import OperationRetry
 
 # Cloudify
@@ -76,13 +77,26 @@ def create(self, params):
         '''
         return self.make_client_call('create_route_table', params)
 
-    def delete(self, params=None):
+    def delete(self, params=None, recurse=True):
         '''
             Deletes an existing AWS EC2 Route Table.
         '''
         self.logger.debug('Deleting %s with parameters: %s'
                           % (self.type_name, params))
-        res = self.client.delete_route_table(**params)
+        try:
+            res = self.client.delete_route_table(**params)
+        except ClientError:
+            if not recurse:
+                raise
+            self.logger.error(
+                'Failed to delete route table because of dependencies. '
+                'Attempting cleanup.')
+            for a in self.properties.get('Associations', []):
+                if not a.get('Main'):
+                    self.logger.info('Disassociating: {}'.format(a))
+                    self.client.disassociate_route_table(
+                        AssociationId=a.get('RouteTableAssociationId'))
+            res = self.delete(params, False)
         self.logger.debug('Response: %s' % res)
         return res
 

cloudify_aws/ec2/resources/securitygroup.py

@@ -122,7 +122,7 @@ def wait(self):
         max_wait = 5
         counter = 0
         while not self.properties:
-            self.logger.debug('Waiting for Route Table to be created.')
+            self.logger.debug('Waiting for Security Group to be created.')
             sleep(5)
             if max_wait > counter:
                 break
@@ -136,7 +136,9 @@ def prepare(ctx, iface, resource_config, **_):
     ctx.instance.runtime_properties['resource_config'] = resource_config
 
 
-@decorators.aws_resource(EC2SecurityGroup, RESOURCE_TYPE)
+@decorators.aws_resource(EC2SecurityGroup,
+                         RESOURCE_TYPE,
+                         waits_for_status=False)
 @decorators.tag_resources
 def create(ctx, iface, resource_config, **_):
     '''Creates an AWS EC2 Security Group'''
@@ -147,8 +149,6 @@ def create(ctx, iface, resource_config, **_):
     # Actually create the resource
     iface.create(params)
     utils.update_resource_id(ctx.instance, iface.resource_id)
-    utils.assign_create_response(
-        iface, ctx.instance.runtime_properties, iface.create_response)
     iface.wait()
 
 

cloudify_aws/ecs/resources/cluster.py

@@ -20,10 +20,7 @@
 
 from __future__ import unicode_literals
 
-# Boto
-
-from botocore.exceptions import ClientError, ParamValidationError
-
+from cloudify.exceptions import NonRecoverableError
 # Cloudify
 from cloudify_aws.common import decorators, utils
 from cloudify_aws.ecs import ECSBase
@@ -41,29 +38,30 @@ class ECSCluster(ECSBase):
     """
     def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
         ECSBase.__init__(self, ctx_node, resource_id, client, logger)
+        self._properties = {}
         self.type_name = RESOURCE_TYPE
         self.describe_cluster_filter = {}
 
     @property
     def properties(self):
         """Gets the properties of an external resource"""
-        try:
-            resources = \
-                self.client.describe_clusters(
-                    **self.describe_cluster_filter
-                )
-        except (ParamValidationError, ClientError):
-            pass
-        else:
-            return None if not resources else resources.get(CLUSTERS)[0]
+        if not self._properties:
+            try:
+                resources = self.make_client_call(
+                    'describe_clusters', self.describe_cluster_filter)
+            except NonRecoverableError:
+                return
+            if CLUSTERS in resources:
+                for resource in resources[CLUSTERS]:
+                    if resource.get(CLUSTER_RESOURCE_NAME) == self.resource_id:
+                        self._properties = resource
+        return self._properties
 
     @property
     def status(self):
         """Gets the status of an external resource"""
-        props = self.properties
-        if not props:
-            return None
-        return props.get('status')
+        if self.properties:
+            return self.properties.get('status')
 
     def create(self, params):
         """

cloudify_aws/ecs/tests/test_cluster.py

@@ -34,7 +34,7 @@ class TestECSCluster(TestBase):
     def setUp(self):
         super(TestECSCluster, self).setUp()
 
-        self.cluster = ECSCluster("ctx_node", resource_id=True,
+        self.cluster = ECSCluster("ctx_node", resource_id='test_cluster_name',
                                   client=True, logger=None)
         self.mock_resource = patch(
             'cloudify_aws.common.decorators.aws_resource', mock_decorator
@@ -49,8 +49,8 @@ def tearDown(self):
 
     def test_class_properties(self):
         effect = self.get_client_error_exception(name=cluster.RESOURCE_TYPE)
-        self.cluster.client = self.make_client_function('describe_clusters',
-                                                        side_effect=effect)
+        self.cluster.client = self.make_client_function(
+            'describe_clusters', side_effect=effect)
         self.assertIsNone(self.cluster.properties)
 
         response = \

cloudify_aws/efs/resources/file_system.py

@@ -115,4 +115,4 @@ def delete(ctx, iface, resource_config, **_):
         params[FILESYSTEM_ID] = iface.resource_id
 
     # Actually delete the resource
-    utils.handle_response(iface, 'delete', params, raise_substrings='')
+    utils.handle_response(iface, 'delete', params, raise_substrings=[''])

cloudify_aws/elb/resources/listener.py

@@ -50,29 +50,31 @@ def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
             resource_id,
             client or Boto3Connection(ctx_node).client('elbv2'),
             logger)
+        self._properties = {}
         self.type_name = RESOURCE_TYPE
 
     @property
     def properties(self):
         '''Gets the properties of an external resource'''
         if not self.resource_id:
             return
-        try:
-            resources = self.client.describe_listeners(
-                ListenerArns=[self.resource_id])
-        except (ParamValidationError, ClientError):
-            pass
-        else:
-            return None \
-                if not resources else resources['Listeners'][0]
+        if not self._properties:
+            try:
+                resources = self.client.describe_listeners(
+                    ListenerArns=[self.resource_id])
+            except (ParamValidationError, ClientError):
+                pass
+            else:
+                if 'Listeners' in resources:
+                    for listener in resources['Listeners']:
+                        if listener.get('ListenerArn') == self.resource_id:
+                            self._properties = listener
+        return self._properties
 
     @property
     def status(self):
         '''Gets the status of an external resource'''
-        props = self.properties
-        if not props:
-            return None
-        return props['State']['Code']
+        return self.properties
 
     def create(self, params):
         '''

cloudify_aws/elb/resources/load_balancer.py

@@ -62,13 +62,13 @@ def properties(self):
         if not self.resource_id:
             return
         if not self._properties:
-            res = self.get_describe_result(
-                {'LoadBalancerNames': [self.resource_id]})
+            res = self.get_describe_result({'Names': [self.resource_id]})
+            self.logger.info('Describe LB response: {}'.format(res))
+            names = []
             if 'LoadBalancers' in res:
                 for elb in res['LoadBalancers']:
                     lb_name = elb.get('LoadBalancerName')
                     name = elb.get('Name')
-                    names = []
                     if lb_name:
                         names.append(lb_name)
                     if name:
@@ -195,6 +195,8 @@ def modify(ctx, iface, resource_config, **_):
         modify_params[LB_ATTR] = modify_params_attributes
         # Actually modify the resource
         attributes = iface.modify_attribute(modify_params)
+        if 'resource_config' not in ctx.instance.runtime_properties:
+            ctx.instance.runtime_properties['resource_config'] = {}
         ctx.instance.runtime_properties['resource_config'][LB_ATTR] = \
             attributes