Merge pull request #446 from cloudify-cosmo/RD-2009-aws-plugin-ec-2-b…

…ase-properties-improvement

Rd 2009 aws plugin ec 2 base properties improvement

.circleci/config.yml

@@ -33,7 +33,7 @@ commands:
 
   run_hello_world_test:
     steps:
-      - run: ecosystem-test local-blueprint-test  -b examples/blueprint-examples/hello-world-example/aws.yaml --test-id=hello-world-example -i aws_region_name=us-east-1 --on-failure=uninstall-force --timeout=1800
+      - run: ecosystem-test local-blueprint-test  -b examples/blueprint-examples/hello-world-example/aws.yaml --test-id=hello-world-example -i aws_region_name=us-east-1 --on-failure=uninstall-force --timeout=1800 --nested-test=.circleci/test_cloudwatch.py
 
   run_cf_test:
     steps:
@@ -64,7 +64,7 @@ commands:
 
   run_ec2_instance_test:
     steps:
-      - run: ecosystem-test local-blueprint-test -b examples/ec2-instance-feature-demo/blueprint.yaml --test-id=ec2-instance --on-failure=uninstall-force --timeout=3000
+      - run: ecosystem-test local-blueprint-test -b examples/ec2-vpc-feature-demo/ipv6.yaml --test-id=ec2-ipv6 --on-failure=uninstall-force --timeout=3000
 
   run_ec2_keys_test:
     steps:

.circleci/test_cloudwatch.py

@@ -0,0 +1,100 @@
+########
+# Copyright (c) 2014-2019 Cloudify Platform Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from os import environ
+from contextlib import contextmanager
+
+import pytest
+
+from boto3 import client
+from ecosystem_tests.dorkl.constansts import logger
+from ecosystem_tests.dorkl import cleanup_on_failure
+from ecosystem_tests.dorkl.exceptions import EcosystemTestException
+from ecosystem_tests.dorkl.cloudify_api import (
+    cloudify_exec,
+    blueprints_upload,
+    deployments_create,
+    executions_start)
+
+TEST_ID = environ.get('__ECOSYSTEM_TEST_ID', 'virtual-machine')
+
+
+@contextmanager
+def test_cleaner_upper():
+    try:
+        yield
+    except Exception:
+        cleanup_on_failure(TEST_ID)
+        raise
+
+
+@pytest.mark.dependency(depends=['test_plan_protection'])
+def test_cloudwatch(*_, **__):
+    with test_cleaner_upper():
+        vm_props = cloud_resources_node_instance_runtime_properties()
+        instance_id = vm_props.get('aws_resource_id')
+        deployment_id = TEST_ID + 'cloudwatch'
+        try:
+            # Upload Cloud Watch Blueprint
+            blueprints_upload(
+                'examples/cloudwatch-feature-demo/blueprint.yaml',
+                deployment_id)
+            # Create Cloud Watch Deployment with Instance ID input
+            deployments_create(deployment_id,
+                               {"aws_instance_id": str(instance_id),
+                                "aws_region_name": "us-west-2"})
+            # Install Cloud Watch Deployment
+            executions_start('install', deployment_id)
+            # Uninstall Cloud Watch Deployment
+            executions_start('uninstall', deployment_id)
+        except:
+            cleanup_on_failure(deployment_id)
+
+
+def cloud_resources_node_instance_runtime_properties():
+    node_instance = node_instance_by_name('vm')
+    logger.info('Node instance: {node_instance}'.format(
+        node_instance=node_instance))
+    if not node_instance:
+        raise RuntimeError('No cloud_resources node instances found.')
+    runtime_properties = node_instance_runtime_properties(
+        node_instance['id'])
+    logger.info('Runtime properties: {runtime_properties}'.format(
+        runtime_properties=runtime_properties))
+    if not runtime_properties:
+        raise RuntimeError(
+            'No cloud_resources runtime_properties found.')
+    return runtime_properties
+
+
+def node_instance_by_name(name):
+    for node_instance in node_instances():
+        if node_instance['node_id'] == name:
+            return node_instance
+    raise Exception('No node instances found.')
+
+
+def node_instance_runtime_properties(name):
+    node_instance = cloudify_exec(
+        'cfy node-instance get {name}'.format(name=name))
+    return node_instance['runtime_properties']
+
+
+def nodes():
+    return cloudify_exec('cfy nodes list')
+
+
+def node_instances():
+    return cloudify_exec('cfy node-instances list -d {}'.format(TEST_ID))

CHANGELOG.txt

@@ -1,3 +1,8 @@
+3.0.2:
+    - RD-4434 Nics for Spot Fleet Requests
+    - RD-2009 EC2 base properties improvement
+    - Fix for EFS
+    - Fix for ECS
 3.0.1:
     - RD-3458 fix iam blueprint & iam status
 3.0.0:
@@ -190,4 +195,4 @@
     - Fix bug that delete External Resource keypairs
 1.4.2:
     - Moved EC2 and VPC to cloudify_aws package
-    - Added an Instance-Subnet connected_to relationship
+    - Added an Instance-Subnet connected_to relationship

cloudify_aws/autoscaling/resources/autoscaling_group.py

@@ -17,11 +17,10 @@
     AWS Autoscaling Group interface
 
 """
-# Standard imports
-from re import sub
 
 # Third party imports
-from cloudify.exceptions import OperationRetry
+from cloudify.exceptions import OperationRetry, NonRecoverableError
+
 from cloudify_aws.common import decorators, utils
 from cloudify_aws.autoscaling import AutoscalingBase
 
@@ -148,26 +147,7 @@ def create(ctx, iface, resource_config, params, **_):
                 INSTANCE_TYPE)
         params[INSTANCE_ID] = instance_id
 
-    subnet_list_string = params.get(SUBNET_LIST)
-    subnet_list = \
-        sub("[^\w]", " ", subnet_list_string).split() if \
-        subnet_list_string else []
-
-    subnet_list = \
-        utils.add_resources_from_rels(
-            ctx.instance,
-            SUBNET_TYPE,
-            subnet_list)
-    subnet_list = \
-        utils.add_resources_from_rels(
-            ctx.instance,
-            SUBNET_TYPE_DEPRECATED,
-            subnet_list)
-
-    if subnet_list:
-        # Remove any duplicate items from subnet list
-        subnet_list = list(set(subnet_list))
-        params[SUBNET_LIST] = ', '.join(subnet_list)
+    get_subnet_list(ctx.instance, params)
 
     # Actually create the resource
     if not iface.resource_id:
@@ -181,9 +161,7 @@ def create(ctx, iface, resource_config, params, **_):
 
 
 @decorators.aws_resource(AutoscalingGroup, RESOURCE_TYPE)
-def stop(iface,
-         resource_config,
-         **_):
+def stop(iface, resource_config, **_):
     """Stops all instances associated with Autoscaling group."""
 
     autoscaling_group = iface.properties
@@ -232,3 +210,28 @@ def delete(iface, resource_config, **_):
              [instance.get(INSTANCE_ID) for instance in instances]})
 
     iface.delete(params)
+
+
+def get_subnet_list(ctx_instance, params):
+    subnet_list = params.get(SUBNET_LIST)
+    subnet_list = subnet_list or []
+    if not isinstance(subnet_list, list):
+        raise NonRecoverableError(
+            'The provided {} is not a list, '
+            'please reformat. Provided value: {}'.format(
+                SUBNET_LIST, subnet_list))
+
+    subnet_list = \
+        utils.add_resources_from_rels(
+            ctx_instance,
+            SUBNET_TYPE,
+            subnet_list)
+    subnet_list = \
+        utils.add_resources_from_rels(
+            ctx_instance,
+            SUBNET_TYPE_DEPRECATED,
+            subnet_list)
+    if subnet_list:
+        # Remove any duplicate items from subnet list
+        subnet_list = list(set(subnet_list))
+        params[SUBNET_LIST] = ', '.join(subnet_list)

cloudify_aws/autoscaling/resources/launch_configuration.py

@@ -168,4 +168,4 @@ def delete(iface, resource_config, **_):
     params = dict() if not resource_config else resource_config.copy()
     if RESOURCE_NAME not in params:
         params.update({RESOURCE_NAME: iface.resource_id})
-    iface.delete(params)
+    utils.handle_response(iface, 'delete', params, ['not found'])

cloudify_aws/common/decorators.py

@@ -425,12 +425,19 @@ def _aws_resource(function,
     aws_config = ctx.instance.runtime_properties.get('aws_config')
     aws_config_kwargs = kwargs.get('aws_config')
 
+    resource_name = None
+    if 'cloudify.nodes.aws.elb.LoadBalancer' in ctx.node.type_hierarchy:
+        resource_name = ctx.node.properties.get(
+            'resource_config', {}).get('Name')
+
     # Attribute needed for AWS resource class
     class_decl_attr = {
         'ctx_node': ctx.node,
         'logger': ctx.logger,
         'resource_id': utils.get_resource_id(
-            node=ctx.node, instance=ctx.instance),
+            node=ctx.node,
+            instance=ctx.instance,
+            resource_name=resource_name),
     }
 
     _put_aws_config_in_class_decl(

cloudify_aws/common/utils.py

@@ -622,9 +622,11 @@ def handle_response(iface,
     :param raisable: The exception to raise if substrings are found.
     :return:
     """
+    exit_substrings = exit_substrings or []
     if isinstance(exit_substrings, text_type):
         exit_substrings = [exit_substrings]
 
+    raise_substrings = raise_substrings or []
     if isinstance(raise_substrings, text_type):
         raise_substrings = [raise_substrings]
 

cloudify_aws/ec2/resources/dhcp.py

@@ -16,8 +16,6 @@
     ~~~~~~~~~~~~~~
     AWS EC2 DhcpOptions interface
 """
-# Boto
-from botocore.exceptions import ClientError, ParamValidationError
 
 # Cloudify
 from cloudify_aws.common import decorators, utils
@@ -40,20 +38,10 @@ class EC2DHCPOptions(EC2Base):
     def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
         EC2Base.__init__(self, ctx_node, resource_id, client, logger)
         self.type_name = RESOURCE_TYPE
-
-    @property
-    def properties(self):
-        """Gets the properties of an external resource"""
-        if not self.resource_id:
-            return
-        params = {DHCPOPTIONS_IDS: [self.resource_id]}
-        try:
-            resources = \
-                self.client.describe_dhcp_options(**params)
-        except (ClientError, ParamValidationError):
-            pass
-        else:
-            return resources.get(DHCPOPTIONS)[0] if resources else None
+        self._describe_call = 'describe_dhcp_options'
+        self._type_key = DHCPOPTIONS
+        self._id_key = DHCPOPTIONS_ID
+        self._ids_key = DHCPOPTIONS_IDS
 
     def create(self, params):
         """

cloudify_aws/ec2/resources/ebs.py

@@ -17,7 +17,6 @@
     AWS EC2 EBS Volume
 """
 # Boto
-from botocore.exceptions import ClientError, ParamValidationError
 from botocore.exceptions import CapacityNotAvailableError
 
 # Cloudify
@@ -60,23 +59,10 @@ class EC2VolumeMixin(object):
     def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
         EC2Base.__init__(self, ctx_node, resource_id, client, logger)
         self.type_name = RESOURCE_TYPE_VOLUME
-
-    @property
-    def properties(self):
-        """
-        Gets the properties of an external resource
-        :return: dict of selected volume
-        """
-        if not self.resource_id:
-            return
-        params = {VOLUME_IDS: [self.resource_id]}
-        try:
-            resources = \
-                self.client.describe_volumes(**params)
-        except (ClientError, ParamValidationError):
-            pass
-        else:
-            return resources.get(VOLUMES)[0] if resources else None
+        self._describe_call = 'describe_volumes'
+        self._type_key = VOLUMES
+        self._id_key = VOLUME_ID
+        self._ids_key = VOLUME_IDS
 
     @property
     def status(self):

cloudify_aws/ec2/resources/elasticip.py

@@ -50,16 +50,20 @@ def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
         EC2Base.__init__(self, ctx_node, resource_id, client, logger)
         self.type_name = RESOURCE_TYPE
         self.allocation_id = None
+        self._describe_call = 'describe_addresses'
+        self._type_key = ADDRESSES
+        self._ids_key = ELASTICIP_IDS
+        self._id_key = ELASTICIP_ID
 
-    def list(self, params=None):
+    def get(self, params=None):
         try:
             if params:
                 resources = self.client.describe_addresses(**params)
             else:
                 resources = self.client.describe_addresses()
         except (ParamValidationError, ClientError):
-            return []
-        return resources.get(ADDRESSES, [])
+            return {}
+        return resources.get(ADDRESSES, {})
 
     def update_allocation_id(self, allocation_id):
         self.allocation_id = allocation_id
@@ -72,10 +76,11 @@ def tag(self, params):
     def properties(self):
         """Gets the properties of an external resource"""
         if not self.resource_id:
-            return
+            return {}
         params = {ELASTICIP_IDS: [self.resource_id]}
-        resources = self.list(params)
-        return resources[0] if resources else None
+        if not self._properties:
+            self._properties = self.get(params)
+        return self._properties[0] if self._properties else {}
 
     @property
     def status(self):
@@ -155,7 +160,7 @@ def create(ctx, iface, resource_config, **_):
     # Actually create the resource
     create_response = None
     if ctx.node.properties.get('use_unassociated_addresses', False):
-        create_response = get_already_allocated_ip(iface.list())
+        create_response = get_already_allocated_ip(iface.get())
     if not create_response:
         create_response = iface.create(params)
     else:

cloudify_aws/ec2/resources/image.py

@@ -43,6 +43,10 @@ def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
         image_filters = ctx_node.properties["resource_config"].get(
             "kwargs", {}).get("Filters")
         self._describe_image_filters = None
+        self._describe_call = "describe_images"
+        self._ids_key = IMAGE_IDS
+        self._type_key = IMAGES
+        self._id_key = IMAGE_ID
         if resource_id:
             self.prepare_describe_image_filter({IMAGE_IDS: [resource_id]})
         elif image_filters: