Merge pull request #327 from cloudify-cosmo/2.5.7-build

2.5.7 build

CHANGELOG.txt

@@ -106,3 +106,6 @@
     - Fixed bug causing problems when using an existing internet gateway.
 2.5.6:
     - Fixed bug causing failure when Tag Specifications are passed to an Elastic IP.
+2.5.7:
+    - Store Lambda payload as valid JSON
+

cloudify_aws/ec2/resources/customer_gateway.py

@@ -77,7 +77,7 @@ def delete(self, params=None):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2CustomerGateway, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 Customer Gateway"""
     # Save the parameters

cloudify_aws/ec2/resources/dhcp.py

@@ -92,7 +92,7 @@ def detach(self, params):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2DHCPOptions, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 DhcpOptions"""
     # Save the parameters

cloudify_aws/ec2/resources/ebs.py

@@ -205,7 +205,7 @@ def _delete_attachment(ctx, iface):
     _detach_ebs(iface, resource_id)
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE_VOLUME)
+@decorators.aws_resource(EC2Volume, resource_type=RESOURCE_TYPE_VOLUME)
 def prepare(ctx, resource_config, **_):
     """
     Prepares an AWS EC2 EBS Volume

cloudify_aws/ec2/resources/elasticip.py

@@ -128,7 +128,7 @@ def get_already_allocated_ip(address_list):
             return create_response
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2ElasticIP, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 ElasticIP"""
     # Save the parameters

cloudify_aws/ec2/resources/eni.py

@@ -123,7 +123,7 @@ def modify_network_interface_attribute(self, params):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2NetworkInterface, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 NetworkInterface"""
     # Save the parameters

cloudify_aws/ec2/resources/nat_gateway.py

@@ -85,7 +85,7 @@ def delete(self, params=None):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2NatGateway, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 NAT Gateway"""
     # Save the parameters

cloudify_aws/ec2/resources/networkacl.py

@@ -96,7 +96,7 @@ def replace(self, params):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2NetworkAcl, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 NetworkAcl"""
     # Save the parameters

cloudify_aws/ec2/resources/networkaclentry.py

@@ -79,7 +79,7 @@ def delete(self, params=None):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2NetworkAclEntry, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 NetworkAcl Entry"""
     # Save the parameters

cloudify_aws/ec2/resources/vpn_gateway.py

@@ -98,7 +98,7 @@ def detach(self, params):
         return res
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EC2VPNGateway, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EC2 VPN Gateway"""
     # Save the parameters

cloudify_aws/eks/resources/node_group.py

@@ -105,7 +105,7 @@ def prepare_describe_node_group_filter(params, iface):
     return iface
 
 
-@decorators.aws_resource(resource_type=RESOURCE_TYPE)
+@decorators.aws_resource(EKSNodeGroup, resource_type=RESOURCE_TYPE)
 def prepare(ctx, resource_config, **_):
     """Prepares an AWS EKS Node Group"""
     # Save the parameters

cloudify_aws/lambda_serverless/resources/function.py

@@ -16,13 +16,18 @@
     ~~~~~~~~~~~~~~~~~~~
     AWS Lambda Function interface
 '''
+import json
+
 from os import remove as os_remove
 from os.path import exists as path_exists
+from contextlib import contextmanager
+
+# Boto
+from botocore.exceptions import ClientError
+
 # Cloudify
 from cloudify_aws.common import decorators, utils
 from cloudify_aws.lambda_serverless import LambdaBase
-# Boto
-from botocore.exceptions import ClientError
 
 RESOURCE_ID = 'FunctionName'
 RESOURCE_TYPE = 'Lambda Function'
@@ -36,9 +41,15 @@ class LambdaFunction(LambdaBase):
     '''
         AWS Lambda Function interface
     '''
-    def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
+    def __init__(self,
+                 ctx_node,
+                 resource_id=None,
+                 client=None,
+                 logger=None,
+                 resource_encoding='utf-8'):
         LambdaBase.__init__(self, ctx_node, resource_id, client, logger)
         self.type_name = RESOURCE_TYPE
+        self.resource_encoding = resource_encoding
 
     @property
     def properties(self):
@@ -79,16 +90,46 @@ def invoke(self, params):
         '''
             Invokes an AWS Lambda Function.
         '''
-        params = params or dict()
-        params.update(dict(FunctionName=self.resource_id))
-        self.logger.debug('Invoking %s with parameters: %s'
-                          % (self.type_name, params))
-        res = self.client.invoke(**params)
+        invoke_params = dict()
+        invoke_params.update(params)
+        invoke_params.update(dict(FunctionName=self.resource_id))
+        with self._encode_payload(invoke_params.get('Payload')) as \
+                encoded_payload:
+            if encoded_payload is not None:
+                invoke_params['Payload'] = encoded_payload
+            self.logger.debug('Invoking %s with parameters: %s'
+                              % (self.type_name, invoke_params))
+            res = self.client.invoke(**invoke_params)
         if res and res.get('Payload'):
-            res['Payload'] = res['Payload'].read()
+            res['Payload'] = self._decode_payload(res['Payload'])
         self.logger.debug('Response: %s' % res)
         return res
 
+    @contextmanager
+    def _encode_payload(self, payload):
+        if isinstance(payload, str):
+            with open(payload, 'r') as payload_file:
+                yield payload_file
+        elif isinstance(payload, dict):
+            yield json.dumps(payload)
+        else:
+            yield payload
+
+    def _decode_payload(self, payload_stream):
+        payload = payload_stream.read()
+        if isinstance(payload, bytes):
+            payload = payload.decode(self.resource_encoding)
+        try:
+            payload = json.loads(payload)
+            if payload.get('body'):
+                try:
+                    payload['body'] = json.loads(payload['body'])
+                except ValueError:
+                    pass
+        except (ValueError, UnicodeDecodeError):
+            pass
+        return payload
+
 
 def _get_subnets_to_attach(ctx, vpc_config):
     # Attach a Subnet Group if it exists

cloudify_aws/lambda_serverless/resources/invoke.py

@@ -35,10 +35,14 @@ def configure(ctx, resource_config, **_):
 def attach_to(ctx, resource_config, **_):
     '''Attaches an Lambda Invoke to something else'''
     rtprops = ctx.source.instance.runtime_properties
+    resource_encoding = \
+        ctx.source.instance.runtime_properties.get('resource_encoding') or \
+        ctx.source.node.properties.get('resource_encoding')
     if utils.is_node_type(ctx.target.node,
                           'cloudify.nodes.aws.lambda.Function'):
         ctx.source.instance.runtime_properties['output'] = LambdaFunction(
             ctx.target.node, logger=ctx.logger,
+            resource_encoding=resource_encoding,
             resource_id=utils.get_resource_id(
                 node=ctx.target.node,
                 instance=ctx.target.instance,

cloudify_aws/lambda_serverless/tests/test_function.py

@@ -153,7 +153,7 @@ def test_class_invoke(self):
             fun.resource_id = 'test_function'
             fake_client = self.make_client_function(
                 'invoke',
-                return_value={'Payload': StringIO(u"text")})
+                return_value={'Payload': StringIO(u'text')})
             fun.client = fake_client
             result = fun.invoke({'param': 'params'})
             self.assertEqual(result, {'Payload': u'text'})
@@ -165,6 +165,45 @@ def test_class_invoke(self):
             result = fun.invoke({'param': 'params'})
             self.assertEqual(result, '')
 
+            fake_client = self.make_client_function(
+                'invoke',
+                return_value={'Payload': StringIO(u'{"text": "test"}')})
+            fun.client = fake_client
+            result = fun.invoke({'param': 'params'})
+            self.assertEqual(result, {'Payload': {"text": "test"}})
+
+    def test_class_invoke_payload(self):
+        ctx = self._get_ctx()
+        with patch(PATCH_PREFIX + 'LambdaBase'):
+            fun = function.LambdaFunction(ctx)
+            fun.logger = MagicMock()
+            fun.resource_id = 'test_function'
+            fake_client = self.make_client_function(
+                'invoke',
+                side_effect=lambda FunctionName, Payload: self.assertEqual(
+                    Payload, 0))
+            fun.client = fake_client
+            result = fun.invoke({'Payload': 0})
+            self.assertEqual(result, None)
+
+            fake_client = self.make_client_function(
+                'invoke',
+                side_effect=lambda FunctionName, Payload: self.assertEqual(
+                    Payload, u'{"key": "value"}'))
+
+            fun.client = fake_client
+            result = fun.invoke({'Payload': {"key": "value"}})
+            self.assertEqual(result, None)
+
+            self._mock_function_file()
+            fake_client = self.make_client_function(
+                'invoke',
+                side_effect=lambda FunctionName, Payload:
+                    self.assertEqual(Payload.read(), 'test'))
+            fun.client = fake_client
+            result = fun.invoke({'Payload': '/tmp/mock_function.txt'})
+            self.assertEqual(result, None)
+
     @patch(u'{0}{1}'.format(PATCH_PREFIX, '_get_iam_role_to_attach'))
     @patch(u'{0}{1}'.format(PATCH_PREFIX, '_get_security_groups_to_attach'))
     @patch(u'{0}{1}'.format(PATCH_PREFIX, '_get_subnets_to_attach'))

plugin.yaml

@@ -2,9 +2,9 @@ plugins:
 
   aws:
     executor: central_deployment_agent
-    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/2.5.6.zip
+    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/2.5.7.zip
     package_name: cloudify-aws-plugin
-    package_version: '2.5.6'
+    package_version: '2.5.7'
 
 data_types:
 
@@ -1392,9 +1392,19 @@ node_types:
       resource_config:
         description: >
           Configuration key-value data to be passed as-is to the corresponding
-          Boto3 method. Key names must match the case that Boto3 requires.
+          Boto3 method, except for dicionary containing Payload key. Payload 
+          content will be encoded according to Boto3 requirements. 
+          If the Payload value is a dictionary it will be JSON encoded and 
+          converted into bytes. If the Payload value is string it will be 
+          treated as the path for file that will be used to populate the value.
+          In other cases (integer, bool, etc.) Payload will be passed as is. 
+          Key names must match the case that Boto3 requires.
         type: cloudify.datatypes.aws.lambda.Invoke.config
         required: false
+      resource_encoding:
+        description: >
+          Encoding used to decode replies
+        default: 'utf-8'
     interfaces:
       cloudify.interfaces.lifecycle:
         configure: