From c595072aaccbd2f9ec4f5d80ab82e3d1c33feb7d Mon Sep 17 00:00:00 2001 From: kaplanyaniv Date: Wed, 3 Nov 2021 17:35:33 +0200 Subject: [PATCH 1/2] added use_external_resource to plugin.yaml added if condition to the prepare function in image.py need to add else function --- cloudify_aws/ec2/resources/image.py | 17 ++++++++++------- plugin.yaml | 4 ++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cloudify_aws/ec2/resources/image.py b/cloudify_aws/ec2/resources/image.py index 8880e7f8..0429a271 100644 --- a/cloudify_aws/ec2/resources/image.py +++ b/cloudify_aws/ec2/resources/image.py @@ -95,10 +95,13 @@ def prepare_describe_image_filter(params, iface): def prepare(ctx, iface, resource_config, **_): """Prepares an AWS EC2 Image""" # Save the parameters - ctx.instance.runtime_properties['resource_config'] = resource_config - iface = \ - prepare_describe_image_filter( - resource_config.copy(), - iface) - ami = iface.properties - utils.update_resource_id(ctx.instance, ami.get(IMAGE_ID)) + if ctx.instance.runtime_properties['use_external_resource']: + ctx.instance.runtime_properties['resource_config'] = resource_config + iface = \ + prepare_describe_image_filter( + resource_config.copy(), + iface) + ami = iface.properties + utils.update_resource_id(ctx.instance, ami.get(IMAGE_ID)) + else: + diff --git a/plugin.yaml b/plugin.yaml index 676e22ed..3d22c52e 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2981,6 +2981,10 @@ node_types: cloudify.nodes.aws.ec2.Image: derived_from: cloudify.nodes.Root properties: + use_external_resource: + type: boolean + default: true + description: indecates if external_resourc should be used <<: *external_resource <<: *client_config <<: *resource_id From 02bce31a433423e067fab6ea71e4ec0e0be6ad2d Mon Sep 17 00:00:00 2001 From: kaplanyaniv Date: Wed, 10 Nov 2021 13:10:40 +0200 Subject: [PATCH 2/2] RD-3358-create-ami-from-ec2 Create AMI from running instance --- CHANGELOG.txt | 2 + cloudify_aws/common/decorators.py | 10 +- cloudify_aws/common/utils.py | 20 ++ cloudify_aws/ec2/resources/image.py | 94 ++++-- cloudify_aws/ec2/tests/test_image.py | 20 +- .../blueprint.yaml | 282 ++++++++++++++++++ plugin.yaml | 112 ++++++- 7 files changed, 508 insertions(+), 32 deletions(-) create mode 100644 examples/ec2-create-image-feature-demo/blueprint.yaml diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9b5d9f02..71d24f73 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -175,3 +175,5 @@ - RD-3369-Return Aarch64 Wagons to Bundle 2.12.13: - RD-3565: Fix Populate Resource issue. +2.12.14: + - RD-3358: Added option to create an EC2 ami image. diff --git a/cloudify_aws/common/decorators.py b/cloudify_aws/common/decorators.py index f88e9845..ab5c2872 100644 --- a/cloudify_aws/common/decorators.py +++ b/cloudify_aws/common/decorators.py @@ -338,6 +338,10 @@ def _aws_resource(function, resource_config = kwargs['resource_config'] resource_id = utils.get_resource_id(node=ctx.node, instance=ctx.instance) # Check if using external + if props.get('use_external_resource') and \ + 'cloudify.nodes.aws.ec2.Image' in ctx.node.type_hierarchy and \ + operation_name == 'create': + pass iface = kwargs.get('iface') if ctx.node.properties.get('use_external_resource', False): ctx.logger.info('{t} ID# {i} is user-provided.'.format( @@ -362,12 +366,12 @@ def _aws_resource(function, return ctx.logger.warn('{t} ID# {i} has force_operation set.'.format( t=resource_type, i=resource_id)) + ctx.logger.debug('Executing: {} with params {}'.format(function, kwargs)) result = function(**kwargs) - if ctx.operation.name == 'cloudify.interfaces.lifecycle.configure' \ - and iface: + if operation_name == 'configure' and iface: iface.populate_resource(ctx) kwargs['iface'] = iface - if ctx.operation.name == 'cloudify.interfaces.lifecycle.delete': + elif operation_name == 'delete': # cleanup runtime after delete keys = list(ctx.instance.runtime_properties.keys()) for key in keys: diff --git a/cloudify_aws/common/utils.py b/cloudify_aws/common/utils.py index 03c385c4..4af2fca6 100644 --- a/cloudify_aws/common/utils.py +++ b/cloudify_aws/common/utils.py @@ -505,6 +505,7 @@ def check_availability_zone(zone): def clean_params(p): + p = dict() if not p else p.copy() if not isinstance(p, dict) or not p: return {} for _k, _v in list(p.items()): @@ -904,3 +905,22 @@ def get_node_instances_by_type_related_to_node_name(node_name, if node_type in node.type_hierarchy and node_name in rels: nodes.append({'node_instance': ni, 'node': node}) return nodes + + +def clean_empty_vals(params): + if isinstance(params, dict): + new_params = {} + for key, val in params.items(): + if isinstance(val, dict) or isinstance(val, list): + val = clean_empty_vals(val) + if val: + new_params[key] = val + return new_params + if isinstance(params, list): + new_params = [] + for val in params: + if isinstance(val, dict) or isinstance(val, list): + val = clean_empty_vals(val) + if val: + new_params.append(val) + return new_params diff --git a/cloudify_aws/ec2/resources/image.py b/cloudify_aws/ec2/resources/image.py index 0429a271..778c5223 100644 --- a/cloudify_aws/ec2/resources/image.py +++ b/cloudify_aws/ec2/resources/image.py @@ -40,15 +40,24 @@ class EC2Image(EC2Base): def __init__(self, ctx_node, resource_id=None, client=None, logger=None): EC2Base.__init__(self, ctx_node, resource_id, client, logger) self.type_name = RESOURCE_TYPE - self.describe_image_filters = {} + image_filters = ctx_node.properties["resource_config"].get( + "kwargs", {}).get("Filters") + self.describe_image_filters = None + if resource_id: + self.prepare_describe_image_filter({IMAGE_IDS: [resource_id]}) + elif image_filters: + self.prepare_describe_image_filter({FILTERS: image_filters}) @property def properties(self): """Gets the properties of an external resource""" params = self.describe_image_filters + if not params: + return try: resources = \ self.client.describe_images(**params) + self.logger.info('Describe images returned: {}'.format(resources)) except (ClientError, ParamValidationError): pass else: @@ -61,7 +70,13 @@ def properties(self): @property def status(self): """Gets the status of an external resource""" - props = self.properties + try: + props = self.properties + except NonRecoverableError as e: + if 'Found no AMIs matching provided filters' in str(e): + props = None + else: + raise e if not props: return None return props['State'] @@ -77,31 +92,68 @@ def create(self, params): return res def delete(self, params=None): - return - + self.logger.debug('Deleting %s' % self.type_name) + self.logger.debug('Deregistering ImageId %s' % params.get('ImageId')) + self.client.deregister_image(**params) -def prepare_describe_image_filter(params, iface): - iface.describe_image_filters = { - DRY_RUN: params.get(DRY_RUN, False), - IMAGE_IDS: params.get(IMAGE_IDS, []), - OWNERS: params.get(OWNERS, []), - EXECUTABLE_USERS: params.get(EXECUTABLE_USERS, []), - FILTERS: params.get(FILTERS, []) - } - return iface + def prepare_describe_image_filter(self, params): + dry_run = params.get(DRY_RUN, False) + image_ids = params.get(IMAGE_IDS, []) + owners = params.get(OWNERS, []) + executable_users = params.get(EXECUTABLE_USERS, []) + filters = params.get(FILTERS, []) + if any([dry_run, image_ids, owners, executable_users, filters]): + self.describe_image_filters = { + DRY_RUN: dry_run, + IMAGE_IDS: image_ids, + OWNERS: owners, + EXECUTABLE_USERS: executable_users, + FILTERS: filters} + self.logger.debug('Updated image filter: {}'.format( + self.describe_image_filters)) @decorators.aws_resource(EC2Image, resource_type=RESOURCE_TYPE) def prepare(ctx, iface, resource_config, **_): """Prepares an AWS EC2 Image""" # Save the parameters - if ctx.instance.runtime_properties['use_external_resource']: + if ctx.node.properties.get('use_external_resource'): ctx.instance.runtime_properties['resource_config'] = resource_config - iface = \ - prepare_describe_image_filter( - resource_config.copy(), - iface) - ami = iface.properties - utils.update_resource_id(ctx.instance, ami.get(IMAGE_ID)) - else: + iface.prepare_describe_image_filter(resource_config) + utils.update_resource_id(ctx.instance, iface.properties.get(IMAGE_ID)) + + +@decorators.aws_resource(EC2Image, resource_type=RESOURCE_TYPE) +@decorators.wait_for_status(status_good=['available'], fail_on_missing=False) +def create(ctx, iface, resource_config, **_): + """Create an AWS EC2 Image""" + if ctx.instance.runtime_properties.get('use_external_resource'): + # if use_external_resource there we are using an existing image + return + + params = utils.clean_params(resource_config) + if 'InstanceId' not in params: + params['InstanceId'] = utils.find_resource_id_by_type( + ctx.instance, 'cloudify.nodes.aws.ec2.Instances') + params = utils.clean_empty_vals(params) + + # Actually create the resource + create_response = iface.create(params) + ctx.instance.runtime_properties['create_response'] = \ + utils.JsonCleanuper(create_response).to_dict() + iface.update_resource_id(create_response['ImageId']) + utils.update_resource_id(ctx.instance, create_response['ImageId']) + +@decorators.aws_resource(EC2Image, resource_type=RESOURCE_TYPE) +@decorators.wait_for_delete(status_deleted=['deregistered']) +def delete(ctx, iface, resource_config, **_): + """delete/deregister an AWS EC2 Image""" + if not ctx.instance.runtime_properties.get('use_external_resource'): + params = {'ImageId': iface.resource_id} + try: + iface.delete(params) + except ClientError as e: + if 'is no longer available' in str(e): + return + raise e diff --git a/cloudify_aws/ec2/tests/test_image.py b/cloudify_aws/ec2/tests/test_image.py index 05b4bcc0..04b33c81 100644 --- a/cloudify_aws/ec2/tests/test_image.py +++ b/cloudify_aws/ec2/tests/test_image.py @@ -37,12 +37,21 @@ class TestEC2Image(TestBase): def setUp(self): - self.image = EC2Image("ctx_node", resource_id=True, - client=True, logger=None) + ctx_node_mock = MagicMock( + properties={'resource_config': {'Name': 'foo'}}) + mock_client = MagicMock() + self.image = EC2Image(ctx_node_mock, + resource_id=True, + client=mock_client, + logger=None) mock1 = patch('cloudify_aws.common.decorators.aws_resource', mock_decorator) mock1.start() reload_module(image) + self.image.describe_image_filters = \ + {'Filters': { + 'name': 'CentOS 7.7.1908 x86_64 with cloud-init (HVM)', + 'owner-id': '057448758665'}} def test_class_properties(self): effect = self.get_client_error_exception(name='EC2 Image') @@ -67,12 +76,10 @@ def test_class_properties(self): def test_class_status(self): value = {} + self.image.client = self.make_client_function('describe_images', return_value=value) - with self.assertRaises(NonRecoverableError) as e: - self.image.status - self.assertEqual(text_type(e.exception), - u"Found no AMIs matching provided filters.") + self.assertIsNone(self.image.status) value = {IMAGES: [None]} self.image.client = self.make_client_function('describe_images', @@ -97,6 +104,7 @@ def test_prepare(self): ctx = self.get_mock_ctx("Image") config = {IMAGE_ID: 'image', OWNERS: 'owner'} iface = MagicMock() + ctx.node.properties['use_external_resource'] = True iface.create = self.mock_return(config) image.prepare(ctx, iface, config) self.assertEqual(ctx.instance.runtime_properties['resource_config'], diff --git a/examples/ec2-create-image-feature-demo/blueprint.yaml b/examples/ec2-create-image-feature-demo/blueprint.yaml new file mode 100644 index 00000000..89bbcd7f --- /dev/null +++ b/examples/ec2-create-image-feature-demo/blueprint.yaml @@ -0,0 +1,282 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +description: > + This blueprint creates an AWS infrastructure environment. + +imports: + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - plugin:cloudify-aws-plugin + - plugin:cloudify-utilities-plugin?version= >=1.22.1 + +inputs: + + aws_region_name: + type: string + default: 'eu-west-1' + + availability_zone: + type: string + description: The availability zone in the AWS Region. + default: { concat: [ { get_input: aws_region_name }, 'b' ] } + + ami_owner_filter: + type: string + description: The AWS AMI owner number. + default: '057448758665' + + ami_name_filter: + type: string + description: The name of the AWS AMI in the AWS region. + default: 'CentOS 7.7.1908 x86_64 with cloud-init (HVM)' + + instance_type: + type: string + default: t2.micro + + agent_user: + description: > + The username of the agent running on the instance created from the image. + default: 'centos' + + agent_key_name: + type: string + default: agent_key + + env_name: + type: string + description: Control parameters for names in resources. + default: 'example' + + ami_image_name: + type: string + description: Control parameters for names in resources. + default: 'my first ami' + +dsl_definitions: + + client_config: &client_config + aws_access_key_id: { get_secret: aws_access_key_id } + aws_secret_access_key: { get_secret: aws_secret_access_key } + region_name: { get_input: aws_region_name } + +node_templates: + + ami_to_create: + type: cloudify.nodes.aws.ec2.Image + properties: + use_external_resource: false + resource_config: + InstanceId: {get_attribute: [vm, aws_resource_id]} + Name: { get_input: ami_image_name } + client_config: *client_config + relationships: + - type: cloudify.relationships.depends_on + target: vm + + vm: + type: cloudify.nodes.aws.ec2.Instances + properties: + client_config: *client_config + agent_config: + install_method: none + user: { get_input: agent_user } + key: { get_attribute: [agent_key, private_key_export] } + resource_config: + ImageId: { get_attribute: [ ami, aws_resource_id ] } + InstanceType: { get_input: instance_type } + kwargs: + UserData: { get_attribute: [ cloud_init, cloud_config ] } + TagSpecifications: + - Tags: + - Key: Name + Value: { get_input: env_name } + use_public_ip: true + relationships: + - type: cloudify.relationships.depends_on + target: ami + - type: cloudify.relationships.depends_on + target: nic + - type: cloudify.relationships.depends_on + target: ip + - type: cloudify.relationships.depends_on + target: cloud_init + + ami: + type: cloudify.nodes.aws.ec2.Image + properties: + resource_config: + kwargs: + Filters: + - Name: name + Values: + - { get_input: ami_name_filter } + - Name: owner-id + Values: + - { get_input: ami_owner_filter } + client_config: *client_config + + ip: + type: cloudify.nodes.aws.ec2.ElasticIP + properties: + client_config: *client_config + relationships: + - type: cloudify.relationships.depends_on + target: nic + + nic: + type: cloudify.nodes.aws.ec2.Interface + properties: + client_config: *client_config + resource_config: + kwargs: + Description: Created by cloudify-getting-started-example. + SubnetId: { get_attribute: [ subnet, aws_resource_id ] } + Groups: + - { get_attribute: [ security_group, aws_resource_id ] } + relationships: + - type: cloudify.relationships.depends_on + target: security_group + - type: cloudify.relationships.depends_on + target: subnet + + security_group_rules: + type: cloudify.nodes.aws.ec2.SecurityGroupRuleIngress + properties: + client_config: *client_config + resource_config: + IpPermissions: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + IpRanges: + - CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + IpRanges: + - CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + IpRanges: + - CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 9990 + ToPort: 9990 + IpRanges: + - CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 2375 + ToPort: 2375 + IpRanges: + - CidrIp: 0.0.0.0/0 + relationships: + - type: cloudify.relationships.contained_in + target: security_group + + security_group: + type: cloudify.nodes.aws.ec2.SecurityGroup + properties: + client_config: *client_config + resource_config: + GroupName: CloudifyGettingStartedExample + Description: Created by cloudify-getting-started-example. + VpcId: { get_attribute: [ vpc, aws_resource_id ] } + relationships: + - type: cloudify.relationships.depends_on + target: vpc + + route_public_subnet_internet_gateway: + type: cloudify.nodes.aws.ec2.Route + properties: + resource_config: + kwargs: + DestinationCidrBlock: '0.0.0.0/0' + client_config: *client_config + relationships: + - type: cloudify.relationships.contained_in + target: routetable + - type: cloudify.relationships.connected_to + target: internet_gateway + interfaces: + cloudify.interfaces.lifecycle: + stop: {} + + routetable: + type: cloudify.nodes.aws.ec2.RouteTable + properties: + client_config: *client_config + relationships: + - type: cloudify.relationships.contained_in + target: vpc + - type: cloudify.relationships.connected_to + target: subnet + + subnet: + type: cloudify.nodes.aws.ec2.Subnet + properties: + client_config: *client_config + resource_config: + CidrBlock: 10.10.4.0/24 + AvailabilityZone: { get_input: availability_zone } + relationships: + - type: cloudify.relationships.depends_on + target: vpc + + internet_gateway: + type: cloudify.nodes.aws.ec2.InternetGateway + properties: + client_config: *client_config + relationships: + - type: cloudify.relationships.connected_to + target: vpc + + vpc: + type: cloudify.nodes.aws.ec2.Vpc + properties: + client_config: *client_config + resource_config: + CidrBlock: 10.10.0.0/16 + + cloud_init: + type: cloudify.nodes.CloudInit.CloudConfig + properties: + resource_config: + users: + - name: { get_input: agent_user } + shell: /bin/bash + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + ssh-authorized-keys: + - { get_attribute: [agent_key, public_key_export] } + relationships: + - type: cloudify.relationships.depends_on + target: agent_key + + agent_key: + type: cloudify.keys.nodes.RSAKey + properties: + resource_config: + key_name: { get_input: agent_key_name } + openssh_format: true + use_secret_store: true + use_secrets_if_exist: true + interfaces: + cloudify.interfaces.lifecycle: + create: + implementation: keys.cloudify_ssh_key.operations.create + inputs: + store_private_key_material: true + +capabilities: + + endpoint: + description: The external endpoint of the application. + value: { get_attribute: [ ip, aws_resource_id ] } + + user: + description: user ID. + value: { get_input: agent_user } + + key_content: + description: Private agent key + value: { get_attribute: [agent_key, private_key_export] } diff --git a/plugin.yaml b/plugin.yaml index 3d22c52e..e873c99c 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2,7 +2,7 @@ plugins: aws: executor: central_deployment_agent package_name: cloudify-aws-plugin - package_version: '2.12.13' + package_version: '2.12.14' data_types: @@ -871,12 +871,115 @@ data_types: description: http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.create_route default: {} + cloudify.datatypes.aws.ec2.Image.config.create.BlockDeviceMappings: + properties: + DeviceName: + type: string + required: false + description: ~ + VirtualName: + type: string + required: false + description: ~ + Ebs: + type: cloudify.datatypes.aws.ec2.Image.config.create.BlockDeviceMappings.Ebs + required: false + description: ~ + NoDevice: + type: string + required: false + description: ~ + kwargs: + description: ~ + default: { } + cloudify.datatypes.aws.ec2.Image.config: properties: + BlockDeviceMappings: + type: cloudify.datatypes.aws.ec2.Image.config.create.BlockDeviceMappings + required: false + description: list contaning a dict of arguments to create an ami. + Description: + type: string + required: false + description: The description of the created image. + DryRun: + type: boolean + required: false + description: ~ + InstanceId: + type: string + required: false + description: The InstanceId to create an image from. + Name: + type: string + required: false + description: The name of the image. + NoReboot: + type: boolean + required: false + description: ~ + TagSpecifications: + type: cloudify.datatypes.aws.TagSpecifications + required: false + description: ~ kwargs: description: "https://docs.cloudify.co/latest/working_with/official_plugins/infrastructure/aws/#cloudify-nodes-aws-ec2-image" default: {} + cloudify.datatypes.aws.ec2.Image.config.create.BlockDeviceMappings.Ebs: + properties: + DeleteOnTermination: + type: boolean + required: false + description: ~ + Iops: + type: integer + required: false + description: ~ + SnapshotId: + type: string + required: false + description: ~ + VolumeSize: + type: integer + required: false + description: ~ + VolumeType: + type: string + required: false + description: ~ + KmsKeyId: + type: string + required: false + description: + Throughput: + type: integer + required: false + description: ~ + OutpostArn: + type: string + required: false + description: ~ + Encrypted: + type: boolean + required: false + description: ~ + kwargs: + description: ~ + default: { } + + cloudify.datatypes.aws.TagSpecifications: + properties: + ResourceType: + type: string + required: false + description: ~ + Tags: + type: list + required: false + + cloudify.datatypes.aws.ec2.Tags.config: properties: kwargs: @@ -2985,7 +3088,6 @@ node_types: type: boolean default: true description: indecates if external_resourc should be used - <<: *external_resource <<: *client_config <<: *resource_id resource_config: @@ -2999,6 +3101,12 @@ node_types: create: implementation: aws.cloudify_aws.ec2.resources.image.prepare inputs: *operation_inputs + configure: + implementation: aws.cloudify_aws.ec2.resources.image.create + inputs: *operation_inputs + delete: + implementation: aws.cloudify_aws.ec2.resources.image.delete + inputs: *operation_inputs cloudify.nodes.aws.ec2.Tags: derived_from: cloudify.nodes.Root