From 1831491c6ab7c9d88deade14f8ab83d0690f314b Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 11:56:00 -0600 Subject: [PATCH 1/7] Collect tags from ELBv2 and RDS --- collect_commands.yaml | 15 +++++++++++++++ commands/collect.py | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/collect_commands.yaml b/collect_commands.yaml index 6718721fe..d5af4b7ba 100644 --- a/collect_commands.yaml +++ b/collect_commands.yaml @@ -109,10 +109,20 @@ Parameters: - Name: DBSnapshotIdentifier Value: rds-describe-db-snapshots.json|.DBSnapshots[]?|.DBSnapshotIdentifier +- Service: rds + Request: list-tags-for-resource + Parameters: + - Name: ResourceName + Value: rds-describe-db-instances.json|.DBInstances[]?|.DBInstanceArn - Service: elb Request: describe-load-balancers - Service: elb Request: describe-load-balancer-policies +- Service: elb + Request: describe-tags + Parameters: + - Name: LoadBalancerNames + Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|.LoadBalancerName - Service: elbv2 Request: describe-load-balancers - Service: elbv2 @@ -125,6 +135,11 @@ Parameters: - Name: TargetGroupArn Value: elbv2-describe-target-groups/*|.TargetGroups[].TargetGroupArn +- Service: elbv2 + Request: describe-tags + Parameters: + - Name: ResourceArns + Value: elbv2-describe-load-balancers.json|.LoadBalancers[]?|[[.LoadBalancerArn]] - Service: redshift Request: describe-clusters - Service: sqs diff --git a/commands/collect.py b/commands/collect.py index 3acfa786b..f0fee5df3 100644 --- a/commands/collect.py +++ b/commands/collect.py @@ -31,7 +31,13 @@ def get_identifier_from_parameter(parameter): def get_filename_from_parameter(parameter): if isinstance(parameter, list): - filename = parameter[1] + if len(parameter) > 1: + filename = parameter[1] + elif isinstance(parameter[0], list): + # For elbv2:describe-tags we need ResourceArns as a list like `[Arn]` + # the yaml file specifies `[[.LoadBalancerArn]]` because just doing + # `[.LoadBalancerArn]` presents other issues, so this extracts out the inner, inner value. + filename = parameter[0][0] else: filename = parameter From f268334ca1fa0e47f7c563fa60a5d24732070eb3 Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 12:00:02 -0600 Subject: [PATCH 2/7] Fix tag collection for classic ELB --- collect_commands.yaml | 2 +- commands/collect.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/collect_commands.yaml b/collect_commands.yaml index d5af4b7ba..f66c6e001 100644 --- a/collect_commands.yaml +++ b/collect_commands.yaml @@ -122,7 +122,7 @@ Request: describe-tags Parameters: - Name: LoadBalancerNames - Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|.LoadBalancerName + Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|[[.LoadBalancerName]] - Service: elbv2 Request: describe-load-balancers - Service: elbv2 diff --git a/commands/collect.py b/commands/collect.py index f0fee5df3..8d572e37a 100644 --- a/commands/collect.py +++ b/commands/collect.py @@ -37,6 +37,7 @@ def get_filename_from_parameter(parameter): # For elbv2:describe-tags we need ResourceArns as a list like `[Arn]` # the yaml file specifies `[[.LoadBalancerArn]]` because just doing # `[.LoadBalancerArn]` presents other issues, so this extracts out the inner, inner value. + # Similar issue for elb:describe-tags filename = parameter[0][0] else: filename = parameter From 0fb86a0a89f31b315d7e3101b01b409af13e4270 Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 12:05:56 -0600 Subject: [PATCH 3/7] Update pylint --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index b9240fafd..d7d7e9a7f 100644 --- a/Pipfile +++ b/Pipfile @@ -22,7 +22,7 @@ autoflake = "==0.7" nose = "==1.3.7" coverage = "==4.4.2" mock = "==2.0.0" -pylint = "==1.8.1" +pylint = "==2.3.1" [requires] python_version = "3.7" From ac5f9e8f102187886d12d3f68865aeb267d982c6 Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 12:45:15 -0600 Subject: [PATCH 4/7] Filter RDS by tags --- commands/prepare.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/commands/prepare.py b/commands/prepare.py index 96e20d60d..388081423 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -28,7 +28,7 @@ import argparse import pyjq from netaddr import IPNetwork, IPAddress -from shared.common import get_account, query_aws, get_regions, is_external_cidr +from shared.common import get_account, query_aws, get_parameter_file, get_regions, is_external_cidr from shared.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Rds, Cidr, Connection __description__ = "Generate network connection information file" @@ -70,6 +70,7 @@ def get_subnets(az): def get_ec2s(subnet, outputfilter): + # Filter the EC2s by the `tags` tag_filter = "" tag_set_conditions = [] for tag_set in outputfilter.get("tags", []): @@ -88,7 +89,7 @@ def get_ec2s(subnet, outputfilter): return pyjq.all(resource_filter.format(subnet.local_id), instances) -def get_elbs(subnet): +def get_elbs(subnet, outputfilter): # ELBs elb_instances = query_aws(subnet.account, "elb-describe-load-balancers", subnet.region) elb_resource_filter = '.LoadBalancerDescriptions[] | select(.VPCId == "{}") | select(.Subnets[] == "{}")' @@ -102,10 +103,36 @@ def get_elbs(subnet): return elbs + albs -def get_rds_instances(subnet): +def get_rds_instances(subnet, outputfilter): instances = query_aws(subnet.account, "rds-describe-db-instances", subnet.region) resource_filter = '.DBInstances[] | select(.DBSubnetGroup.Subnets != null and .DBSubnetGroup.Subnets[].SubnetIdentifier == "{}")' - return pyjq.all(resource_filter.format(subnet.local_id), instances) + rds_instances = pyjq.all(resource_filter.format(subnet.local_id), instances) + + if 'tags' not in outputfilter: + return rds_instances + + # There are tags requested, so we need to filter these + tag_filter = "" + tag_set_conditions = [] + for tag_set in outputfilter.get("tags", []): + conditions = [c.split("=") for c in tag_set.split(",")] + condition_queries = [] + for pair in conditions: + if len(pair) == 2: + condition_queries.append('.{} == "{}"'.format(pair[0], pair[1])) + tag_set_conditions.append('(' + ' and '.join(condition_queries) + ')') + tag_filter = 'select(.TagList | from_entries | ' + ' or '.join(tag_set_conditions) + ')' + + filtered_instances = [] + for rds in rds_instances: + tags = get_parameter_file(subnet.region, 'rds', 'list-tags-for-resource', rds['DBInstanceArn']) + if tags is None: + continue + + if pyjq.first(tag_filter, tags) is not None: + filtered_instances.append(rds) + + return filtered_instances def get_sgs(vpc): @@ -254,14 +281,14 @@ def build_data_structure(account_data, config, outputfilter): subnet.addChild(ec2) # Get RDS's - for rds_json in get_rds_instances(subnet): + for rds_json in get_rds_instances(subnet, outputfilter): rds = Rds(subnet, rds_json) if not outputfilter["read_replicas"] and rds.node_type == "rds_rr": continue subnet.addChild(rds) # Get ELB's - for elb_json in get_elbs(subnet): + for elb_json in get_elbs(subnet, outputfilter): elb = Elb(subnet, elb_json) subnet.addChild(elb) From 4bb7e2d8ce91cb759b111e81b02d47cd4c46a0ef Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 13:33:03 -0600 Subject: [PATCH 5/7] Filtering for elb and elbv2s --- commands/prepare.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/commands/prepare.py b/commands/prepare.py index 388081423..313fb212a 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -100,7 +100,39 @@ def get_elbs(subnet, outputfilter): alb_resource_filter = '.LoadBalancers[] | select(.VpcId == "{}") | select(.AvailabilityZones[].SubnetId == "{}")' albs = pyjq.all(alb_resource_filter.format(subnet.vpc.local_id, subnet.local_id), alb_instances) - return elbs + albs + if 'tags' not in outputfilter: + return elbs + albs + + # There are tags requested, so we need to filter these + tag_filter = "" + tag_set_conditions = [] + for tag_set in outputfilter.get("tags", []): + conditions = [c.split("=") for c in tag_set.split(",")] + condition_queries = [] + for pair in conditions: + if len(pair) == 2: + condition_queries.append('.{} == "{}"'.format(pair[0], pair[1])) + tag_set_conditions.append('(' + ' and '.join(condition_queries) + ')') + tag_filter = 'select(.TagDescriptions[0].Tags | from_entries | ' + ' or '.join(tag_set_conditions) + ')' + + filtered_elbs = [] + for elb in elbs: + tags = get_parameter_file(subnet.region, 'elb', 'describe-tags', elb['LoadBalancerName']) + if tags is None: + continue + + if pyjq.first(tag_filter, tags) is not None: + filtered_elbs.append(elb) + + for elb in albs: + tags = get_parameter_file(subnet.region, 'elbv2', 'describe-tags', elb['LoadBalancerArn']) + if tags is None: + continue + + if pyjq.first(tag_filter, tags) is not None: + filtered_elbs.append(elb) + + return filtered_elbs def get_rds_instances(subnet, outputfilter): From 504706be8d076559e1c3385ad467a8f06c9a5f1f Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 13:36:34 -0600 Subject: [PATCH 6/7] Docs updated for tags --- docs/network_visualizations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/network_visualizations.md b/docs/network_visualizations.md index 03e060c96..fc6079d6e 100644 --- a/docs/network_visualizations.md +++ b/docs/network_visualizations.md @@ -14,6 +14,7 @@ There are a number of filtering options that can be applied here to reduce the n The most useful filtering options: * `--regions`: Restrict the diagram to a set regions, ex. `us-east-1,us-east-2` * `--vpc-ids` and `--vpc-names`: Restrict the diagram to a set of VPCs. +* `--tags`: Filter by tags, for exmaple `--tags Env=Prod --tags Env=Test,Name=Bastion` will filter to all resources tagged with a key `Env` that has value `Prod`, or where `Env=Test` and `Name=Bastion`. In this way, a the tags in a set are AND'd, and the tag sets are OR'd. * `--collapse-by-tag`: This is very useful to provide a tag name, and all nodes with that tag will be reduced to a single displayed node. The other filtering options are: From 2021671a2fe340a1e067a8aff45eb1614e95ff94 Mon Sep 17 00:00:00 2001 From: Scott Piper Date: Wed, 17 Apr 2019 13:36:43 -0600 Subject: [PATCH 7/7] Version bump --- cloudmapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudmapper.py b/cloudmapper.py index 513ffbe21..305bae127 100755 --- a/cloudmapper.py +++ b/cloudmapper.py @@ -31,7 +31,7 @@ import importlib import commands -__version__ = "2.5.2" +__version__ = "2.5.3" def show_help(commands): print("CloudMapper {}".format(__version__))