From 76bec2daf9db281268838985d9ee309f6a01b88c Mon Sep 17 00:00:00 2001 From: tzarski0 <92273798+tzarski0@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:54:55 +0100 Subject: [PATCH 1/4] Update endpoints_sdwan.yaml --- examples/endpoints_sdwan.yaml | 339 +++++++++++++++++++++++++--------- 1 file changed, 256 insertions(+), 83 deletions(-) diff --git a/examples/endpoints_sdwan.yaml b/examples/endpoints_sdwan.yaml index bf7daab..b5d3e2e 100644 --- a/examples/endpoints_sdwan.yaml +++ b/examples/endpoints_sdwan.yaml @@ -1,5 +1,250 @@ - name: feature_templates endpoint: /template/feature/object/%i +- name: transport_feature_profile + endpoint: /v1/feature-profile/sdwan/transport + children: + - name: transport_tracker + endpoint: /tracker + - name: transport_routing_bgp + endpoint: /routing/bgp + - name: transport_wan_vpn + endpoint: /wan/vpn + children: + - name: transport_wan_vpn_interface_cellular + endpoint: /interface/cellular + - name: transport_wan_vpn_interface_gre + endpoint: /interface/gre + - name: transport_wan_vpn_interface_t1_e1_serial + endpoint: /interface/serial + - name: transport_wan_vpn_interface_ipsec + endpoint: /interface/ipsec + - name: transport_wan_vpn_interface_ethernet + endpoint: /interface/ethernet + children: + - name: transport_wan_vpn_interface_ethernet_feature_associate_ipv6_tracker_group + endpoint: /ipv6-trackergroup + - name: transport_wan_vpn_interface_ethernet_feature_associate_tracker_group + endpoint: /trackergroup + - name: transport_wan_vpn_interface_ethernet_feature_associate_tracker + endpoint: /tracker + - name: transport_wan_vpn_interface_ethernet_feature_associate_ipv6_tracker + endpoint: /ipv6-tracker + - name: transport_wan_vpn_feature_associate_routing_bgp + endpoint: /routing/bgp + - name: transport_wan_vpn_feature_associate_routing_ospfv3_ipv4 + endpoint: /routing/ospfv3/ipv4 + - name: transport_wan_vpn_feature_associate_routing_ospf + endpoint: /routing/ospf + - name: transport_wan_vpn_feature_associate_routing_ospfv3_ipv6 + endpoint: /routing/ospfv3/ipv6 + - name: transport_routing_ospfv3_ipv4 + endpoint: /routing/ospfv3/ipv4 + - name: transport_t1_e1_controller + endpoint: /t1-e1-controller + - name: transport_gps + endpoint: /gps + - name: transport_routing_ospf + endpoint: /routing/ospf + - name: transport_cellular_controller + endpoint: /cellular-controller + - name: transport_ipv4_acl + endpoint: /ipv4-acl + - name: transport_management_vpn + endpoint: /management/vpn + children: + - name: transport_management_vpn_interface_ethernet + endpoint: /interface/ethernet + - name: transport_cellular_profile + endpoint: /cellular-profile + - name: transport_ipv6_tracker_group + endpoint: /ipv6-trackergroup + - name: transport_route_policy + endpoint: /route-policy + - name: transport_ipv6_acl + endpoint: /ipv6-acl + - name: transport_routing_ospfv3_ipv6 + endpoint: /routing/ospfv3/ipv6 + - name: transport_ipv6_tracker + endpoint: /ipv6-tracker + - name: transport_tracker_group + endpoint: /trackergroup +- name: service_feature_profile + endpoint: /v1/feature-profile/sdwan/service + children: + - name: service_routing_ospf + endpoint: /routing/ospf + - name: service_lan_vpn + endpoint: /lan/vpn + children: + - name: service_lan_vpn_interface_ethernet + endpoint: /interface/ethernet + children: + - name: service_lan_vpn_ethernet_interface_feature_associate_dhcp_server + endpoint: /dhcp-server + - name: service_lan_vpn_interface_ipsec + endpoint: /interface/ipsec + children: + - name: service_lan_vpn_ethernet_ipsec_feature_associate_dhcp_server + endpoint: /dhcp-server + - name: service_lan_vpn_interface_gre + endpoint: /interface/gre + - name: service_lan_vpn_interface_svi + endpoint: /interface/svi + children: + - name: service_lan_vpn_ethernet_svi_feature_associate_dhcp_server + endpoint: /dhcp-server + - name: service_lan_vpn_feature_associate_routing_bgp + endpoint: /routing/bgp + - name: service_lan_vpn_feature_associate_routing_ospfv3_ipv4 + endpoint: /routing/ospfv3/ipv4 + - name: service_lan_vpn_feature_associate_routing_ospfv3_ipv6 + endpoint: /routing/ospfv3/ipv6 + - name: service_lan_vpn_feature_associate_routing_ospf + endpoint: /routing/ospf + - name: service_lan_vpn_feature_associate_routing_eigrp + endpoint: /routing/eigrp + - name: service_lan_vpn_feature_associate_mulitcast + endpoint: /routing/multicast + - name: service_switchport + endpoint: /switchport + - name: service_routing_ospfv3_ipv4 + endpoint: /routing/ospfv3/ipv4 + - name: service_wireless_lan + endpoint: /wirelesslan + - name: service_ipv4_acl + endpoint: /ipv4-acl + - name: service_object_tracker + endpoint: /objecttracker + - name: service_object_tracker_group + endpoint: /objecttrackergroup + - name: service_multicast + endpoint: /routing/multicast + - name: service_routing_eigrp + endpoint: /routing/eigrp + - name: service_dhcp_server + endpoint: /dhcp-server + - name: service_ipv6_acl + endpoint: /ipv6-acl + - name: service_tracker_group + endpoint: /trackergroup + - name: service_tracker + endpoint: /tracker + - name: service_routing_bgp + endpoint: /routing/bgp + - name: service_routing_ospfv3_ipv6 + endpoint: /routing/ospfv3/ipv6 + - name: service_route_policy + endpoint: /route-policy +- name: system_feature_profile + endpoint: /v1/feature-profile/sdwan/system + children: + - name: system_banner + endpoint: /banner + - name: system_ipv4_device_access + endpoint: /ipv4-device-access-policy + - name: system_omp + endpoint: /omp + - name: system_bfd + endpoint: /bfd + - name: system_remote_access + endpoint: /remote-access + - name: system_aaa + endpoint: /aaa + - name: system_security + endpoint: /security + - name: system_snmp + endpoint: /snmp + - name: system_logging + endpoint: /logging + - name: system_ntp + endpoint: /ntp + - name: system_perfomance_monitoring + endpoint: /perfmonitor + - name: system_ipv6_device_access + endpoint: /ipv6-device-access-policy + - name: system_global + endpoint: /global + - name: system_mrf + endpoint: /mrf + - name: system_basic + endpoint: /basic + - name: system_flexible_port_speed + endpoint: /flexible-port-speed +- name: policy_object_feature_profile + endpoint: /v1/feature-profile/sdwan/policy-object + children: + - name: policy_object_security_url_block_list + endpoint: /security-urllist + - name: policy_object_security_url_allow_list + endpoint: /security-urllist + - name: policy_object_security_ips_signature + endpoint: /security-ipssignature + - name: policy_object_security_port_list + endpoint: /security-port + - name: policy_object_app_probe_class + endpoint: /app-probe + - name: policy_object_ipv6_prefix_list + endpoint: /ipv6-prefix + - name: policy_object_standard_community_list + endpoint: /standard-community + - name: policy_object_data_ipv6_prefix_list + endpoint: /data-ipv6-prefix + - name: policy_object_tloc_list + endpoint: /tloc + - name: policy_object_as_path_list + endpoint: /as-path + - name: policy_object_policer + endpoint: /policer + - name: policy_object_security_fqdn_list + endpoint: /security-fqdn + - name: policy_object_security_data_ipv4_prefix_list + endpoint: /security-data-ip-prefix + - name: policy_object_security_identity_list + endpoint: /security-identity + - name: policy_object_vpn_group + endpoint: /vpn-group + - name: policy_object_preferred_color_group + endpoint: /preferred-color-group + - name: policy_object_ipv4_prefix_list + endpoint: /prefix + - name: policy_object_security_scalable_group_tag_list + endpoint: /security-scalablegrouptag + - name: policy_object_security_local_domain_list + endpoint: /security-localdomain + - name: policy_object_mirror + endpoint: /mirror + - name: policy_object_extended_community_list + endpoint: /ext-community + - name: policy_object_application_list + endpoint: /app-list + - name: policy_object_color_list + endpoint: /color + - name: policy_object_security_geolocation_list + endpoint: /security-geolocation + - name: policy_object_class_map + endpoint: /class + - name: policy_object_data_ipv4_prefix_list + endpoint: /data-prefix + - name: policy_object_expanded_community_list + endpoint: /expanded-community + - name: policy_object_security_local_application_list + endpoint: /security-localapp + - name: policy_object_sla_class_list + endpoint: /sla-class +- name: application_priority_feature_profile + endpoint: /v1/feature-profile/sdwan/application-priority + children: + - name: application_priority_qos + endpoint: /qos-policy + - name: application_priority_traffic_policy + endpoint: /traffic-policy +- name: other_feature_profile + endpoint: /v1/feature-profile/sdwan/other + children: + - name: other_ucse + endpoint: /ucse + - name: other_thousandeyes + endpoint: /thousandeyes - name: as_path_list_policy_object endpoint: /template/policy/list/aspath/ - name: custom_control_topology_policy_definition @@ -8,6 +253,11 @@ endpoint: /template/policy/list/tloc/ - name: advanced_malware_protection_policy_definition endpoint: /template/policy/definition/advancedMalwareProtection/ +- name: cli_feature_profile + endpoint: /v1/feature-profile/sdwan/cli + children: + - name: cli_config_feature + endpoint: /config - name: ips_signature_list_policy_object endpoint: /template/policy/list/ipssignature/ - name: geo_location_list_policy_object @@ -36,10 +286,14 @@ endpoint: /template/policy/definition/aclv6/ - name: localized_policy endpoint: /template/policy/vedge/ +- name: embedded_security_feature_profile + endpoint: /v1/feature-profile/sdwan/embedded-security - name: block_url_list_policy_object endpoint: /template/policy/list/urlblacklist/ - name: rewrite_rule_policy_definition endpoint: /template/policy/definition/rewriterule/ +- name: sig_security_feature_profile + endpoint: /v1/feature-profile/sdwan/sig-security - name: data_ipv4_prefix_list_policy_object endpoint: /template/policy/list/dataprefix/ - name: region_list_policy_object @@ -114,6 +368,8 @@ endpoint: /template/policy/definition/acl/ - name: zone_list_policy_object endpoint: /template/policy/list/zone/ +- name: dns_security_feature_profile + endpoint: /v1/feature-profile/sdwan/dns-security - name: policer_policy_object endpoint: /template/policy/list/policer/ - name: vedge_inventory @@ -122,86 +378,3 @@ endpoint: /template/policy/list/ipv6prefix/ - name: zone_based_firewall_policy_definition endpoint: /template/policy/definition/zonebasedfw/ -- name: system_feature_profile - endpoint: /v1/feature-profile/sdwan/system - children: - - name: system_banner - endpoint: /banner - - name: system_ipv4_device_access - endpoint: /ipv4-device-access-policy - - name: system_omp - endpoint: /omp - - name: system_bfd - endpoint: /bfd - - name: system_remote_access - endpoint: /remote-access - - name: system_aaa - endpoint: /aaa - - name: system_security - endpoint: /security - - name: system_snmp - endpoint: /snmp - - name: system_logging - endpoint: /logging - - name: system_ntp - endpoint: /ntp - - name: system_perfomance_monitoring - endpoint: /perfmonitor - - name: system_ipv6_device_access - endpoint: /ipv6-device-access-policy - - name: system_global - endpoint: /global - - name: system_mrf - endpoint: /mrf - - name: system_basic - endpoint: /basic - - name: system_flexible_port_speed - endpoint: /flexible-port-speed -- name: transport_feature_profile - endpoint: /v1/feature-profile/sdwan/transport - children: - - name: transport_routing_bgp - endpoint: /routing/bgp - - name: transport_wan_vpn_interface_cellular - endpoint: /wan/vpn/%s/interface/cellular - - name: transport_wan_vpn_interface_gre - endpoint: /wan/vpn/%s/interface/gre - - name: transport_wan_vpn - endpoint: /wan/vpn - - name: transport_wan_vpn_interface_t1_e1_serial - endpoint: /wan/vpn/%s/interface/serial - - name: transport_management_vpn - endpoint: /management/vpn - - name: transport_wan_vpn_interface_ipsec - endpoint: /wan/vpn/%s/interface/ipsec - - name: transport_management_vpn_interface_ethernet - endpoint: /management/vpn/%s/interface/ethernet - - name: transport_wan_vpn_interface_ethernet - endpoint: /wan/vpn/%s/interface/ethernet - - name: transport_ipv6_tracker - endpoint: /ipv6-tracker -- name: service_feature_profile - endpoint: /v1/feature-profile/sdwan/service - children: - - name: service_lan_vpn_interface_ethernet - endpoint: /lan/vpn/%s/interface/ethernet - - name: service_lan_vpn_interface_ipsec - endpoint: /lan/vpn/%s/interface/ipsec - - name: service_lan_vpn - endpoint: /lan/vpn - - name: service_lan_vpn_interface_gre - endpoint: /lan/vpn/%s/interface/gre - - name: service_tracker - endpoint: /tracker - - name: service_lan_vpn_interface_svi - endpoint: /lan/vpn/%s/interface/svi -- name: application_priority_feature_profile - endpoint: /v1/feature-profile/sdwan/application-priority - children: - - name: application_priority_qos_policy - endpoint: /qos-policy -- name: cli_feature_profile - endpoint: /v1/feature-profile/sdwan/cli - children: - - name: cli_config_profile_parcel - endpoint: /config/ From 4e01b460ab84a04a5d9fb9d5deb63b13d4ba20f3 Mon Sep 17 00:00:00 2001 From: tzarski0 <92273798+tzarski0@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:55:34 +0100 Subject: [PATCH 2/4] Update github_repo_wrapper.py --- nac_collector/github_repo_wrapper.py | 130 ++++++++++++--------------- 1 file changed, 56 insertions(+), 74 deletions(-) diff --git a/nac_collector/github_repo_wrapper.py b/nac_collector/github_repo_wrapper.py index 1749226..a893a4c 100644 --- a/nac_collector/github_repo_wrapper.py +++ b/nac_collector/github_repo_wrapper.py @@ -145,7 +145,7 @@ def get_definitions(self): def parent_children(self, endpoints_list): """ Adjusts the endpoints_list list to include parent-child relationships - for endpoints containing `%v`. It separates the endpoints into parent and + for endpoints containing `%v` and `%s`. It separates the endpoints into parent and child entries, modifying the YAML output structure to reflect this hierarchy. Args: @@ -157,87 +157,69 @@ def parent_children(self, endpoints_list): self.logger.info("Adjusting endpoints for parent-child relationships") modified_endpoints = [] - # Dictionary to hold parents and their children + # Dictionary to hold parents and their children based on paths parent_map = {} - # First, identify all potential children with '%v' + # Function to split endpoint and register it in the hierarchy + def register_endpoint(parts, name): + current_level = parent_map + base_endpoint = parts[0] + + # Register base endpoint + if base_endpoint not in current_level: + current_level[base_endpoint] = {"names": [], "children": {}} + current_level = current_level[base_endpoint] + + # Process each subsequent segment + for part in parts[1:]: + if part not in current_level["children"]: + current_level["children"][part] = {"names": [], "children": {}} + current_level = current_level["children"][part] + + # Add the name to the list of names for this segment + # This is to handle a case where there are two endpoint_data + # with different name but same endpoint url + if name not in current_level["names"]: + current_level["names"].append(name) + + # Process each endpoint for endpoint_data in endpoints_list: endpoint = endpoint_data["endpoint"] name = endpoint_data["name"] - if "%v" in endpoint: - base_endpoint, child_path = endpoint.split("/%v", 1) - # Identify the base endpoint and treat this entry as a child - child_entry = {"name": name, "endpoint": child_path} - - # Add this child to the corresponding parent in parent_map - if base_endpoint in parent_map: - parent_map[base_endpoint]["children"].append(child_entry) + # Split the endpoint by placeholders and slashes + parts = [] + remaining = endpoint + while remaining: + if "%v" in remaining or "%s" in remaining: + pre, _, post = remaining.partition('%v' if '%v' in remaining else '%s') + parts.append(pre.rstrip("/")) + remaining = post else: - # If parent doesn't exist, create it - parent_map[base_endpoint] = { - "name": None, # Parent name to be set later - "endpoint": base_endpoint, - "children": [child_entry], - } - if "%s" in endpoint: - for parent_map_key in parent_map: - children = parent_map[parent_map_key]["children"] - to_add = [] - for l1_children in children: - if "%s" in l1_children["endpoint"]: - base_endpoint, child_path = l1_children["endpoint"].split( - "%s", 1 - ) - child_entry = {"name": name, "endpoint": child_path} - # Collect all child entries to be added - for child in children: - if base_endpoint.rstrip("/") == child[ - "endpoint" - ].rstrip("/"): - value_exists = any( - child_entry["endpoint"] in d.values() - for d in child.get("children", "") - ) - if not value_exists: - to_add.append((child, child_entry)) - # Add all collected child entries - for parent, child_entry in to_add: - if "children" in parent: - parent["children"].append(child_entry) - else: - parent["children"] = [child_entry] - - # Now go through endpoints to fill out parent details - for endpoint_data in endpoints_list: - endpoint = endpoint_data["endpoint"] - name = endpoint_data["name"] + parts.append(remaining.rstrip("/" if '%v' in endpoint or '%s' in endpoint or '/v1/feature-profile/' in endpoint else "")) + break + + # Register the endpoint in the hierarchy + register_endpoint(parts, name) + + # Convert the hierarchical map to a list format + def build_hierarchy(node): + """ + Recursively build the YAML structure from the hierarchical dictionary. + """ + output = [] + for part, content in node.items(): + # Create an entry for each name associated with this endpoint + for name in content["names"]: + entry = {"name": name, "endpoint": part} + if content["children"]: + entry["children"] = build_hierarchy(content["children"]) + output.append(entry) + return output + + # Build the final list from the parent_map + modified_endpoints = build_hierarchy(parent_map) - # Normalize the input path by removing any trailing slashes - endpoint = endpoint.rstrip("/") - - if endpoint in parent_map: - # This is a confirmed parent endpoint - parent_map[endpoint]["name"] = name - else: - # This endpoint is not a parent; no children reference it - if "%v" not in endpoint and "%s" not in endpoint: - # Standalone endpoint, add directly to modified_endpoints - modified_endpoints.append(endpoint_data) - - # Add all valid parent-child structures to the modified_endpoints list - for _, parent_data in parent_map.items(): - # Add to modified list only if it has children - if parent_data["children"]: - # Remove the entry of child object with %s - parent_data["children"] = [ - child - for child in parent_data["children"] - if "%s" not in child.get("endpoint", "") - ] - modified_endpoints.append(parent_data) - - # Return the modified endpoints list return modified_endpoints def _delete_repo(self): From 399a6ec48cd231cf820697541507747af80f0c88 Mon Sep 17 00:00:00 2001 From: tzarski0 <92273798+tzarski0@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:56:13 +0100 Subject: [PATCH 3/4] Update cisco_client_sdwan.py --- nac_collector/cisco_client_sdwan.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nac_collector/cisco_client_sdwan.py b/nac_collector/cisco_client_sdwan.py index 52f5195..1685a97 100644 --- a/nac_collector/cisco_client_sdwan.py +++ b/nac_collector/cisco_client_sdwan.py @@ -402,7 +402,7 @@ def extract_feature_parcel( ): parcel_type = parcel["parcelType"] if parcel_type.startswith(upstream_parcel_type): - parcel_type = parcel_type[len(upstream_parcel_type) :].lstrip("/") + parcel_type = parcel_type[len(upstream_parcel_type):].lstrip("/") parcel_id = parcel["parcelId"] new_endpoint = upstream_endpoint + "/" + parcel_type + "/" + parcel_id response = self.get_request(self.base_url + new_endpoint) @@ -415,9 +415,12 @@ def extract_feature_parcel( children_endpoint_type = ( parcel_type + "/" + self.strip_backslash(children_endpoint["endpoint"]) ) - children_endpoint_type = self.strip_backslash(children_endpoint_type) + children_endpoint_type1 = self.strip_backslash(children_endpoint_type) + children_endpoint_type2 = self.strip_backslash(children_endpoint_type) + if children_endpoint_type.startswith(parcel_type): + children_endpoint_type2 = children_endpoint_type1[len(parcel_type):].lstrip("/") for subparcel in parcel.get("subparcels", []): - if subparcel["parcelType"] == children_endpoint_type: + if subparcel["parcelType"] in [children_endpoint_type1, children_endpoint_type2]: children_entries.append( self.extract_feature_parcel( new_endpoint, From acf078a59ea92b7d716704128f1d8fb88261274a Mon Sep 17 00:00:00 2001 From: tzarski0 <92273798+tzarski0@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:22:38 +0100 Subject: [PATCH 4/4] fix formatting --- nac_collector/cisco_client_sdwan.py | 11 ++++++++--- nac_collector/github_repo_wrapper.py | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/nac_collector/cisco_client_sdwan.py b/nac_collector/cisco_client_sdwan.py index 1685a97..ab6a9a7 100644 --- a/nac_collector/cisco_client_sdwan.py +++ b/nac_collector/cisco_client_sdwan.py @@ -402,7 +402,7 @@ def extract_feature_parcel( ): parcel_type = parcel["parcelType"] if parcel_type.startswith(upstream_parcel_type): - parcel_type = parcel_type[len(upstream_parcel_type):].lstrip("/") + parcel_type = parcel_type[len(upstream_parcel_type) :].lstrip("/") parcel_id = parcel["parcelId"] new_endpoint = upstream_endpoint + "/" + parcel_type + "/" + parcel_id response = self.get_request(self.base_url + new_endpoint) @@ -418,9 +418,14 @@ def extract_feature_parcel( children_endpoint_type1 = self.strip_backslash(children_endpoint_type) children_endpoint_type2 = self.strip_backslash(children_endpoint_type) if children_endpoint_type.startswith(parcel_type): - children_endpoint_type2 = children_endpoint_type1[len(parcel_type):].lstrip("/") + children_endpoint_type2 = children_endpoint_type1[ + len(parcel_type) : + ].lstrip("/") for subparcel in parcel.get("subparcels", []): - if subparcel["parcelType"] in [children_endpoint_type1, children_endpoint_type2]: + if subparcel["parcelType"] in [ + children_endpoint_type1, + children_endpoint_type2, + ]: children_entries.append( self.extract_feature_parcel( new_endpoint, diff --git a/nac_collector/github_repo_wrapper.py b/nac_collector/github_repo_wrapper.py index a893a4c..1c11955 100644 --- a/nac_collector/github_repo_wrapper.py +++ b/nac_collector/github_repo_wrapper.py @@ -192,11 +192,21 @@ def register_endpoint(parts, name): remaining = endpoint while remaining: if "%v" in remaining or "%s" in remaining: - pre, _, post = remaining.partition('%v' if '%v' in remaining else '%s') + pre, _, post = remaining.partition( + "%v" if "%v" in remaining else "%s" + ) parts.append(pre.rstrip("/")) remaining = post else: - parts.append(remaining.rstrip("/" if '%v' in endpoint or '%s' in endpoint or '/v1/feature-profile/' in endpoint else "")) + parts.append( + remaining.rstrip( + "/" + if "%v" in endpoint + or "%s" in endpoint + or "/v1/feature-profile/" in endpoint + else "" + ) + ) break # Register the endpoint in the hierarchy