Merge branch 'develop' into develop

mypy.ini

@@ -41,7 +41,7 @@ ignore_missing_imports = True
 [mypy-lxml.*]
 ignore_missing_imports = True
 
-[mypy-ciscoconfparse]
+[mypy-netutils.*]
 ignore_missing_imports = True
 
 [mypy-textfsm]

napalm/base/helpers.py

@@ -12,11 +12,11 @@
 # third party libs
 import jinja2
 import textfsm
-from ciscoconfparse import CiscoConfParse
 from lxml import etree
 from netaddr import EUI
 from netaddr import IPAddress
 from netaddr import mac_unix
+from netutils.config.parser import IOSConfigParser
 
 try:
     from ttp import quick_parse as ttp_quick_parse
@@ -134,42 +134,84 @@ def load_template(
     return cls.load_merge_candidate(config=configuration)
 
 
-def cisco_conf_parse_parents(
+def netutils_parse_parents(
     parent: str, child: str, config: Union[str, List[str]]
 ) -> List[str]:
     """
-    Use CiscoConfParse to find parent lines that contain a specific child line.
+    Use Netutils to find parent lines that contain a specific child line.
 
     :param parent: The parent line to search for
     :param child:  The child line required under the given parent
     :param config: The device running/startup config
     """
-    if type(config) == str:
-        config = config.splitlines()  # type: ignore
-    parse = CiscoConfParse(config)
-    cfg_obj = parse.find_parents_w_child(parent, child)
-    return cfg_obj
+    # Check if the config is a list, if it is a list, then join it to make a string.
+    if isinstance(config, list):
+        config = "\n".join(config)
+        config = config + "\n"
+
+    # Config tree is the entire configuration in a tree format,
+    # followed by getting the individual lines that has the formats:
+    # ConfigLine(config_line=' ip address 192.0.2.10 255.255.255.0',
+    # parents=('interface GigabitEthernet1',))
+    # ConfigLine(config_line='Current configuration : 1624 bytes', parents=())
+    config_tree = IOSConfigParser(str(config))
+    configuration_lines = config_tree.build_config_relationship()
+
+    # Return config is the list that will be returned
+    return_config = []
+
+    # Loop over each of the configuration lines
+    for line in configuration_lines:
+        # Loop over any line that has a parent line. If there are no parents for a line item then
+        # the parents is an empty tuple.
+        for parent_line in line.parents:
+            if (
+                child in line.config_line
+                and re.match(parent, parent_line) is not None
+                and parent_line not in return_config
+            ):
+                return_config.append(parent_line)
+
+    return return_config
 
 
-def cisco_conf_parse_objects(
+def netutils_parse_objects(
     cfg_section: str, config: Union[str, List[str]]
 ) -> List[str]:
     """
-    Use CiscoConfParse to find and return a section of Cisco IOS config.
+    Use Netutils to find and return a section of Cisco IOS config.
     Similar to "show run | section <cfg_section>"
 
     :param cfg_section: The section of the config to return eg. "router bgp"
     :param config: The running/startup config of the device to parse
     """
+    # Check if the config is a list, if it is a list, then join it to make a string.
+    if isinstance(config, list):
+        config = "\n".join(config)
+        config = config + "\n"
+
+    # Config tree is the entire configuration in a tree format,
+    # followed by getting the individual lines that has the formats:
+    # ConfigLine(config_line=' ip address 192.0.2.10 255.255.255.0',
+    # parents=('interface GigabitEthernet1',))
+    # ConfigLine(config_line='Current configuration : 1624 bytes', parents=())
+    config_tree = IOSConfigParser(str(config))
+    lines = config_tree.build_config_relationship()
+
+    # Return config is the list that will be returned
     return_config = []
-    if type(config) is str:
-        config = config.splitlines()  # type: ignore
-    parse = CiscoConfParse(config)
-    cfg_obj = parse.find_objects(cfg_section)
-    for parent in cfg_obj:
-        return_config.append(parent.text)
-        for child in parent.all_children:
-            return_config.append(child.text)
+    for line in lines:
+        # The parent configuration is expected on the function that this is replacing,
+        # add the parent line to the base of the return_config
+        if cfg_section in line.config_line:
+            return_config.append(line.config_line)
+        # Check if the tuple is greater than 0
+        if len(line.parents) > 0:
+            # Check the eldest parent, if that is part of the config section, then append
+            # the current line being checked to it.
+            if cfg_section in line.parents[0]:
+                return_config.append(line.config_line)
+
     return return_config
 
 

napalm/ios/ios.py

@@ -876,11 +876,13 @@ def get_lldp_neighbors(self):
             lldp[intf_name] = []
             for lldp_entry in entries:
                 hostname = lldp_entry["remote_system_name"]
+                port = lldp_entry["remote_port"]
                 # Match IOS behaviour of taking remote chassis ID
                 # When lacking a system name (in show lldp neighbors)
                 if not hostname:
-                    hostname = lldp_entry["remote_chassis_id"]
-                lldp_dict = {"port": lldp_entry["remote_port"], "hostname": hostname}
+                    hostname = napalm.base.helpers.mac(lldp_entry["remote_chassis_id"])
+                    port = napalm.base.helpers.mac(port)
+                lldp_dict = {"port": port, "hostname": hostname}
                 lldp[intf_name].append(lldp_dict)
 
         return lldp
@@ -901,6 +903,14 @@ def get_lldp_neighbors_detail(self, interface=""):
         if len(lldp_entries) == 0:
             return {}
 
+        # format chassis_id for consistency
+        for entry in lldp_entries:
+            entry["remote_chassis_id"] = napalm.base.helpers.convert(
+                napalm.base.helpers.mac,
+                entry["remote_chassis_id"],
+                entry["remote_chassis_id"],
+            )
+
         # Older IOS versions don't have 'Local Intf' defined in LLDP detail.
         # We need to get them from the non-detailed command
         # which is in the same sequence as the detailed output
@@ -1203,6 +1213,7 @@ def get_interfaces_ip(self):
             m = re.match(INTERNET_ADDRESS, line)
             if m:
                 ip, prefix = m.groups()
+                ip = napalm.base.helpers.ip(ip)
                 ipv4.update({ip: {"prefix_length": int(prefix)}})
                 interfaces[interface_name] = {"ipv4": ipv4}
 
@@ -1220,10 +1231,12 @@ def get_interfaces_ip(self):
                 m = re.match(LINK_LOCAL_ADDRESS, line)
                 if m:
                     ip = m.group(1)
+                    ip = napalm.base.helpers.ip(ip, 6)
                     ipv6.update({ip: {"prefix_length": 10}})
                 m = re.match(GLOBAL_ADDRESS, line)
                 if m:
                     ip, prefix = m.groups()
+                    ip = napalm.base.helpers.ip(ip, 6)
                     ipv6.update({ip: {"prefix_length": int(prefix)}})
 
         # Interface without ipv6 doesn't appears in show ipv6 interface
@@ -1320,21 +1333,20 @@ def build_prefix_limit(af_table, limit, prefix_percent, prefix_timeout):
                 }
             return prefix_limit
 
-        # Get BGP config using ciscoconfparse because some old devices dont support "| sec bgp"
+        # Get BGP config using netutils because some old devices dont support "| sec bgp"
         cfg = self.get_config(retrieve="running")
-        cfg = cfg["running"].splitlines()
-        bgp_config_text = napalm.base.helpers.cisco_conf_parse_objects(
-            "router bgp", cfg
+        bgp_config_list = napalm.base.helpers.netutils_parse_objects(
+            "router bgp", cfg["running"]
         )
         bgp_asn = napalm.base.helpers.regex_find_txt(
-            r"router bgp (\d+)", bgp_config_text, default=0
+            r"router bgp (\d+)", bgp_config_list, default=0
         )
         # Get a list of all neighbors and groups in the config
         all_neighbors = set()
         all_groups = set()
         bgp_group_neighbors = {}
         all_groups.add("_")
-        for line in bgp_config_text:
+        for line in bgp_config_list:
             if " neighbor " in line:
                 if re.search(IP_ADDR_REGEX, line) is not None:
                     all_neighbors.add(re.search(IP_ADDR_REGEX, line).group())
@@ -1351,8 +1363,8 @@ def build_prefix_limit(af_table, limit, prefix_percent, prefix_timeout):
             if neighbor:
                 if bgp_neighbor != neighbor:
                     continue
-            afi_list = napalm.base.helpers.cisco_conf_parse_parents(
-                r"\s+address-family.*", bgp_neighbor, bgp_config_text
+            afi_list = napalm.base.helpers.netutils_parse_parents(
+                r"\s+address-family.*", bgp_neighbor, bgp_config_list
             )
             try:
                 afi = afi_list[0]
@@ -1363,8 +1375,8 @@ def build_prefix_limit(af_table, limit, prefix_percent, prefix_timeout):
             if "vrf" in str(afi_list):
                 continue
             else:
-                neighbor_config = napalm.base.helpers.cisco_conf_parse_objects(
-                    bgp_neighbor, bgp_config_text
+                neighbor_config = napalm.base.helpers.netutils_parse_objects(
+                    bgp_neighbor, bgp_config_list
                 )
             # For group_name- use peer-group name, else VRF name, else "_" for no group
             group_name = napalm.base.helpers.regex_find_txt(
@@ -1454,16 +1466,16 @@ def build_prefix_limit(af_table, limit, prefix_percent, prefix_timeout):
                     "neighbors": bgp_group_neighbors.get("_", {}),
                 }
                 continue
-            neighbor_config = napalm.base.helpers.cisco_conf_parse_objects(
-                group_name, bgp_config_text
+            neighbor_config = napalm.base.helpers.netutils_parse_objects(
+                group_name, bgp_config_list
             )
             multipath = False
-            afi_list = napalm.base.helpers.cisco_conf_parse_parents(
-                r"\s+address-family.*", group_name, neighbor_config
+            afi_list = napalm.base.helpers.netutils_parse_parents(
+                r"\s+address-family.*", group_name, bgp_config_list
             )
             for afi in afi_list:
-                afi_config = napalm.base.helpers.cisco_conf_parse_objects(
-                    afi, bgp_config_text
+                afi_config = napalm.base.helpers.netutils_parse_objects(
+                    afi, bgp_config_list
                 )
                 multipath = bool(
                     napalm.base.helpers.regex_find_txt(r" multipath", str(afi_config))
@@ -2383,7 +2395,7 @@ def get_ntp_peers(self):
         ntp_stats = self.get_ntp_stats()
 
         return {
-            ntp_peer.get("remote"): {}
+            napalm.base.helpers.ip(ntp_peer.get("remote")): {}
             for ntp_peer in ntp_stats
             if ntp_peer.get("remote")
         }
@@ -3251,7 +3263,10 @@ def ping(
                     results_array = []
                     for i in range(probes_received):
                         results_array.append(
-                            {"ip_address": str(destination), "rtt": 0.0}
+                            {
+                                "ip_address": napalm.base.helpers.ip(str(destination)),
+                                "rtt": 0.0,
+                            }
                         )
                     ping_dict["success"].update({"results": results_array})
 
@@ -3374,7 +3389,7 @@ def traceroute(
                     current_probe += 1
                 # If current_element contains msec record the entry for probe
                 elif "msec" in current_element:
-                    ip_address = str(ip_address)
+                    ip_address = napalm.base.helpers.ip(str(ip_address))
                     host_name = str(host_name)
                     rtt = float(current_element.replace("msec", ""))
                     results[current_hop]["probes"][current_probe][

requirements.txt

@@ -10,9 +10,9 @@ pyYAML
 pyeapi>=0.8.2
 netmiko>=3.3.0,<4.0.0
 junos-eznc>=2.2.1
-ciscoconfparse
 scp
 lxml>=4.3.0
 ncclient
 ttp
 ttp_templates
+netutils>=1.0.0

test/ios/mocked_data/test_get_interfaces_ip/normal/expected_result.json

@@ -1,34 +1,40 @@
 {
-	"Vlan20": {
-		"ipv4": {
-			"172.29.50.3": {
-				"prefix_length": 27
-			}
-		},
-		"ipv6": {
-			"FE80::C009:24FF:FEAC:0": {
-				"prefix_length": 10
-			},
-			"2002::1": {
-				"prefix_length": 64
-			}
-		}
-	},
-	"Vlan41": {
-		"ipv4": {
-			"172.29.52.34": {
-				"prefix_length": 27
-			},
-			"192.168.81.34": {
-				"prefix_length": 24
-			}
-		}
-	},
-	"Loopback0": {
-		"ipv6": {
-			"FE80::C009:24FF:FEAC:0": {"prefix_length": 10},
-			"2001::1": {"prefix_length": 128},
-			"2001::2": {"prefix_length": 128}
-		}
-	}
+  "Vlan20": {
+    "ipv4": {
+      "172.29.50.3": {
+        "prefix_length": 27
+      }
+    },
+    "ipv6": {
+      "fe80::c009:24ff:feac:0": {
+        "prefix_length": 10
+      },
+      "2002::1": {
+        "prefix_length": 64
+      }
+    }
+  },
+  "Vlan41": {
+    "ipv4": {
+      "172.29.52.34": {
+        "prefix_length": 27
+      },
+      "192.168.81.34": {
+        "prefix_length": 24
+      }
+    }
+  },
+  "Loopback0": {
+    "ipv6": {
+      "fe80::c009:24ff:feac:0": {
+        "prefix_length": 10
+      },
+      "2001::1": {
+        "prefix_length": 128
+      },
+      "2001::2": {
+        "prefix_length": 128
+      }
+    }
+  }
 }

test/ios/mocked_data/test_get_lldp_neighbors/missing_hostname/expected_result.json

@@ -1,14 +1,14 @@
 {
-    "FastEthernet0/24": [
-        {
-            "port": "Fa0/19",
-            "hostname": "switch1"
-        }
-    ],
-    "FastEthernet0/17": [
-        {
-            "port": "480f.cf28.8a1b",
-            "hostname": "480f.cf28.8a1b"
-        }
-    ]
+  "FastEthernet0/24": [
+    {
+      "port": "Fa0/19",
+      "hostname": "switch1"
+    }
+  ],
+  "FastEthernet0/17": [
+    {
+      "port": "48:0F:CF:28:8A:1B",
+      "hostname": "48:0F:CF:28:8A:1B"
+    }
+  ]
 }