Filter unneeded header configuration lines from IOS and NX-OS get_con…

…fig (#1190)

napalm/base/helpers.py

@@ -5,6 +5,7 @@
 import sys
 import itertools
 import logging
+from collections import Iterable
 
 # third party libs
 import jinja2
@@ -46,7 +47,7 @@ def load_template(
     template_path=None,
     openconfig=False,
     jinja_filters={},
-    **template_vars
+    **template_vars,
 ):
     try:
         search_path = []
@@ -460,3 +461,24 @@ def transform_lldp_capab(capabilities):
         )
     else:
         return []
+
+
+def generate_regex_or(filters):
+    """
+    Build a regular expression logical-or from a list/tuple of regex patterns.
+
+    This allows a single regular expression operation to be used in contexts when a loop
+    and multiple patterns would otherwise be necessary.
+
+    For example, (pattern1|pattern2|pattern3)
+
+    Return the pattern.
+    """
+    if isinstance(filters, str) or not isinstance(filters, Iterable):
+        raise ValueError("filters argument must be an iterable, but can't be a string.")
+
+    return_pattern = r"("
+    for pattern in filters:
+        return_pattern += rf"{pattern}|"
+    return_pattern += r")"
+    return return_pattern

napalm/ios/ios.py

@@ -41,6 +41,7 @@
     textfsm_extractor,
     split_interface,
     abbreviated_interface_name,
+    generate_regex_or,
 )
 from napalm.base.netmiko_helpers import netmiko_args
 
@@ -3376,19 +3377,31 @@ def get_config(self, retrieve="all", full=False):
         since IOS does not support candidate configuration.
         """
 
+        # The output of get_config should be directly usable by load_replace_candidate()
+        # IOS adds some extra, unneeded lines that should be filtered.
+        filter_strings = [
+            r"^Building configuration.*$",
+            r"^Current configuration :.*$",
+            r"^! Last configuration change at.*$",
+            r"^! NVRAM config last updated at.*$",
+        ]
+        filter_pattern = generate_regex_or(filter_strings)
+
         configs = {"startup": "", "running": "", "candidate": ""}
         # IOS only supports "all" on "show run"
         run_full = " all" if full else ""
 
         if retrieve in ("startup", "all"):
             command = "show startup-config"
             output = self._send_command(command)
-            configs["startup"] = output
+            output = re.sub(filter_pattern, "", output, flags=re.M)
+            configs["startup"] = output.strip()
 
         if retrieve in ("running", "all"):
             command = "show running-config{}".format(run_full)
             output = self._send_command(command)
-            configs["running"] = output
+            output = re.sub(filter_pattern, "", output, flags=re.M)
+            configs["running"] = output.strip()
 
         return configs
 

napalm/nxos/nxos.py

@@ -42,6 +42,7 @@
 from napalm.base.exceptions import MergeConfigException
 from napalm.base.exceptions import CommandErrorException
 from napalm.base.exceptions import ReplaceConfigException
+from napalm.base.helpers import generate_regex_or
 from napalm.base.netmiko_helpers import netmiko_args
 import napalm.base.constants as c
 
@@ -515,16 +516,29 @@ def _disable_confirmation(self):
         self._send_command_list(["terminal dont-ask"])
 
     def get_config(self, retrieve="all", full=False):
+
+        # NX-OS adds some extra, unneeded lines that should be filtered.
+        filter_strings = [
+            r"!Command: show .*$",
+            r"!Time:.*\d{4}\s*$",
+            r"Startup config saved at:.*$",
+        ]
+        filter_pattern = generate_regex_or(filter_strings)
+
         config = {"startup": "", "running": "", "candidate": ""}  # default values
         # NX-OS only supports "all" on "show run"
         run_full = " all" if full else ""
 
         if retrieve.lower() in ("running", "all"):
             command = f"show running-config{run_full}"
-            config["running"] = str(self._send_command(command, raw_text=True))
+            output = self._send_command(command, raw_text=True)
+            output = re.sub(filter_pattern, "", output, flags=re.M)
+            config["running"] = output.strip()
         if retrieve.lower() in ("startup", "all"):
             command = "show startup-config"
-            config["startup"] = str(self._send_command(command, raw_text=True))
+            output = self._send_command(command, raw_text=True)
+            output = re.sub(filter_pattern, "", output, flags=re.M)
+            config["startup"] = output.strip()
         return config
 
     def get_lldp_neighbors(self):

test/ios/mocked_data/test_get_config/normal/expected_result.json

@@ -1,5 +1,5 @@
 {
-	"startup": "!\n! Last configuration change at 18:41:02 UTC Thu Nov 24 2016\n!\nversion 15.5\nservice timestamps debug datetime msec\nservice timestamps log datetime msec\nno platform punt-keepalive disable-kernel-core\nplatform console auto\n!\nhostname CSR1\n!\nboot-start-marker\nboot-end-marker\n!\n!\nenable password cisco\n!\naaa new-model\n!\n!\naaa authentication login default local\naaa authorization exec default local\n!\n!\n!\n!\n!\naaa session-id common\n!\nip vrf MGMT\n!\n!\n!\n!\n!\n!\n!\n!\n!\n\n\nip domain name example.local\n\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nsubscriber templating\n!\nmultilink bundle-name authenticated\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nlicense udi pid CSR1000V sn 9OSEGKJXRHE\nspanning-tree extend system-id\n!\nusername cisco privilege 15 password 0 cisco\n!\nredundancy\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\ninterface Loopback0\n ip address 1.1.1.1 255.255.255.255\n!\ninterface GigabitEthernet1\n ip vrf forwarding MGMT\n ip address 192.168.35.121 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet2\n ip address 10.1.1.1 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet3\n no ip address\n shutdown\n negotiation auto\n!\nrouter ospf 1\n redistribute connected subnets\n network 10.1.1.0 0.0.0.255 area 0\n!\n!\nvirtual-service csr_mgmt\n!\nip forward-protocol nd\n!\nno ip http server\nno ip http secure-server\n!\n!\n!\n!\n!\n!\ncontrol-plane\n!\n !\n !\n !\n !\n!\n!\n!\n!\n!\nline con 0\nline vty 0 4\n!\n!\nend",
-	"running": "Building configuration...\n\nCurrent configuration : 1366 bytes\n!\n! Last configuration change at 18:41:02 UTC Thu Nov 24 2016\n!\nversion 15.5\nservice timestamps debug datetime msec\nservice timestamps log datetime msec\nno platform punt-keepalive disable-kernel-core\nplatform console auto\n!\nhostname CSR1\n!\nboot-start-marker\nboot-end-marker\n!\n!\nenable password cisco\n!\naaa new-model\n!\n!\naaa authentication login default local\naaa authorization exec default local\n!\n!\n!\n!\n!\naaa session-id common\n!\nip vrf MGMT\n!\n!\n!\n!\n!\n!\n!\n!\n!\n\n\nip domain name example.local\n\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nsubscriber templating\n!\nmultilink bundle-name authenticated\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nlicense udi pid CSR1000V sn 9OSEGKJXRHE\nspanning-tree extend system-id\n!\nusername cisco privilege 15 password 0 cisco\n!\nredundancy\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\ninterface Loopback0\n ip address 1.1.1.1 255.255.255.255\n!\ninterface GigabitEthernet1\n ip vrf forwarding MGMT\n ip address 192.168.35.121 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet2\n ip address 10.1.1.1 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet3\n no ip address\n shutdown\n negotiation auto\n!\nrouter ospf 1\n redistribute connected subnets\n network 10.1.1.0 0.0.0.255 area 0\n!\n!\nvirtual-service csr_mgmt\n!\nip forward-protocol nd\n!\nno ip http server\nno ip http secure-server\n!\n!\n!\n!\n!\n!\ncontrol-plane\n!\n !\n !\n !\n !\n!\n!\n!\n!\n!\nline con 0\nline vty 0 4\n!\n!\nend",
+	"startup": "!\n\n!\nversion 15.5\nservice timestamps debug datetime msec\nservice timestamps log datetime msec\nno platform punt-keepalive disable-kernel-core\nplatform console auto\n!\nhostname CSR1\n!\nboot-start-marker\nboot-end-marker\n!\n!\nenable password cisco\n!\naaa new-model\n!\n!\naaa authentication login default local\naaa authorization exec default local\n!\n!\n!\n!\n!\naaa session-id common\n!\nip vrf MGMT\n!\n!\n!\n!\n!\n!\n!\n!\n!\n\n\nip domain name example.local\n\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nsubscriber templating\n!\nmultilink bundle-name authenticated\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nlicense udi pid CSR1000V sn 9OSEGKJXRHE\nspanning-tree extend system-id\n!\nusername cisco privilege 15 password 0 cisco\n!\nredundancy\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\ninterface Loopback0\n ip address 1.1.1.1 255.255.255.255\n!\ninterface GigabitEthernet1\n ip vrf forwarding MGMT\n ip address 192.168.35.121 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet2\n ip address 10.1.1.1 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet3\n no ip address\n shutdown\n negotiation auto\n!\nrouter ospf 1\n redistribute connected subnets\n network 10.1.1.0 0.0.0.255 area 0\n!\n!\nvirtual-service csr_mgmt\n!\nip forward-protocol nd\n!\nno ip http server\nno ip http secure-server\n!\n!\n!\n!\n!\n!\ncontrol-plane\n!\n !\n !\n !\n !\n!\n!\n!\n!\n!\nline con 0\nline vty 0 4\n!\n!\nend",
+	"running": "!\n\n!\nversion 15.5\nservice timestamps debug datetime msec\nservice timestamps log datetime msec\nno platform punt-keepalive disable-kernel-core\nplatform console auto\n!\nhostname CSR1\n!\nboot-start-marker\nboot-end-marker\n!\n!\nenable password cisco\n!\naaa new-model\n!\n!\naaa authentication login default local\naaa authorization exec default local\n!\n!\n!\n!\n!\naaa session-id common\n!\nip vrf MGMT\n!\n!\n!\n!\n!\n!\n!\n!\n!\n\n\nip domain name example.local\n\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nsubscriber templating\n!\nmultilink bundle-name authenticated\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\nlicense udi pid CSR1000V sn 9OSEGKJXRHE\nspanning-tree extend system-id\n!\nusername cisco privilege 15 password 0 cisco\n!\nredundancy\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\n!\ninterface Loopback0\n ip address 1.1.1.1 255.255.255.255\n!\ninterface GigabitEthernet1\n ip vrf forwarding MGMT\n ip address 192.168.35.121 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet2\n ip address 10.1.1.1 255.255.255.0\n negotiation auto\n!\ninterface GigabitEthernet3\n no ip address\n shutdown\n negotiation auto\n!\nrouter ospf 1\n redistribute connected subnets\n network 10.1.1.0 0.0.0.255 area 0\n!\n!\nvirtual-service csr_mgmt\n!\nip forward-protocol nd\n!\nno ip http server\nno ip http secure-server\n!\n!\n!\n!\n!\n!\ncontrol-plane\n!\n !\n !\n !\n !\n!\n!\n!\n!\n!\nline con 0\nline vty 0 4\n!\n!\nend",
 	"candidate": ""
 }

test/nxos/mocked_data/test_get_config/normal/expected_result.json

@@ -1,5 +1,5 @@
 {
-    "running": "!Command: show running-config\n!Time: Thu Dec 15 18:05:42 2016\n\nversion 7.3(1)D1(1)\npower redundancy-mode redundant\n\nhostname nxos-spine1\nvdc nxos-spine1 id 1\n  limit-resource module-type m1 m1xl m2xl f2e\n  allocate interface Ethernet2/1-48\n  allocate interface Ethernet3/1-48\n  allocate interface Ethernet4/1-48\n  limit-resource vlan minimum 16 maximum 4094\n  limit-resource vrf minimum 2 maximum 4096\n  limit-resource port-channel minimum 0 maximum 768\n  limit-resource u4route-mem minimum 96 maximum 96\n  limit-resource u6route-mem minimum 24 maximum 24\n  limit-resource m4route-mem minimum 58 maximum 58\n  limit-resource m6route-mem minimum 8 maximum 8\n\nfeature scp-server\ncfs eth distribute\nfeature interface-vlan\nfeature hsrp\nfeature vpc\nfeature nxapi",
-    "startup": "!Command: show running-config\n!Time: Thu Dec 15 18:05:42 2016\n\nversion 7.3(1)D1(1)\npower redundancy-mode redundant\n\nhostname nxos-spine1\nvdc nxos-spine1 id 1\n  limit-resource module-type m1 m1xl m2xl f2e\n  allocate interface Ethernet2/1-48\n  allocate interface Ethernet3/1-48\n  allocate interface Ethernet4/1-48\n  limit-resource vlan minimum 16 maximum 4094\n  limit-resource vrf minimum 2 maximum 4096\n  limit-resource port-channel minimum 0 maximum 768\n  limit-resource u4route-mem minimum 96 maximum 96\n  limit-resource u6route-mem minimum 24 maximum 24\n  limit-resource m4route-mem minimum 58 maximum 58\n  limit-resource m6route-mem minimum 8 maximum 8\n\nfeature scp-server\ncfs eth distribute\nfeature interface-vlan\nfeature hsrp\nfeature vpc\nfeature nxapi",
+    "running": "version 7.3(1)D1(1)\npower redundancy-mode redundant\n\nhostname nxos-spine1\nvdc nxos-spine1 id 1\n  limit-resource module-type m1 m1xl m2xl f2e\n  allocate interface Ethernet2/1-48\n  allocate interface Ethernet3/1-48\n  allocate interface Ethernet4/1-48\n  limit-resource vlan minimum 16 maximum 4094\n  limit-resource vrf minimum 2 maximum 4096\n  limit-resource port-channel minimum 0 maximum 768\n  limit-resource u4route-mem minimum 96 maximum 96\n  limit-resource u6route-mem minimum 24 maximum 24\n  limit-resource m4route-mem minimum 58 maximum 58\n  limit-resource m6route-mem minimum 8 maximum 8\n\nfeature scp-server\ncfs eth distribute\nfeature interface-vlan\nfeature hsrp\nfeature vpc\nfeature nxapi",
+    "startup": "version 7.3(1)D1(1)\npower redundancy-mode redundant\n\nhostname nxos-spine1\nvdc nxos-spine1 id 1\n  limit-resource module-type m1 m1xl m2xl f2e\n  allocate interface Ethernet2/1-48\n  allocate interface Ethernet3/1-48\n  allocate interface Ethernet4/1-48\n  limit-resource vlan minimum 16 maximum 4094\n  limit-resource vrf minimum 2 maximum 4096\n  limit-resource port-channel minimum 0 maximum 768\n  limit-resource u4route-mem minimum 96 maximum 96\n  limit-resource u6route-mem minimum 24 maximum 24\n  limit-resource m4route-mem minimum 58 maximum 58\n  limit-resource m6route-mem minimum 8 maximum 8\n\nfeature scp-server\ncfs eth distribute\nfeature interface-vlan\nfeature hsrp\nfeature vpc\nfeature nxapi",
     "candidate": ""
 }