Inventory: Implement grouping and extracing rack groups, and rack rol…

…es (#203)

plugins/inventory/nb_inventory.py

@@ -105,6 +105,8 @@
                 - tenant
                 - racks
                 - rack
+                - rack_groups
+                - rack_role
                 - tags
                 - tag
                 - device_roles
@@ -350,6 +352,8 @@ def group_extractors(self):
             self._pluralize_group_by("site"): self.extract_site,
             self._pluralize_group_by("tenant"): self.extract_tenant,
             self._pluralize_group_by("rack"): self.extract_rack,
+            "rack_groups": self.extract_rack_groups,
+            "rack_role": self.extract_rack_role,
             self._pluralize_group_by("tag"): self.extract_tags,
             self._pluralize_group_by("role"): self.extract_device_role,
             self._pluralize_group_by("platform"): self.extract_platform,
@@ -394,6 +398,29 @@ def _pluralize(self, extracted_value):
         else:
             return extracted_value
 
+    def _objects_array_following_parents(
+        self, initial_object_id, object_lookup, object_parent_lookup
+    ):
+        objects = []
+
+        object_id = initial_object_id
+
+        # Keep looping until the object has no parent
+        while object_id is not None:
+            object_slug = object_lookup[object_id]
+
+            if object_slug in objects:
+                # Won't ever happen - defensively guard against infinite loop
+                break
+
+            objects.append(object_slug)
+
+            # Get the parent of this object
+            object_id = object_parent_lookup[object_id]
+
+        return objects
+
+
     def extract_disk(self, host):
         return host.get("disk")
 
@@ -434,6 +461,35 @@ def extract_rack(self, host):
         except Exception:
             return
 
+    def extract_rack_groups(self, host):
+        # A host may have a rack. A rack may have a rack_group. A rack_group may have a parent rack_group.
+        # Produce a list of rack_groups:
+        # - it will be empty if the device has no rack, or the rack has no rack_group
+        # - it will have 1 element if the rack's group has no parent
+        # - it will have multiple elements if the rack's group has a parent group
+
+        rack = host.get("rack", None)
+        if not isinstance(rack, dict):
+            # Device has no rack
+            return None
+
+        rack_id = rack.get("id", None)
+        if rack_id is None:
+            # Device has no rack
+            return None
+
+        return self._objects_array_following_parents(
+            initial_object_id=self.racks_group_lookup[rack_id],
+            object_lookup=self.rack_groups_lookup,
+            object_parent_lookup=self.rack_group_parent_lookup,
+        )
+
+    def extract_rack_role(self, host):
+        try:
+            return self.racks_role_lookup[host["rack"]["id"]]
+        except Exception:
+            return
+
     def extract_site(self, host):
         try:
             return self._pluralize(self.sites_lookup[host["site"]["id"]])
@@ -543,21 +599,11 @@ def extract_regions(self, host):
             # Device has no site
             return []
 
-        regions = []
-        region_id = self.sites_region_lookup[site_id]
-
-        # Keep looping until the region has no parent
-        while region_id is not None:
-            region_slug = self.regions_lookup[region_id]
-            if region_slug in regions:
-                # Won't ever happen - defensively guard against infinite loop
-                break
-            regions.append(region_slug)
-
-            # Get the parent of this region
-            region_id = self.regions_parent_lookup[region_id]
-
-        return regions
+        return self._objects_array_following_parents(
+            initial_object_id=self.sites_region_lookup[site_id],
+            object_lookup=self.regions_lookup,
+            object_parent_lookup=self.regions_parent_lookup,
+        )
 
     def extract_cluster(self, host):
         try:
@@ -601,9 +647,7 @@ def get_region_for_site(site):
                 return (site["id"], None)
 
         # Dictionary of site id to region id
-        self.sites_region_lookup = dict(
-            filter(lambda x: x is not None, map(get_region_for_site, sites))
-        )
+        self.sites_region_lookup = dict(map(get_region_for_site, sites))
 
     def refresh_regions_lookup(self):
         url = self.api_endpoint + "/api/dcim/regions/?limit=0"
@@ -632,6 +676,37 @@ def refresh_racks_lookup(self):
         racks = self.get_resource_list(api_url=url)
         self.racks_lookup = dict((rack["id"], rack["name"]) for rack in racks)
 
+        def get_group_for_rack(rack):
+            try:
+                return (rack["id"], rack["group"]["id"])
+            except Exception:
+                return (rack["id"], None)
+
+        def get_role_for_rack(rack):
+            try:
+                return (rack["id"], rack["role"]["slug"])
+            except Exception:
+                return (rack["id"], None)
+
+        self.racks_group_lookup = dict(map(get_group_for_rack, racks))
+        self.racks_role_lookup = dict(map(get_role_for_rack, racks))
+
+    def refresh_rack_groups_lookup(self):
+        url = self.api_endpoint + "/api/dcim/rack-groups/?limit=0"
+        rack_groups = self.get_resource_list(api_url=url)
+        self.rack_groups_lookup = dict(
+            (rack_group["id"], rack_group["slug"]) for rack_group in rack_groups
+        )
+
+        def get_rack_group_parent(rack_group):
+            try:
+                return (rack_group["id"], rack_group["parent"]["id"])
+            except Exception:
+                return (rack_group["id"], None)
+
+        # Dictionary of rack group id to parent rack group id
+        self.rack_group_parent_lookup = dict(map(get_rack_group_parent, rack_groups))
+
     def refresh_device_roles_lookup(self):
         url = self.api_endpoint + "/api/dcim/device-roles/?limit=0"
         device_roles = self.get_resource_list(api_url=url)
@@ -671,13 +746,8 @@ def get_cluster_group(cluster):
             except Exception:
                 return (cluster["id"], None)
 
-        self.clusters_type_lookup = dict(
-            filter(lambda x: x is not None, map(get_cluster_type, clusters))
-        )
-
-        self.clusters_group_lookup = dict(
-            filter(lambda x: x is not None, map(get_cluster_group, clusters))
-        )
+        self.clusters_type_lookup = dict(map(get_cluster_type, clusters))
+        self.clusters_group_lookup = dict(map(get_cluster_group, clusters))
 
     def refresh_services(self):
         url = self.api_endpoint + "/api/ipam/services/?limit=0"
@@ -825,6 +895,7 @@ def lookup_processes(self):
             self.refresh_regions_lookup,
             self.refresh_tenants_lookup,
             self.refresh_racks_lookup,
+            self.refresh_rack_groups_lookup,
             self.refresh_device_roles_lookup,
             self.refresh_platforms_lookup,
             self.refresh_device_types_lookup,

tests/integration/netbox-deploy.py

@@ -158,23 +158,34 @@
 core_switch = nb.dcim.device_roles.get(slug="core-switch")
 
 
-## Create Racks
-racks = [{"name": "Test Rack", "slug": "test-rack", "site": test_site2.id}]
-created_racks = nb.dcim.racks.create(racks)
-test_rack = nb.dcim.racks.get(slug="test-rack")
-
-
 ## Create Rack Groups
 rack_groups = [
-    {"name": "Test Rack Group", "slug": "test-rack-group", "site": test_site.id}
+    {"name": "Test Rack Group", "slug": "test-rack-group", "site": test_site.id},
+    {"name": "Parent Rack Group", "slug": "parent-rack-group", "site": test_site.id},
 ]
 created_rack_groups = nb.dcim.rack_groups.create(rack_groups)
 
+### Create Rack Group Parent relationship
+created_rack_groups[0].parent = created_rack_groups[1]
+created_rack_groups[0].save()
 
 ## Create Rack Roles
 rack_roles = [{"name": "Test Rack Role", "slug": "test-rack-role", "color": "4287f5"}]
 created_rack_roles = nb.dcim.rack_roles.create(rack_roles)
 
+## Create Racks
+racks = [
+    {
+        "name": "Test Rack Site 2",
+        "site": test_site2.id,
+        "role": created_rack_roles[0].id,
+    },
+    {"name": "Test Rack", "site": test_site.id, "group": created_rack_groups[0].id},
+]
+created_racks = nb.dcim.racks.create(racks)
+test_rack = nb.dcim.racks.get(name="Test Rack")  # racks don't have slugs
+test_rack_site2 = nb.dcim.racks.get(name="Test Rack Site 2")
+
 
 ## Create Devices
 devices = [
@@ -190,13 +201,14 @@
         "device_type": cisco_test.id,
         "device_role": core_switch.id,
         "site": test_site.id,
+        "rack": test_rack.id,
     },
     {
         "name": "R1-Device",
         "device_type": cisco_test.id,
         "device_role": core_switch.id,
         "site": test_site2.id,
-        "rack": test_rack.id,
+        "rack": test_rack_site2.id,
     },
     {
         "name": "Test Nexus One",

tests/integration/targets/inventory/files/test-inventory-legacy.json

@@ -13,8 +13,10 @@
                 "manufacturers": [
                     "cisco"
                 ],
+                "rack_groups": [],
+                "rack_role": "test-rack-role",
                 "racks": [
-                    "Test Rack"
+                    "Test Rack Site 2"
                 ],
                 "regions": [],
                 "services": [],
@@ -107,6 +109,13 @@
                 "manufacturers": [
                     "cisco"
                 ],
+                "rack_groups": [
+                    "test-rack-group",
+                    "parent-rack-group"
+                ],
+                "racks": [
+                    "Test Rack"
+                ],
                 "regions": [
                     "test-region",
                     "parent-region"

tests/integration/targets/inventory/files/test-inventory-options-flatten.json

@@ -14,6 +14,11 @@
         ]
     },
     "Test_Rack": {
+        "hosts": [
+            "TestDeviceR1"
+        ]
+    },
+    "Test_Rack_Site_2": {
         "hosts": [
             "R1-Device"
         ]
@@ -25,7 +30,9 @@
                 "interfaces": [],
                 "is_virtual": false,
                 "manufacturer": "cisco",
-                "rack": "Test Rack",
+                "rack": "Test Rack Site 2",
+                "rack_groups": [],
+                "rack_role": "test-rack-role",
                 "regions": [],
                 "role": "core-switch",
                 "services": [],
@@ -251,6 +258,11 @@
                 "interfaces": [],
                 "is_virtual": false,
                 "manufacturer": "cisco",
+                "rack": "Test Rack",
+                "rack_groups": [
+                    "test-rack-group",
+                    "parent-rack-group"
+                ],
                 "regions": [
                     "test-region",
                     "parent-region"
@@ -720,15 +732,19 @@
             "Test_Cluster",
             "Test_Cluster_2",
             "Test_Rack",
+            "Test_Rack_Site_2",
             "cisco",
             "cisco_test",
             "core_switch",
             "is_virtual",
             "nexus_parent",
             "other_region",
+            "parent_rack_group",
             "parent_region",
             "test_cluster_group",
             "test_cluster_type",
+            "test_rack_group",
+            "test_rack_role",
             "test_site2",
             "ungrouped"
         ]
@@ -771,6 +787,11 @@
             "Test Nexus One"
         ]
     },
+    "parent_rack_group": {
+        "hosts": [
+            "TestDeviceR1"
+        ]
+    },
     "parent_region": {
         "children": [
             "test_region"
@@ -794,6 +815,16 @@
             "test104-vm"
         ]
     },
+    "test_rack_group": {
+        "hosts": [
+            "TestDeviceR1"
+        ]
+    },
+    "test_rack_role": {
+        "hosts": [
+            "R1-Device"
+        ]
+    },
     "test_region": {
         "children": [
             "test_site"

tests/integration/targets/inventory/files/test-inventory-options-flatten.yml

@@ -17,6 +17,8 @@ group_by:
   - site
   - tenant
   - rack
+  - rack_groups
+  - rack_role
   - tag
   - role
   - device_type