Update CI for NetBox 4.0 (#1230)

* Add CI for NetBox 4.0
* Update inventories for NetBox 4.0

.github/workflows/main.yml

@@ -25,12 +25,12 @@ jobs:
       #  uses: chartboost/ruff-action@v1
 
   unit_testing:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: linting
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
 
     steps:
 
@@ -71,19 +71,17 @@ jobs:
     runs-on: ubuntu-latest
     needs: unit_testing
     env:
-      python-version: "3.9"
+      python-version: "3.10"
     strategy:
       fail-fast: false
       matrix:
         include:
-          - VERSION: "v3.5"
-            NETBOX_DOCKER_VERSION: 2.6.1
           - VERSION: "v3.6"
             NETBOX_DOCKER_VERSION: 2.7.0
           - VERSION: "v3.7"
             NETBOX_DOCKER_VERSION: 2.7.0
-        # If we want to integration test wiht all supported Python:
-        #python-version: ["3.9", "3.10", "3.11"]
+          - VERSION: "v4.0"
+            NETBOX_DOCKER_VERSION: 2.9.1
 
     steps:
 

pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Mikhail Yohman <[email protected]>"]
 license = "GPLv3"
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.10"
 ansible-core = "2.15.9"
 black = "*"
 codecov = "*"
@@ -15,7 +15,7 @@ deepdiff = "*"
 cryptography = "*"
 jinja2 = "*"
 jmespath = "*"
-pynetbox = "^7"
+pynetbox = "^7.3"
 pytest = "*"
 pytest-mock = "*"
 pytest-xdist = "*"
@@ -27,6 +27,7 @@ importlib-metadata = "*"
 pylint = "*"
 pytz = "*"
 packaging = "*"
+importlib-resources = "*"
 
 [tool.poetry.dev-dependencies]
 

tests/integration/netbox-deploy.py

@@ -576,7 +576,10 @@ def make_netbox_calls(endpoint, payload):
         "type": "vxlan",
     },
 ]
-created_l2vpns = make_netbox_calls(nb.ipam.l2vpns, l2vpns)
+if nb_version >= version.parse("3.7"):
+    created_l2vpns = make_netbox_calls(nb.vpn.l2vpns, l2vpns)
+else:
+    created_l2vpns = make_netbox_calls(nb.ipam.l2vpns, l2vpns)
 
 if ERRORS:
     sys.exit(

tests/integration/targets/inventory-v4.0/.gitignore

@@ -0,0 +1 @@
+runme_config

tests/integration/targets/inventory-v4.0/aliases

@@ -0,0 +1 @@
+# https://docs.ansible.com/ansible/devel/dev_guide/testing/sanity/integration-aliases.html

tests/integration/targets/inventory-v4.0/compare_inventory_json.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+# Inspired by community.aws collection script_inventory_ec2 test
+# https://github.com/ansible-collections/community.aws/blob/master/tests/integration/targets/script_inventory_ec2/inventory_diff.py
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import argparse
+import json
+import sys
+from operator import itemgetter
+
+from deepdiff import DeepDiff
+
+# NetBox includes "created" and "last_updated" times on objects. These end up in the interfaces objects that are included verbatim from the NetBox API.
+# "url" may be different if local tests use a different host/port
+# Remove these from files saved in git as test data
+KEYS_REMOVE = frozenset(["created", "last_updated", "url"])
+
+# Ignore these when performing diffs as they will be different for each test run
+# (Was previously keys specific to NetBox 2.6)
+KEYS_IGNORE = frozenset()
+
+# Rack Groups became hierarchical in NetBox 2.8. Don't bother comparing against test data in NetBox 2.7
+KEYS_IGNORE_27 = frozenset(
+    [
+        "rack_groups",  # host var
+        "rack_group_parent_rack_group",  # group, group_names_raw = False
+        "parent_rack_group",  # group, group_names_raw = True
+    ]
+)
+
+
+# Assume the object will not be recursive, as it originally came from JSON
+def remove_keys(obj, keys):
+    if isinstance(obj, dict):
+        keys_to_remove = keys.intersection(obj.keys())
+        for key in keys_to_remove:
+            del obj[key]
+
+        for key, value in obj.items():
+            remove_keys(value, keys)
+
+    elif isinstance(obj, list):
+        # Iterate over temporary copy, as we may remove items
+        for item in obj[:]:
+            if isinstance(item, str) and item in keys:
+                # List contains a string that we want to remove
+                # eg. a group name in list of groups
+                obj.remove(item)
+            remove_keys(item, keys)
+
+
+def sort_hostvar_arrays(obj):
+    meta = obj.get("_meta")
+    if not meta:
+        return
+
+    hostvars = meta.get("hostvars")
+    if not hostvars:
+        return
+
+    for _, host in hostvars.items():
+        if interfaces := host.get("interfaces"):
+            host["interfaces"] = sorted(interfaces, key=itemgetter("id"))
+
+        if services := host.get("services"):
+            host["services"] = sorted(services, key=itemgetter("id"))
+
+
+def read_json(filename):
+    with open(filename, "r", encoding="utf-8") as file:
+        return json.loads(file.read())
+
+
+def write_json(filename, data):
+    with open(filename, "w", encoding="utf-8") as file:
+        json.dump(data, file, indent=4)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Diff Ansible inventory JSON output")
+    parser.add_argument(
+        "filename_a",
+        metavar="ORIGINAL.json",
+        type=str,
+        help="Original json to test against",
+    )
+    parser.add_argument(
+        "filename_b",
+        metavar="NEW.json",
+        type=str,
+        help="Newly generated json to compare against original",
+    )
+    parser.add_argument(
+        "--write",
+        action="store_true",
+        help=(
+            "When comparing files, various keys are removed. "
+            "This option will not compare the files, and instead writes ORIGINAL.json to NEW.json after removing these keys. "
+            "This is used to clean the test json files before saving to the git repo. "
+            "For example, this removes dates. "
+        ),
+    )
+    parser.add_argument(
+        "--netbox-version",
+        metavar="VERSION",
+        type=str,
+        help=(
+            "Apply comparison specific to NetBox version. "
+            "For example, rack_groups arrays will only contain a single item in v2.7, so are ignored in the comparison."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    data_a = read_json(args.filename_a)
+
+    if args.write:
+        # When writing test data, only remove "remove_keys" that will change on every git commit.
+        # This makes diffs more easily readable to ensure changes to test data look correct.
+        remove_keys(data_a, KEYS_REMOVE)
+        sort_hostvar_arrays(data_a)
+        write_json(args.filename_b, data_a)
+
+    else:
+        data_b = read_json(args.filename_b)
+
+        # Ignore keys that we don't want to diff, in addition to the ones removed that change on every commit
+        keys = KEYS_REMOVE.union(KEYS_IGNORE)
+        remove_keys(data_a, keys)
+        remove_keys(data_b, keys)
+
+        sort_hostvar_arrays(data_a)
+        sort_hostvar_arrays(data_b)
+
+        # Perform the diff
+        result = DeepDiff(data_a, data_b, ignore_order=True)
+
+        if result:
+            # Dictionary is not empty - print differences
+            print(json.dumps(result, sort_keys=True, indent=4))
+            sys.exit(1)
+        else:
+            # Success, no differences
+            sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()