Migrate to Nornir 3.x and refactor inventory to support multiple SOT

examples/multi_site_02/network_importer.toml

@@ -8,7 +8,7 @@ import_vlans = "config"
 
 nbr_workers = 1
 
-[inventory]
+[inventory.settings]
 filter="region=ni_multi_site_02"
 
 [logs]

examples/spine_leaf_01/network_importer.toml

@@ -6,7 +6,7 @@ import_prefixes = true
 import_cabling = "config"
 import_vlans = "config"
 
-[inventory]
+[inventory.settings]
 filter = "site=ni_spine_leaf_01"
 
 [logs]

network_importer/adapters/base.py

@@ -1,17 +1,4 @@
-"""BaseAdapter for the network importer.
-
-(c) 2020 Network To Code
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+"""BaseAdapter for the network importer."""
 from diffsync import DiffSync
 from network_importer.models import Site, Device, Interface, IPAddress, Cable, Vlan, Prefix
 
@@ -27,16 +14,24 @@ class BaseAdapter(DiffSync):
     vlan = Vlan
     prefix = Prefix
 
-    def __init__(self, nornir, config):
-        """Initialize the base adapter and store the nornir object locally."""
+    settings_class = None
+    settings = None
+
+    def __init__(self, nornir, settings):
+        """Initialize the base adapter and store the Nornir object locally."""
         super().__init__()
         self.nornir = nornir
-        self.config = self._validate_config(config)
+        self.settings = self._validate_settings(settings)
+
+    def _validate_settings(self, settings):
+        """Load and validate the configuration based on the settings_class."""
+        if self.settings_class:
+            if settings and isinstance(dict, settings):
+                return self.settings_class(**settings)  # pylint: disable=not-callable
+
+            return self.settings_class()  # pylint: disable=not-callable
 
-    @classmethod
-    def _validate_config(cls, config):
-        """Placeholder method that can be reimplemented by each adapter to validate if its config is valid."""
-        return config
+        return settings
 
     def load(self):
         """Load the local cache with data from the remove system."""

network_importer/adapters/nautobot_api/adapter.py

@@ -1,17 +1,4 @@
-"""NautobotAPIAdapter class.
-
-(c) 2020 Network To Code
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+"""NautobotAPIAdapter class."""
 import logging
 import warnings
 
@@ -31,6 +18,7 @@
     NautobotVlan,
 )
 from network_importer.adapters.nautobot_api.tasks import query_device_info_from_nautobot
+from network_importer.adapters.nautobot_api.settings import InventorySettings, AdapterSettings
 
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 
@@ -56,12 +44,13 @@ class NautobotAPIAdapter(BaseAdapter):
     nautobot = None
     nautobot_version = None
 
+    settings_class = AdapterSettings
+
     type = "Nautobot"
 
     query_device_info_from_nautobot = query_device_info_from_nautobot
 
-    @staticmethod
-    def _is_tag_present(nautobot_obj):
+    def _is_tag_present(self, nautobot_obj):
         """Find if tag is present for a given object."""
         if isinstance(nautobot_obj, dict) and not nautobot_obj.get("tags", None):  # pylint: disable=no-else-return
             return False
@@ -73,20 +62,19 @@ def _is_tag_present(nautobot_obj):
         elif not nautobot_obj["tags"]:
             return False
 
-        for tag in config.SETTINGS.nautobot.model_flag_tags:
+        for tag in self.settings.model_flag_tags:
             if tag in nautobot_obj["tags"]:
                 LOGGER.debug(
                     "Tag (%s) found for object %s. Marked for diffsync flag assignment.", tag, nautobot_obj,
                 )
                 return True
         return False
 
-    @staticmethod
-    def apply_model_flag(diffsync_obj, nautobot_obj):
+    def apply_model_flag(self, diffsync_obj, nautobot_obj):
         """Helper function for DiffSync Flag assignment."""
-        model_flag = config.SETTINGS.nautobot.model_flag
+        model_flag = self.settings.model_flag
 
-        if model_flag and NautobotAPIAdapter._is_tag_present(nautobot_obj):
+        if model_flag and self._is_tag_present(nautobot_obj):
             LOGGER.info(
                 "DiffSync model flag (%s) applied to object %s", model_flag, nautobot_obj,
             )
@@ -109,9 +97,10 @@ def _check_nautobot_version(self):
 
     def load(self):
         """Initialize pynautobot and load all data from nautobot in the local cache."""
-        self.nautobot = pynautobot.api(url=config.SETTINGS.nautobot.address, token=config.SETTINGS.nautobot.token)
+        inventory_settings = InventorySettings(**config.SETTINGS.inventory.settings)
+        self.nautobot = pynautobot.api(url=inventory_settings.address, token=inventory_settings.token)
 
-        if not config.SETTINGS.nautobot.verify_ssl:
+        if not inventory_settings.verify_ssl:
             self.nautobot.http_session.verify_ssl = False
         else:
             self.nautobot.http_session.verify_ssl = True

network_importer/adapters/nautobot_api/exceptions.py

@@ -1,17 +1,4 @@
-"""Custom Exceptions for the NautobotAPIAdapter.
-
-(c) 2020 Network To Code
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+"""Custom Exceptions for the NautobotAPIAdapter."""
 
 
 class NautobotObjectNotValid(Exception):

network_importer/adapters/nautobot_api/inventory.py

@@ -0,0 +1,174 @@
+"""Norning Inventory for nautobot."""
+# Disable too-many-arguments and too-many-locals pylint tests for this file. These are both necessary
+# pylint: disable=R0913,R0914,E1101,W0613
+
+from typing import Any, List, Optional
+import requests
+import pynautobot
+
+from nornir.core.inventory import Defaults, Group, Groups, Hosts, Inventory, ParentGroups, ConnectionOptions
+from nornir.core.plugins.inventory import InventoryPluginRegister
+from network_importer.inventory import NetworkImporterInventory, NetworkImporterHost
+from network_importer.utils import build_filter_params
+from network_importer.adapters.nautobot_api.settings import InventorySettings
+
+
+class NautobotAPIInventory(NetworkImporterInventory):
+    """Nautobot API Inventory Class."""
+
+    settings_class = InventorySettings
+
+    # pylint: disable=dangerous-default-value, too-many-branches, too-many-statements
+    def __init__(
+        self,
+        username: Optional[str],
+        password: Optional[str],
+        enable: Optional[bool],
+        supported_platforms: Optional[List[str]],
+        limit=Optional[str],
+        settings: InventorySettings = InventorySettings(),
+        **kwargs: Any,
+    ) -> None:
+        """Nornir Inventory Plugin for Nautobot API."""
+        super().__init__(
+            username=username,
+            password=password,
+            enable=enable,
+            limit=limit,
+            supported_platforms=supported_platforms,
+            settings=settings,
+            **kwargs,
+        )
+
+        self.settings = self.settings_class(**settings)
+
+        # Build Filter based on inventory_params filter and on limit
+        self.filter_parameters = {}
+        build_filter_params(self.settings.filter.split((",")), self.filter_parameters)
+        if limit:
+            if "=" not in limit:
+                self.filter_parameters["name"] = limit
+            else:
+                build_filter_params(limit.split((",")), self.filter_parameters)
+
+        if "exclude" not in self.filter_parameters.keys():
+            self.filter_parameters["exclude"] = "config_context"
+
+        # Instantiate nautobot session using pynautobot
+        self.session = pynautobot.api(url=self.settings.address, token=self.settings.token)
+        if not self.settings.verify_ssl:
+            session = requests.Session()
+            session.verify = False
+            self.session.http_session = session
+
+    def load(self):
+        """Load inventory by fetching devices from nautobot."""
+        if self.filter_parameters:
+            devices: List[pynautobot.modules.dcim.Devices] = self.session.dcim.devices.filter(**self.filter_parameters)
+        else:
+            devices: List[pynautobot.modules.dcim.Devices] = self.session.dcim.devices.all()
+
+        # fetch all platforms from nautobot and build mapping:   platform:  napalm_driver
+        platforms = self.session.dcim.platforms.all()
+        platforms_mapping = {platform.slug: platform.napalm_driver for platform in platforms if platform.napalm_driver}
+
+        hosts = Hosts()
+        groups = Groups()
+        defaults = Defaults()
+
+        global_group = Group(
+            name="global", connection_options={"netmiko": ConnectionOptions(), "napalm": ConnectionOptions()}
+        )
+
+        # Pull the login and password from the NI config object if available
+        if self.username:
+            global_group.username = self.username
+
+        if self.password:
+            global_group.password = self.password
+            if self.enable:
+                global_group.connection_options["netmiko"].extras = {
+                    "secret": self.password,
+                    "global_delay_factor": self.settings.global_delay_factor,
+                    "banner_timeout": self.settings.banner_timeout,
+                    "conn_timeout": self.settings.conn_timeout,
+                }
+                global_group.connection_options["napalm"].extras = {"optional_args": {"secret": self.password}}
+
+        for dev in devices:
+            # nautobot allows devices to be unnamed, but the Nornir model does not allow this
+            # If a device is unnamed we will set the name to the id of the device in nautobot
+            dev_name = dev.name or dev.id
+            host = NetworkImporterHost(name=dev_name, connection_options=ConnectionOptions())
+
+            # Only add virtual chassis master as inventory element
+            if dev.virtual_chassis and dev.virtual_chassis.master:
+                if dev.id != dev.virtual_chassis.master.id:
+                    continue
+                host.data["virtual_chassis"] = True
+
+            else:
+                host.data["virtual_chassis"] = False
+
+            # If supported_platforms is provided
+            # skip all devices that do not match the list of supported platforms
+            # TODO need to see if we can filter when doing the query directly
+            if self.supported_platforms:
+                if not dev.platform:
+                    continue
+
+                if dev.platform.slug not in self.supported_platforms:
+                    continue
+
+            # Add value for IP address
+            if self.settings.use_primary_ip and dev.primary_ip:
+                host.hostname = dev.primary_ip.address.split("/")[0]
+            elif self.settings.use_primary_ip and not dev.primary_ip:
+                host.is_reachable = False
+                host.not_reachable_reason = "primary ip not defined in nautobot"
+            elif not self.settings.use_primary_ip and self.settings.fqdn:
+                host.hostname = f"{dev.name}.{self.settings.fqdn}"
+            elif not self.settings.use_primary_ip:
+                host.hostname = dev.name
+            else:
+                host.hostname = dev_name
+
+            host.site_name = dev.site.slug
+
+            host.data["serial"] = dev.serial
+            host.data["vendor"] = dev.device_type.manufacturer.slug
+            host.data["asset_tag"] = dev.asset_tag
+            host.data["custom_fields"] = dev.custom_fields
+            host.data["site_id"] = dev.site.id
+            host.data["device_id"] = dev.id
+            host.data["role"] = dev.device_role.slug
+            host.data["model"] = dev.device_type.slug
+
+            # Attempt to add 'platform' based of value in 'slug'
+            if dev.platform and dev.platform.slug in platforms_mapping:
+                host.connection_options = {"napalm": ConnectionOptions(platform=platforms_mapping[dev.platform.slug])}
+
+            if dev.platform:
+                host.platform = dev.platform.slug
+            else:
+                host.platform = None
+
+            host.groups = ParentGroups([global_group])
+
+            if dev.site.slug not in groups.keys():
+                groups[dev.site.slug] = {}
+
+            if dev.device_role.slug not in groups.keys():
+                groups[dev.device_role.slug] = {}
+
+            if host.hostname and host.platform:
+                host.is_reachable = True
+
+            # Assign temporary dict to outer dict
+
+            hosts[dev_name] = host
+
+        return Inventory(hosts=hosts, groups=groups, defaults=defaults)
+
+
+InventoryPluginRegister.register("NautobotAPIInventory", NautobotAPIInventory)

network_importer/adapters/nautobot_api/models.py

@@ -1,17 +1,4 @@
-"""Extension of the base Models for the NautobotAPIAdapter.
-
-(c) 2020 Network To Code
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+"""Extension of the base Models for the NautobotAPIAdapter."""
 from typing import Optional
 import logging
 
@@ -344,6 +331,8 @@ def create(cls, diffsync: "DiffSync", ids: dict, attrs: dict) -> Optional["DiffS
         try:
             item = super().create(ids=ids, diffsync=diffsync, attrs=attrs)
             nb_params = item.translate_attrs_for_nautobot(attrs)
+            # Add status because it's a mandatory field.
+            nb_params["status"] = "active"
             ip_address = diffsync.nautobot.ipam.ip_addresses.create(**nb_params)
         except pynautobot.core.query.RequestError as exc:
             LOGGER.warning("Unable to create the ip address %s in %s (%s)", ids["address"], diffsync.name, exc.error)