Import Improvements

docs/dev/generator.md

@@ -63,16 +63,16 @@ The first data iteration constructs the wrapping structure, which includes:
 
 - `SourceAdapter` with all source model `adapter.wrappers`.
     - The `SourceAdapter` manages `SourceModelWrapper` and `NautobotModelWrapper` instances.
-- A `SourceModelWrapper` for each source content type, with `source_wrapper.fields` detailing how to import the source data.
+- A `SourceModelWrapper` for each source content type, with `fields` detailing how to import the source data.
     - Each `SourceModelWrapper` instance corresponds to a single `NautobotModelWrapper` instance.
 - A `NautobotModelWrapper` for each Nautobot content type, detailing `nautobot_wrapper.fields` and types, aiding in constructing the `DiffSyncModel` instances.
     - A single `NautobotModelWrapper` instance can be referenced by multiple `SourceModelWrapper` instances.
 
-During this phase, all non-defined but present source fields are appended to the `source_wrapper.fields`, focusing on field names, not values.
+During this phase, all non-defined but present source fields are appended to the `SourceModelWrapper.fields`, focusing on field names, not values.
 
 ### Creating Source Importers
 
-Convert each `source_wrapper.fields` item into a callable based on previously-established field definitions. The callables convert the source data into the `DiffSyncModel` constructor's expected structure.
+Convert each `SourceModelWrapper.fields` item into a callable based on previously-established field definitions. The callables convert the source data into the `DiffSyncModel` constructor's expected structure.
 
 In this stage, the structure described in the previous section is enhanced.
 
@@ -86,7 +86,7 @@ For each source record, the importer attempts to read the corresponding Nautobot
 
 ### Updating Referenced Content Types
 
-The updating of `content_types` fields, based on cached references, occurs in this phase. It's possible to define forwarding references using `source_wrapper.set_references_forwarding()`, e.g. references to `dcim.location` are forwarded to `dcim.locationtype`.
+The updating of `content_types` fields, based on cached references, occurs in this phase. It's possible to define forwarding references using `SourceModelWrapper.set_references_forwarding()`, e.g. references to `dcim.location` are forwarded to `dcim.locationtype`.
 
 ### Syncing to Nautobot
 

nautobot_netbox_importer/diffsync/__init__.py

@@ -1 +1,4 @@
-"""DiffSync adapter and model implementation for nautobot-netbox-importer."""
+"""DiffSync adapter and model implementation for nautobot-netbox-importer.
+
+This folder is an importer implementation specific to NetBox, in opposite to `generator` folder, that is a generic Source => Nautobot importer.
+"""

nautobot_netbox_importer/diffsync/adapters/netbox.py

@@ -2,27 +2,32 @@
 
 from gzip import GzipFile
 from pathlib import Path
-from typing import Callable, Generator, NamedTuple, Union
+from typing import Callable, Generator, NamedTuple, Sequence, Union
 from urllib.parse import ParseResult, urlparse
 
 import ijson
 import requests
 from django.core.management import call_command
 from django.db.transaction import atomic
+from packaging.version import Version
 
 from nautobot_netbox_importer.base import GENERATOR_SETUP_MODULES, logger, register_generator_setup
+from nautobot_netbox_importer.diffsync.models.cables import create_missing_cable_terminations
 from nautobot_netbox_importer.diffsync.models.dcim import fix_power_feed_locations, unrack_zero_uheight_devices
 from nautobot_netbox_importer.generator import SourceAdapter, SourceDataGenerator, SourceRecord
 from nautobot_netbox_importer.summary import Pathable
 
 for _name in (
     "base",
+    "cables",
     "circuits",
+    "content_types",
     "custom_fields",
     "dcim",
     "ipam",
     "locations",
     "object_change",
+    "tags",
     "virtualization",
 ):
     register_generator_setup(f"nautobot_netbox_importer.diffsync.models.{_name}")
@@ -46,13 +51,17 @@ class NetBoxImporterOptions(NamedTuple):
     bypass_data_validation: bool = False
     print_summary: bool = False
     update_paths: bool = False
+    deduplicate_prefixes: bool = False
     fix_powerfeed_locations: bool = False
     sitegroup_parent_always_region: bool = False
+    create_missing_cable_terminations: bool = False
     tag_issues: bool = False
     unrack_zero_uheight_devices: bool = True
     save_json_summary_path: str = ""
     save_text_summary_path: str = ""
     trace_issues: bool = False
+    customizations: Sequence[str] = []
+    netbox_version: Version = Version("3.7")
 
 
 AdapterSetupFunction = Callable[[SourceAdapter], None]
@@ -76,18 +85,28 @@ def __init__(self, input_ref: _FileRef, options: NetBoxImporterOptions, job=None
 
         self.options = options
 
+        for name in options.customizations:
+            if name:
+                register_generator_setup(name)
+
         for name in GENERATOR_SETUP_MODULES:
             setup = __import__(name, fromlist=["setup"]).setup
             setup(self)
 
     def load(self) -> None:
         """Load data from NetBox."""
         self.import_data()
+
         if self.options.fix_powerfeed_locations:
             fix_power_feed_locations(self)
+
         if self.options.unrack_zero_uheight_devices:
             unrack_zero_uheight_devices(self)
-        self.post_import()
+
+        if self.options.create_missing_cable_terminations:
+            create_missing_cable_terminations(self)
+
+        self.post_load()
 
     def import_to_nautobot(self) -> None:
         """Import a NetBox export file into Nautobot."""

nautobot_netbox_importer/diffsync/models/base.py

@@ -1,85 +1,21 @@
 """NetBox to Nautobot Base Models Mapping."""
 
-from diffsync.enum import DiffSyncModelFlags
+from packaging.version import Version
 
-from nautobot_netbox_importer.base import RecordData
-from nautobot_netbox_importer.generator import DiffSyncBaseModel, SourceAdapter, SourceField, fields
+from nautobot_netbox_importer.diffsync.adapters.netbox import NetBoxAdapter
+from nautobot_netbox_importer.diffsync.models.locations import define_locations
+from nautobot_netbox_importer.generator import fields
 
-from .locations import define_locations
 
-
-def _define_tagged_object(field: SourceField) -> None:
-    wrapper = field.wrapper
-    adapter = wrapper.adapter
-    tag_wrapper = adapter.get_or_create_wrapper("extras.tag")
-
-    def tagged_object_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
-        object_id = source.get(field.name, None)
-        if not object_id:
-            return
-
-        tag = source.get(tag_field.name, None)
-        content_type = source.get(content_type_field.name, None)
-        if not tag or not content_type:
-            raise ValueError(f"Missing content_type or tag for tagged object {object_id}")
-
-        tag_uuid = tag_wrapper.get_pk_from_uid(tag)
-        related_wrapper = adapter.get_or_create_wrapper(content_type)
-        result = related_wrapper.get_pk_from_uid(object_id)
-        field.set_nautobot_value(target, result)
-        tag_field.set_nautobot_value(target, tag_uuid)
-        content_type_field.set_nautobot_value(target, related_wrapper.nautobot.content_type_instance.pk)
-        related_wrapper.add_reference(tag_wrapper, tag_uuid)
-
-    field.set_importer(tagged_object_importer)
-    tag_field = field.handle_sibling("tag", "tag")
-    content_type_field = field.handle_sibling("content_type", "content_type")
-
-
-def _setup_content_types(adapter: SourceAdapter) -> None:
-    """Map NetBox content types to Nautobot.
-
-    Automatically calculate NetBox content type IDs, if not provided, based on the order of the content types.
-    """
-    netbox = {"id": 0}
-
-    def define_app_label(field: SourceField) -> None:
-        def content_types_mapper_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
-            app_label = source["app_label"]
-            model = source["model"]
-            netbox["id"] += 1
-            uid = source.get("id", None)
-            if uid:
-                if uid != netbox["id"]:
-                    raise ValueError(f"Content type id mismatch: {uid} != {netbox['id']}")
-            else:
-                uid = netbox["id"]
-
-            wrapper = adapter.get_or_create_wrapper(f"{app_label}.{model}")
-            adapter.content_type_ids_mapping[uid] = wrapper
-            field.set_nautobot_value(target, app_label)
-
-        field.set_importer(content_types_mapper_importer)
-
-    adapter.configure_model(
-        "contenttypes.ContentType",
-        identifiers=["app_label", "model"],
-        flags=DiffSyncModelFlags.IGNORE,
-        nautobot_flags=DiffSyncModelFlags.IGNORE,
-        fields={
-            "app_label": define_app_label,
-        },
-    )
-
-
-def setup(adapter: SourceAdapter) -> None:
+def setup(adapter: NetBoxAdapter) -> None:
     """Map NetBox base models to Nautobot."""
-    adapter.disable_model("sessions.session", "Nautobot has own sessions, sessions should never cross apps.")
+    netbox_version = adapter.options.netbox_version
+
     adapter.disable_model("admin.logentry", "Not directly used in Nautobot.")
-    adapter.disable_model("users.userconfig", "May not have a 1 to 1 translation to Nautobot.")
     adapter.disable_model("auth.permission", "Handled via a Nautobot model and may not be a 1 to 1.")
-
-    _setup_content_types(adapter)
+    adapter.disable_model("extras.imageattachment", "Images are not imported yet.")
+    adapter.disable_model("sessions.session", "Nautobot has own sessions, sessions should never cross apps.")
+    adapter.disable_model("users.userconfig", "May not have a 1 to 1 translation to Nautobot.")
 
     adapter.configure_model(
         "extras.Status",
@@ -89,18 +25,7 @@ def setup(adapter: SourceAdapter) -> None:
         },
     )
     adapter.configure_model("extras.role")
-    adapter.configure_model(
-        "extras.tag",
-        fields={
-            "object_types": "content_types",
-        },
-    )
-    adapter.configure_model(
-        "extras.TaggedItem",
-        fields={
-            "object_id": _define_tagged_object,
-        },
-    )
+
     adapter.configure_model(
         "extras.ConfigContext",
         fields={
@@ -110,13 +35,15 @@ def setup(adapter: SourceAdapter) -> None:
     )
     adapter.configure_model(
         # pylint: disable=hard-coded-auth-user
-        "auth.User",
+        "auth.User" if netbox_version < Version("4") else "users.User",
         nautobot_content_type="users.User",
         identifiers=["username"],
         fields={
             "last_login": fields.disable("Should not be attempted to migrate"),
             "password": fields.disable("Should not be attempted to migrate"),
             "user_permissions": fields.disable("Permissions import is not implemented yet"),
+            "object_permissions": fields.disable("Permissions import is not implemented yet"),
+            "groups": fields.disable("Groups import is not implemented yet"),
         },
     )
     adapter.configure_model(