From b32ec48dbace55537ffd30411e33dae87b83e23a Mon Sep 17 00:00:00 2001 From: Seth Grover <13872653+mmguero@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:06:32 -0600 Subject: [PATCH 01/39] fix(repo): add REPO_URL=local sentinel to bypass git entirely (#124) DTLRepo always ran a clone or a fetch, even when REPO_PATH already held the library contents locally. validate_git_url() rejected the old 'local' value outright, and even a valid URL still triggered a real git fetch against the remote on every run. REPO_URL=local now skips Repo(), clone_from(), and fetch() entirely. REPO_PATH is used as-is and must already contain device-types/, module-types/, and rack-types/ (no .git required, and none is used). --- core/repo.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/repo.py b/core/repo.py index 869bedd6..74688641 100644 --- a/core/repo.py +++ b/core/repo.py @@ -419,11 +419,13 @@ class DTLRepo: def __init__(self, config, handle): """Initialize repository management, updating an existing clone or creating a new one. - If the target path already holds a Git clone, the repository will be updated from - its configured remote; otherwise the provided URL is validated and a new clone is - created. The initializer sets instance attributes used by other methods (handler, - supported YAML extensions, URL, repo path, branch, repo reference, and current - working directory). + If REPO_URL is the sentinel "local", no git operation of any kind is performed — + REPO_PATH is used as-is and must already contain the device-type file tree. + Otherwise, if the target path already holds a Git clone, the repository will be + updated from its configured remote; if not, the provided URL is validated and a new + clone is created. The initializer sets instance attributes used by other methods + (handler, supported YAML extensions, URL, repo path, branch, repo reference, and + current working directory). Args: config (RunConfig): Supplies `repo_url`, `repo_branch`, and `repo_path`. @@ -441,6 +443,14 @@ def __init__(self, config, handle): self.repo = None self.cwd = os.getcwd() + if str(self.url).strip().casefold() == "local": + if not os.path.isdir(self.get_absolute_path()): + raise InvalidRepoPathError( + self.repo_path, reason="REPO_URL=local requires REPO_PATH to already contain the library files" + ) + self.handle.log(f"REPO_URL=local: using {self.get_absolute_path()} as-is, no git operations") + return + is_path_valid, path_error = validate_repo_path(self.repo_path) if not is_path_valid: raise InvalidRepoPathError(self.repo_path, reason=path_error) From 5694aaa8965afd3d88e5146649b2d78550b902b6 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:35:07 +0200 Subject: [PATCH 02/39] fix(repo): validate the local library layout and share the check with export (#126) * fix(repo): validate the local library layout and share the check with export REPO_URL=local only checked that REPO_PATH was a directory. An unrelated or empty path passed that check, and discover_vendors skips type directories that are absent, so the run imported nothing and still exited 0. Export mode already had the check it needed, in Exporter._verify_repo_available. Both sides now read one definition of what makes a checkout a library: LIBRARY_TYPE_DIRS and library_dirs_present() in core/repo.py. The sentinel value moves to LOCAL_REPO_URL in core/config.py, beside the other REPO_* defaults, so config and repo cannot drift on it. Local mode still skips validate_repo_path on purpose: that check demands write access, which a read-only or air-gapped mount cannot give, and no import step writes to REPO_PATH. A test pins the read-only case. REPO_BRANCH is ignored under the sentinel, so a run that sets both now says so through the existing config notice mechanism instead of looking like it checked the branch out. Documents the mode in the README and .env.example, including the read-only Docker mount, which is the case that motivated the sentinel. * fix(import): treat an absent type root as empty instead of crashing The layout check accepts any one of device-types/, module-types/, and rack-types/, matching what export mode already accepted. plan_vendor then called get_devices() on all three regardless, and get_devices() lists the directory, so a local checkout holding only some of them raised FileNotFoundError before importing the types it did hold. A device-types-only library is the likeliest local layout, and it crashed on module-types. _parse_vendor_racks already had the guard this needed. It is now _parse_vendor_files and all three roots go through it, so the guard cannot apply to one root and not the others again. The repo mock in test_nb_dt_import pointed at /tmp/devices, /tmp/modules and /tmp/rack-types, paths that never existed. Nothing stat'd them, so the mock passed for a filesystem that was not there, which is why this went unseen. It now points at a real empty library tree, and the tests that matched on those literal paths match on the directory names instead. Found by CodeRabbit on #126. --- .env.example | 6 +++ README.md | 25 ++++++++++- core/config.py | 14 +++++- core/export.py | 8 ++-- core/import_run.py | 25 ++++++----- core/repo.py | 49 +++++++++++++++----- tests/test_config.py | 19 ++++++++ tests/test_exporter.py | 30 +++++++++++++ tests/test_import_run.py | 31 +++++++++++++ tests/test_nb_dt_import.py | 36 ++++++++++----- tests/test_repo.py | 91 ++++++++++++++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index be9a7476..0c6b5dc4 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,12 @@ NETBOX_URL=https://netbox.example.org REPO_BRANCH=master REPO_URL=https://github.com/netbox-community/devicetype-library.git +# Set REPO_URL=local to read REPO_PATH as it stands, with no git operation at all: +# no clone, no fetch, and no .git needed. REPO_PATH must already hold at least one of +# device-types/, module-types/, or rack-types/. REPO_BRANCH is ignored in this mode. +# Use it for air-gapped runs or a library you version yourself. +#REPO_URL=local + # Local path where the device-type library repository is cloned. # Defaults to a "repo" directory in the project root when not set. # Use an absolute path or a path relative to where you run the script. diff --git a/README.md b/README.md index 447a98e9..5b004a04 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ and device, creating anything that is missing from NetBox while skipping entries | --- | --- | --- | --- | | `NETBOX_URL` | ✅ | — | URL of your NetBox instance | | `NETBOX_TOKEN` | ✅ | — | API token with write access | -| `REPO_URL` | | community library | Git URL of the device-type library to clone | +| `REPO_URL` | | community library | Git URL of the device-type library to clone. Set to `local` to read `REPO_PATH` as it stands, with no git operation (see [Offline / local library](#offline--local-library)) | | `REPO_BRANCH` | | `master` | Branch to check out | | `REPO_PATH` | | `./repo` | Local path where the library is cloned. Accepts absolute or relative paths. | | `VENDORS` | | all | Comma-separated vendors to import (same effect as `--vendors`) | @@ -435,6 +435,29 @@ uv run nb-dt-import.py --vendors nokia --verify-images - After replacing a local image file with a higher-quality version and wanting NetBox to pick it up +#### Offline / local library + +Set `REPO_URL=local` to point the importer at a directory that already holds the library and +skip git completely: no clone, no fetch, and no `.git` needed. `REPO_PATH` must contain at +least one of `device-types/`, `module-types/`, or `rack-types/`, and an import stops with an +error if it does not, rather than reporting an empty library as nothing to do. + +Nothing in an import writes to `REPO_PATH`, so a read-only mount works. `REPO_BRANCH` is +ignored in this mode, and a run that sets both says so. + +```shell +REPO_URL=local REPO_PATH=/srv/devicetype-library uv run nb-dt-import.py +``` + +```shell +docker run --rm -e REPO_URL=local -e REPO_PATH=/library \ + -v /srv/devicetype-library:/library:ro \ + -e NETBOX_URL -e NETBOX_TOKEN ghcr.io/marcinpsk/device-type-library-import +``` + +**When to use**: air-gapped networks, a library you version yourself, or any run that must not +reach the network for the library. + #### Export Mode `--export-diff` runs in the opposite direction to every other mode: instead of importing the diff --git a/core/config.py b/core/config.py index f71d9074..7a67f694 100644 --- a/core/config.py +++ b/core/config.py @@ -11,6 +11,8 @@ DEFAULT_REPO_URL = "https://github.com/netbox-community/devicetype-library.git" DEFAULT_REPO_BRANCH = "master" +# REPO_URL value that reads REPO_PATH as it stands, with no git operation at all. +LOCAL_REPO_URL = "local" DEFAULT_GRAPHQL_PAGE_SIZE = 5000 DEFAULT_PRELOAD_THREADS = 8 @@ -20,6 +22,11 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def is_local_repo_url(url): + """Return True when *url* is the sentinel that turns off every git operation.""" + return str(url or "").strip().casefold() == LOCAL_REPO_URL + + class ConfigError(FatalError): """A configuration value the run cannot start with, phrased for the person who set it.""" @@ -102,7 +109,7 @@ def build_argument_parser(env): "--url", "--git", default=_text(env, "REPO_URL", DEFAULT_REPO_URL), - help="Git URL with valid Device Type YAML files", + help=f'Git URL with valid Device Type YAML files, or "{LOCAL_REPO_URL}" to read REPO_PATH with no git', ) parser.add_argument( "--slugs", @@ -264,6 +271,11 @@ def resolve_run_config(argv=None, env=None): # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: + notices.append( + f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " + "and checks out no branch." + ) return RunConfig( netbox_url=_text(env, "NETBOX_URL"), diff --git a/core/export.py b/core/export.py index 02490761..c3e43aa1 100644 --- a/core/export.py +++ b/core/export.py @@ -28,12 +28,10 @@ serialize_rack_type, ) from core.netbox_api import IMAGE_EXTENSIONS, _build_auth_header +from core.repo import LIBRARY_TYPE_DIRS, library_dirs_present _SKIP = object() # sentinel: image already exists, no download needed -# Top-level directories that make a checkout a device-type library. -_LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") - # Maps Content-Type to a canonical extension for extension-less attachments. _CONTENT_TYPE_EXT = { "image/png": ".png", @@ -474,11 +472,11 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: def _verify_repo_available(self) -> None: """Raise FileNotFoundError when the library is absent, which would otherwise read as an empty one.""" - if any((self.repo_path / name).is_dir() for name in _LIBRARY_TYPE_DIRS): + if library_dirs_present(self.repo_path): return raise FileNotFoundError( f"No device-type library found at {self.repo_path}: expected at least one of " - f"{', '.join(_LIBRARY_TYPE_DIRS)}. Export mode does not clone the library. " + f"{', '.join(LIBRARY_TYPE_DIRS)}. Export mode does not clone the library. " "Clone it to that path, or run an import first, which clones it for you." ) diff --git a/core/import_run.py b/core/import_run.py index 37a3caa9..00da6870 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -529,12 +529,15 @@ def _log_run_summary(handle, summary): handle.log("These duplicates would otherwise oscillate on every run. Please report/fix them upstream.") -def _parse_vendor_racks(repo, racks_path, vendor_name, slugs): - """Parse rack types for one vendor when the rack path exists.""" - if not os.path.isdir(racks_path): +def _parse_vendor_files(repo, base_path, vendor_name, slugs): + """Parse one vendor's types under *base_path*, treating an absent type root as empty. + + A local library may ship only some of the three type roots, which is a layout, not a fault. + """ + if not os.path.isdir(base_path): return [] - rack_files, _ = repo.get_devices(racks_path, [vendor_name.casefold()]) - return repo.parse_files(rack_files, slugs=slugs) + files, _ = repo.get_devices(base_path, [vendor_name.casefold()]) + return repo.parse_files(files, slugs=slugs) def _finalize_task_registry(progress, task_registry): @@ -641,16 +644,18 @@ def plan_vendor(self, selection, vendor): device_files = slug_resolved["device_files"].get(vendor["slug"], []) device_types = self.repo.parse_files(device_files) if device_files else [] else: - device_files, _ = self.repo.get_devices(selection.devices_path, [vendor["name"].casefold()]) - device_types = self.repo.parse_files(device_files, slugs=self.config.slugs or []) + device_types = _parse_vendor_files( + self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] + ) if self.netbox.modules: module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] else: - module_files, _ = self.repo.get_devices(selection.modules_path, [vendor["name"].casefold()]) - module_types = self.repo.parse_files(module_files, slugs=self.config.slugs or []) + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) else: module_types = [] @@ -659,7 +664,7 @@ def plan_vendor(self, selection, vendor): if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] else: - rack_types = _parse_vendor_racks( + rack_types = _parse_vendor_files( self.repo, selection.racks_path, vendor["name"], diff --git a/core/repo.py b/core/repo.py index 74688641..57656278 100644 --- a/core/repo.py +++ b/core/repo.py @@ -11,8 +11,12 @@ from git import Repo, exc import yaml +from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +# Top-level directories that make a checkout a device-type library. +LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") + class GitCommandError(FatalError): """A Git command that failed for a repository.""" @@ -184,6 +188,11 @@ def _safe_index_load(path: str): return _safe_pickle_load(path) +def library_dirs_present(path): + """Return True when *path* holds at least one device-type library directory.""" + return any(os.path.isdir(os.path.join(str(path), name)) for name in LIBRARY_TYPE_DIRS) + + def validate_git_url(url): """Determine whether a Git remote URL is allowed (HTTPS, SSH, or file://). @@ -419,11 +428,11 @@ class DTLRepo: def __init__(self, config, handle): """Initialize repository management, updating an existing clone or creating a new one. - If REPO_URL is the sentinel "local", no git operation of any kind is performed — - REPO_PATH is used as-is and must already contain the device-type file tree. - Otherwise, if the target path already holds a Git clone, the repository will be - updated from its configured remote; if not, the provided URL is validated and a new - clone is created. The initializer sets instance attributes used by other methods + If REPO_URL is the sentinel "local", no git operation runs at all: REPO_PATH is read + as it stands and must already hold the device-type file tree. Otherwise, if the target + path already holds a Git clone, the repository will be updated from its configured + remote; if not, the provided URL is validated and a new clone is created. The + initializer sets instance attributes used by other methods (handler, supported YAML extensions, URL, repo path, branch, repo reference, and current working directory). @@ -443,12 +452,8 @@ def __init__(self, config, handle): self.repo = None self.cwd = os.getcwd() - if str(self.url).strip().casefold() == "local": - if not os.path.isdir(self.get_absolute_path()): - raise InvalidRepoPathError( - self.repo_path, reason="REPO_URL=local requires REPO_PATH to already contain the library files" - ) - self.handle.log(f"REPO_URL=local: using {self.get_absolute_path()} as-is, no git operations") + if is_local_repo_url(self.url): + self._use_local_checkout() return is_path_valid, path_error = validate_repo_path(self.repo_path) @@ -467,6 +472,28 @@ def __init__(self, config, handle): raise InvalidGitURLError(self.url, reason=error_msg) self.clone_repo() + def _use_local_checkout(self): + """Accept REPO_PATH as it stands: no clone, no fetch, and no .git needed. + + Skips validate_repo_path on purpose: it demands write access, which a read-only + or air-gapped mount cannot give, and no import step writes to REPO_PATH. + """ + path = self.get_absolute_path() + if not os.path.isdir(path): + raise InvalidRepoPathError( + self.repo_path, + reason=f"REPO_URL={LOCAL_REPO_URL} needs REPO_PATH to be an existing directory", + ) + if not library_dirs_present(path): + raise InvalidRepoPathError( + self.repo_path, + reason=( + f"No device-type library found: expected at least one of {', '.join(LIBRARY_TYPE_DIRS)} " + f"inside it. REPO_URL={LOCAL_REPO_URL} does not clone the library." + ), + ) + self.handle.log(f"REPO_URL={LOCAL_REPO_URL}: reading {path} as it stands, with no git operation") + def get_relative_path(self): """Get the repository path configured for this instance relative to the current working directory. diff --git a/tests/test_config.py b/tests/test_config.py index 0bb65ee4..1aae1838 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -120,3 +120,22 @@ def test_an_explicit_repo_path_is_taken_as_given(self, tmp_path): config = _resolve(REPO_PATH=str(tmp_path)) assert config.repo_path == str(tmp_path) + + +class TestLocalRepoUrlIgnoresTheBranch: + """REPO_URL=local checks out nothing, so a REPO_BRANCH set beside it silently does nothing.""" + + def test_a_branch_set_beside_the_sentinel_is_reported_as_ignored(self): + config = _resolve(REPO_URL="local", REPO_BRANCH="feature") + + assert any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + def test_the_sentinel_alone_needs_no_notice(self): + config = _resolve(REPO_URL="local") + + assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + def test_a_branch_set_beside_a_real_url_needs_no_notice(self): + config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") + + assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices diff --git a/tests/test_exporter.py b/tests/test_exporter.py index ae5d1797..a356e528 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1485,3 +1485,33 @@ def test_all_known_content_types_are_recognised(self): for ct, ext in _CONTENT_TYPE_EXT.items(): result = _sanitize_attachment_filename("img", "/media/img", ct) assert result.endswith(ext), f"Expected {ext} for {ct}, got {result}" + + +class TestRepoAvailability: + """Export mode and REPO_URL=local must agree on what makes a directory a library checkout.""" + + def _exporter(self, tmp_path, repo_path): + settings = replace(_make_settings(tmp_path), repo_path=str(repo_path)) + return Exporter(settings, _make_handle(), str(tmp_path / "extra"), False, None) + + @pytest.mark.parametrize("present", ["device-types", "module-types", "rack-types"]) + def test_one_library_directory_is_enough(self, present, tmp_path): + repo = tmp_path / "repo" + (repo / present).mkdir(parents=True) + + self._exporter(tmp_path, repo)._verify_repo_available() + + def test_a_directory_without_library_directories_stops_the_export(self, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + + with pytest.raises(FileNotFoundError, match="No device-type library found"): + self._exporter(tmp_path, repo)._verify_repo_available() + + def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "device-types").write_text("not a directory") + + with pytest.raises(FileNotFoundError, match="No device-type library found"): + self._exporter(tmp_path, repo)._verify_repo_available() diff --git a/tests/test_import_run.py b/tests/test_import_run.py index caeb96fa..59f14405 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -8,6 +8,7 @@ from core.errors import VendorSelectionError from core.import_run import ImportRun, RunSummary, VendorPlan, _process_device_types from core.log_handler import LogHandler +from core.repo import DTLRepo class _ComponentCache: @@ -277,3 +278,33 @@ def test_execute_releases_console_when_progress_setup_fails(make_config, tmp_pat assert progress_factory.exited is True assert handle.console is None assert netbox.device_types.components.close_count == 1 + + +class TestPartialLibraryLayouts: + """A local checkout may hold one type root only, so planning must not read the absent ones.""" + + TYPE_DIRS = ("device-types", "module-types", "rack-types") + + def _repo(self, tmp_path, present, make_config): + """Build a real DTLRepo over a checkout holding *present* alone.""" + vendor = tmp_path / present / "TestVendor" + vendor.mkdir(parents=True) + (vendor / "thing.yaml").write_text("manufacturer: TestVendor\nmodel: Test\nslug: test\n") + config = make_config(repo_url="local", repo_path=str(tmp_path)) + return DTLRepo(config, LogHandler(False)), config + + @pytest.mark.parametrize("present", TYPE_DIRS) + def test_one_type_root_plans_without_reading_the_absent_ones(self, present, make_config, tmp_path): + repo, config = self._repo(tmp_path, present, make_config) + run = ImportRun(config, repo, _NetBoxBoundary(), LogHandler(False), _ProgressFactory()) + + selection = run.discover() + plan = run.plan_vendor(selection, {"name": "TestVendor", "slug": "testvendor"}) + + parsed = { + "device-types": len(plan.device_types), + "module-types": len(plan.module_types), + "rack-types": len(plan.rack_types), + } + assert parsed[present] == 1, parsed + assert sum(parsed.values()) == 1, parsed diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 71da96d2..929231ca 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -308,13 +308,27 @@ def test_items_per_second_column_uses_elapsed_fallback_when_finished_speed_missi _PROJECT_ROOT = Path(__file__).resolve().parents[1] +_LIBRARY_ROOT = None + + +@pytest.fixture(scope="session", autouse=True) +def _real_library_root(tmp_path_factory): + """Point the repo mocks at a real library tree: the pipeline stats these paths before reading them.""" + global _LIBRARY_ROOT + root = tmp_path_factory.mktemp("library") + for name in ("device-types", "module-types", "rack-types"): + (root / name).mkdir() + _LIBRARY_ROOT = root + yield root + + def _make_mock_repo(device_types=None): """Return a pre-configured DTLRepo mock with no files by default.""" mock_repo = MagicMock() mock_repo.get_devices.return_value = ([], []) - mock_repo.get_devices_path.return_value = "/tmp/devices" - mock_repo.get_modules_path.return_value = "/tmp/modules" - mock_repo.get_racks_path.return_value = "/tmp/rack-types" + mock_repo.get_devices_path.return_value = str(_LIBRARY_ROOT / "device-types") + mock_repo.get_modules_path.return_value = str(_LIBRARY_ROOT / "module-types") + mock_repo.get_racks_path.return_value = str(_LIBRARY_ROOT / "rack-types") mock_repo.discover_vendors.return_value = [] mock_repo.parse_files.return_value = device_types if device_types is not None else [] mock_repo.resolve_slug_files.return_value = None # no pickle available by default @@ -905,7 +919,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): repo = _make_mock_repo() repo.discover_vendors.return_value = [{"name": "Vendor One", "slug": "vendor-one"}] repo.get_devices.side_effect = lambda path, vendors=None: ( - (["module.yaml"], []) if "modules" in path else ([], []) + (["module.yaml"], []) if "module-types" in path else ([], []) ) repo.parse_files.side_effect = lambda files, slugs=None: [module_type] if files == ["module.yaml"] else [] MockRepo.return_value = repo @@ -1600,13 +1614,13 @@ def test_no_pulse_bar_column_uses_static_empty_bar_for_unknown_total(self, nb_dt assert bar.total == 1.0 assert bar.completed == 0.0 - def test_parse_vendor_racks_calls_repo_when_directory_exists(self, nb_dt_import): + def test_parse_vendor_files_calls_repo_when_directory_exists(self, nb_dt_import): repo = MagicMock() repo.get_devices.return_value = (["rack.yaml"], []) repo.parse_files.return_value = [{"model": "Rack"}] with patch("core.import_run.os.path.isdir", return_value=True): - result = import_run_module._parse_vendor_racks(repo, "/racks", "nokia", ["rack"]) + result = import_run_module._parse_vendor_files(repo, "/racks", "nokia", ["rack"]) assert result == [{"model": "Rack"}] repo.get_devices.assert_called_once_with("/racks", ["nokia"]) @@ -1844,7 +1858,7 @@ def _parse_files(files, slugs=None, progress=None): return [] mock_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) mock_repo.parse_files.side_effect = _parse_files mock_nb = _make_mock_netbox() @@ -1878,7 +1892,7 @@ def test_main_stops_preload_job_in_finally_on_error(self, nb_dt_import): mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Cisco", "slug": "cisco"}] mock_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) mock_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["device.yaml"] else [] @@ -1964,14 +1978,14 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c dtl_repo = _make_mock_repo() dtl_repo.get_devices.side_effect = lambda path, vendors=None: ( ([f"{vendors[0]}-{path.split('/')[-1]}.yaml"], []) - if vendors and vendors[0] == "cisco" and path in {"/tmp/modules", "/tmp/rack-types"} + if vendors and vendors[0] == "cisco" and path.endswith(("module-types", "rack-types")) else ([], []) ) dtl_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["resolved.yaml"] else [{"manufacturer": {"slug": "cisco"}, "model": "M", "slug": "m"}] - if files == ["cisco-modules.yaml"] + if files == ["cisco-module-types.yaml"] else [] ) netbox = _make_mock_netbox(modules=True) @@ -2030,7 +2044,7 @@ def test_import_run_stops_preload_in_finally_on_error(self, make_config): dtl_repo = _make_mock_repo() dtl_repo.discover_vendors.return_value = [{"slug": "cisco", "name": "Cisco"}] dtl_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) dtl_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["device.yaml"] else [] diff --git a/tests/test_repo.py b/tests/test_repo.py index a260c8bb..ab4e695a 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -154,6 +154,97 @@ def test_invalid_path_raises_before_repository_access(self, tmp_path): _dtl_repo(config, str(invalid_path), LogHandler(False)) +class TestDTLRepoLocalMode: + """REPO_URL=local reads REPO_PATH as it stands: no clone, no fetch, no .git.""" + + @staticmethod + def _make_library(path, *dir_names): + """Create a library checkout at *path* holding one YAML file under each named directory.""" + path.mkdir(parents=True, exist_ok=True) + for name in dir_names: + vendor = path / name / "TestVendor" + vendor.mkdir(parents=True) + (vendor / "device.yaml").write_text("manufacturer: TestVendor\nmodel: Test\n") + return path + + def _init_local(self, repo_path, url="local", handle=None): + config = MagicMock(repo_url=url, repo_branch="master") + return _dtl_repo(config, str(repo_path), handle or LogHandler(False)) + + def test_reads_the_checkout_without_running_git(self, tmp_path, mock_git_repo): + """The whole point of the sentinel: the files are read and git is never reached.""" + library = self._make_library(tmp_path / "library", "device-types") + + repo = self._init_local(library) + files, vendors = repo.get_devices(repo.get_devices_path()) + + assert [os.path.basename(path) for path in files] == ["device.yaml"] + assert vendors == [{"name": "TestVendor", "slug": "testvendor"}] + assert [parsed["model"] for parsed in repo.parse_files(files)] == ["Test"] + assert not (library / ".git").exists() + mock_git_repo.assert_not_called() + mock_git_repo.clone_from.assert_not_called() + + @pytest.mark.parametrize("value", ["local", "LOCAL", "Local", " local "]) + def test_the_sentinel_ignores_case_and_padding(self, value, tmp_path, mock_git_repo): + library = self._make_library(tmp_path / "library", "device-types") + + self._init_local(library, url=value) + + mock_git_repo.assert_not_called() + mock_git_repo.clone_from.assert_not_called() + + @pytest.mark.parametrize("present", ["device-types", "module-types", "rack-types"]) + def test_one_library_directory_is_enough(self, present, tmp_path, mock_git_repo): + """Export mode accepts any one of the three, so local import mode must agree.""" + library = self._make_library(tmp_path / "library", present) + + self._init_local(library) + + mock_git_repo.assert_not_called() + + def test_a_missing_directory_is_reported_as_a_path_error(self, tmp_path): + with pytest.raises(InvalidRepoPathError, match="existing directory"): + self._init_local(tmp_path / "absent") + + def test_a_directory_without_library_directories_is_rejected(self, tmp_path): + """An unrelated path would otherwise import nothing and still exit successfully.""" + empty = tmp_path / "library" + empty.mkdir() + + with pytest.raises(InvalidRepoPathError, match="No device-type library found"): + self._init_local(empty) + + def test_a_stray_file_named_like_a_library_directory_is_rejected(self, tmp_path): + library = tmp_path / "library" + library.mkdir() + (library / "device-types").write_text("not a directory") + + with pytest.raises(InvalidRepoPathError, match="No device-type library found"): + self._init_local(library) + + @pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores the write bit") + def test_a_read_only_checkout_is_accepted(self, tmp_path): + """The air-gapped case: nothing writes to REPO_PATH, so a read-only mount must work.""" + library = self._make_library(tmp_path / "library", "device-types") + library.chmod(0o555) + + try: + repo = self._init_local(library) + assert repo.get_devices(repo.get_devices_path())[0] + finally: + library.chmod(0o755) + + def test_the_branch_is_left_unused(self, tmp_path, mock_git_repo): + """No checkout happens, so REPO_BRANCH must not reach git (config warns about it instead).""" + library = self._make_library(tmp_path / "library", "device-types") + config = MagicMock(repo_url="local", repo_branch="a-branch-that-does-not-exist") + + _dtl_repo(config, str(library), LogHandler(False)) + + mock_git_repo.assert_not_called() + + class TestDTLRepoRealGit: """Clone/pull branching driven against a real local Git repository (no git mocks).""" From d5e848aa2c898cf3ef0b029c6dd5316443696705 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:02:18 +0200 Subject: [PATCH 03/39] feat(module-bay-types): carry NetBox 4.7 module bay types through the importer (#132) --- core/change_detector.py | 139 ++++++- core/compat.py | 20 + core/component_registry.py | 24 +- core/export.py | 11 +- core/graphql_client.py | 63 +++- core/module_bay_types.py | 241 ++++++++++++ core/nb_serializer.py | 30 +- core/netbox_api.py | 238 +++++++++++- tests/conftest.py | 8 +- tests/helpers.py | 199 ++++++++++ tests/test_component_registry.py | 17 +- tests/test_exporter.py | 32 +- tests/test_graphql_client.py | 143 ++++++- tests/test_module_bay_type_sync.py | 577 +++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 250 +++++++++++++ tests/test_nb_dt_import.py | 46 ++- tests/test_nb_serializer.py | 66 ++++ tests/test_netbox_api.py | 243 ++++++------ tests/test_relation_scope.py | 188 ++++++++++ tests/test_suite_hygiene.py | 19 + 20 files changed, 2383 insertions(+), 171 deletions(-) create mode 100644 core/module_bay_types.py create mode 100644 tests/test_module_bay_type_sync.py create mode 100644 tests/test_module_bay_types.py create mode 100644 tests/test_relation_scope.py diff --git a/core/change_detector.py b/core/change_detector.py index 24fbd214..12ea6f0d 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -10,7 +10,7 @@ from typing import Any, List, Optional from enum import Enum -from core.component_registry import COMPONENT_TYPES +from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES from core.normalization import normalize_values from core.formatting import log_property_diffs from core.schema_reader import load_properties_for_type @@ -26,6 +26,110 @@ class ChangeType(Enum): COMPONENT_REMOVED = "component_removed" +def _manufacturer_slug_of(yaml_data): + """Return the owning manufacturer's slug from a parsed definition. + + ``core.repo`` rewrites every YAML manufacturer to ``{"slug": ...}`` before the importer + sees it, so that is the usual shape here. + """ + manufacturer = (yaml_data or {}).get("manufacturer") + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or "" + return manufacturer or "" + + +def _relation_properties(comp_type): + """Return the relation field names the registry declares for *comp_type*.""" + component = BY_YAML_KEY.get(comp_type) + return component.relations if component else () + + +def _is_relation_list(value): + """Return True when *value* is a list of non-empty strings, the only shape a reference takes.""" + return isinstance(value, list) and all(isinstance(item, str) and item for item in value) + + +def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): + """Return a PropertyChange when a relation differs, or None when there is nothing to do. + + A relation is an unordered set of related objects. Where the catalog is available the + comparison is between *identities*, ``(manufacturer slug, slug)``, because two + manufacturers may define the same class name and a bay holding the wrong one reads as + equal on names alone. Without a catalog it falls back to comparing names, which is + what a caller that has not wired one can still do. + + An omitted key leaves the relation unmanaged and an empty list clears it, but a bare + ``module_bay_types:`` parses as None and a malformed entry is not a name: those are left + unmanaged, because reading them as empty would clear a restriction nobody removed. A + field the query did not return is skipped for the same reason. + """ + if prop not in yaml_comp: + return None + declared = yaml_comp.get(prop) + if not _is_relation_list(declared): + if handle is not None: + handle.log( + f"Ignored {prop} on {yaml_comp.get('name', 'Unknown')!r}: expected a list of names, got {declared!r}" + ) + return None + netbox_value = getattr(netbox_comp, prop, _MISSING) + if netbox_value is _MISSING: + return None + + if catalog is not None and manufacturer: + from core.module_bay_types import ModuleBayTypeError + + try: + wanted = catalog.identities_for(manufacturer, declared) + except ModuleBayTypeError: + # Reported as a change so the write path sees it; "no change" is silent. + return PropertyChange( + property_name=prop, + old_value=sorted(_relation_names(netbox_value)), + new_value=sorted(set(declared)), + ) + current = _relation_identities(netbox_value) + if current is not None and wanted == current: + return None + if current is not None: + return PropertyChange( + property_name=prop, old_value=sorted(_relation_names(netbox_value)), new_value=sorted(set(declared)) + ) + + yaml_names = frozenset(declared) + netbox_names = frozenset(_relation_names(netbox_value)) + if yaml_names == netbox_names: + return None + return PropertyChange(property_name=prop, old_value=sorted(netbox_names), new_value=sorted(yaml_names)) + + +def _relation_identities(value): + """Return {(manufacturer slug, slug)} for a relation, or None if the shape lacks them. + + A read path that returns only ``id`` and ``name`` cannot answer the scope question, so + the caller falls back to comparing names rather than inventing an identity. + """ + identities = set() + for item in value or []: + slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) + manufacturer = item.get("manufacturer") if isinstance(item, dict) else getattr(item, "manufacturer", None) + owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) + if not slug or not owner: + return None + identities.add((owner, slug)) + return frozenset(identities) + + +def _relation_names(value): + """Return the ``name`` of each related object NetBox returned for a relation.""" + names = [] + for item in value or []: + name = item.get("name") if isinstance(item, dict) else getattr(item, "name", None) + if name is not None: + names.append(name) + return names + + @dataclass class PropertyChange: """Represents a single property change.""" @@ -339,7 +443,11 @@ def _compare_components( # Check for property changes on existing component existing = existing_components[comp_name] prop_changes = self._compare_component_properties( - yaml_comp, existing, component.compare_properties, comp_type=yaml_key + yaml_comp, + existing, + component.compare_properties, + comp_type=yaml_key, + manufacturer=_manufacturer_slug_of(yaml_data), ) if prop_changes: changes.append( @@ -353,14 +461,31 @@ def _compare_components( return changes + def _relation_catalog(self): + """Return the module bay type catalog, or None when this detector has no real one. + + Tests build the detector around a stand-in for DeviceTypes, and a stand-in cannot + answer what a reference means. Returning None there keeps the name comparison, + which is what those tests are asserting on. + """ + from core.module_bay_types import ModuleBayTypeCatalog + + catalog = getattr(self.device_types, "module_bay_types", None) + return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None + def _compare_component_properties( self, yaml_comp: dict, netbox_comp, properties: List[str], comp_type: str = "", + manufacturer: str = "", ) -> List[PropertyChange]: - """Compare properties between YAML and NetBox component.""" + """Compare properties between YAML and NetBox component. + + *manufacturer* is the owning manufacturer's slug, used to resolve a relation + reference to the object it means rather than the name it is written as. + """ changes = [] for prop in properties: @@ -426,6 +551,14 @@ def _compare_component_properties( ) continue + if prop in _relation_properties(comp_type): + relation_change = _relation_change( + prop, yaml_comp, netbox_comp, self._relation_catalog(), manufacturer, self.handle + ) + if relation_change is not None: + changes.append(relation_change) + continue + # Only compare properties explicitly present in the YAML component; # an omitted property means the YAML doesn't manage it (absent key != removal). if prop not in yaml_comp: diff --git a/core/compat.py b/core/compat.py index 62c33bf1..97c59bf5 100644 --- a/core/compat.py +++ b/core/compat.py @@ -14,6 +14,26 @@ from __future__ import annotations +import re + +# Selecting or sending the relation below this release fails the whole query. +MODULE_BAY_TYPE_MINIMUM_VERSION = (4, 7) + + +def parse_netbox_version(version) -> tuple[int, int]: + """Return ``(major, minor)`` from a NetBox version string. + + Padded to two parts so a single-component string cannot raise, and tolerant of the + suffixes NetBox ships ("4.7.0-beta2"). + """ + raw = [int(re.sub(r"\D.*", "", part.strip()) or "0") for part in str(version).split(".")] + return tuple((raw + [0, 0])[:2]) # type: ignore[return-value] + + +def supports_module_bay_types(version) -> bool: + """Return True when this NetBox release exposes the module bay type relation.""" + return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION + def device_type_filter_key(new_filters: bool) -> str: """Return the correct filter parameter name for device-type component queries. diff --git a/core/component_registry.py b/core/component_registry.py index ed991052..a1fc18e7 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -17,6 +17,12 @@ LINK_POWER_PORT = "power_port" LINK_REAR_PORTS = "rear_ports" +# Fields holding names that must become NetBox ids before a POST. +RELATION_MODULE_BAY_TYPES = "module_bay_types" + +# Relation fields a module type carries itself, rather than through one of its components. +MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) + @dataclass(frozen=True) class ComponentType: @@ -27,19 +33,30 @@ class ComponentType: label: str fields: tuple[str, ...] module_types: bool = True + relations: tuple[str, ...] = field(default_factory=tuple) graphql_extra: tuple[str, ...] = field(default_factory=tuple) compare_extra: tuple[str, ...] = field(default_factory=tuple) link: Optional[str] = None @property def graphql_fields(self): - """Fields to select in a GraphQL query, including the id every consumer needs.""" - return ["id", *self.fields, *self.graphql_extra] + """Fields to select in a GraphQL query, including the id every consumer needs. + + A relation is a list of related objects, so it is selected by name rather than + read as a scalar. + """ + return ["id", *self.fields, *self.graphql_extra, *self.graphql_relation_fields] + + @property + def graphql_relation_fields(self): + """GraphQL selections for this row's relations, one nested block per relation.""" + # slug plus owning manufacturer is the identity; the name alone is ambiguous. + return [f"{name} {{ id name slug manufacturer {{ slug }} }}" for name in self.relations] @property def compare_properties(self): """Properties change detection compares between YAML and NetBox.""" - return [*self.fields, *self.compare_extra] + return [*self.fields, *self.compare_extra, *self.relations] @property def list_key(self): @@ -121,6 +138,7 @@ def create_label(self, parent_type): endpoint="module_bay_templates", label="Module Bay", fields=("name", "position", "label", "description"), + relations=(RELATION_MODULE_BAY_TYPES,), ), ) diff --git a/core/export.py b/core/export.py index c0dd57ac..f0911ee6 100644 --- a/core/export.py +++ b/core/export.py @@ -241,6 +241,9 @@ def run(self, progress=None) -> None: ) self.handle.log(f"Export-diff: fetching NetBox device/module/rack types{scope}") + # Decided before the first query: the selection depends on the answer. + self.graphql.detect_module_bay_type_support() + # ── Fetch all types from NetBox ────────────────────────────────────── by_model, by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) all_mt = self.graphql.get_module_types(manufacturer_slugs=self.vendor_slugs) @@ -624,13 +627,7 @@ def _fetch_vendor_components(self, mfr_slug: str) -> tuple: def _fetch_one(endpoint_name): if not getattr(_thread_local, "graphql", None): - client = NetBoxGraphQLClient( - self.graphql.url, - self.graphql.token, - self.graphql.ignore_ssl, - self.graphql.handle, - self.graphql.DEFAULT_PAGE_SIZE, - ) + client = self.graphql.clone() _thread_local.graphql = client with _clients_lock: _clients.append(client) diff --git a/core/graphql_client.py b/core/graphql_client.py index 70baabfd..4a8fe786 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -5,12 +5,14 @@ compatible with the existing REST-based code in ``netbox_api.py``. """ +import copy import threading import time from collections.abc import Sequence import requests +from core.compat import supports_module_bay_types from core.component_registry import BY_ENDPOINT # Module-level dedup: tracks (url, requested_page_size) pairs that have already @@ -102,6 +104,9 @@ def _to_dotdict(obj): # connections, which is transient during a long paginated run. _RETRYABLE_STATUSES = {429, 500, 502, 503, 504} +# One small metadata read; a run that cannot get it fails at the next query anyway. +_STATUS_TIMEOUT_SECONDS = 30 + def _response_body_detail(response): """Return the response body as a suffix for an error message, truncated and stripped.""" @@ -136,7 +141,7 @@ class NetBoxGraphQLClient: or raise the server's ``MAX_PAGE_SIZE`` setting to match. """ - def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): + def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000, supports_module_bay_types=False): """Store connection parameters for later use in :meth:`query`. Args: @@ -148,6 +153,8 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): ``print`` when not provided. page_size: Default number of items per GraphQL page (default: 5 000). + supports_module_bay_types: True when NetBox is >= 4.7 and its schema + exposes the module bay type relation. """ self.DEFAULT_PAGE_SIZE = page_size self.url = url.rstrip("/") @@ -155,22 +162,28 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): self.token = token self.ignore_ssl = ignore_ssl self._handle = handle + self.supports_module_bay_types = supports_module_bay_types + + self._session = self._new_session() - self._session = requests.Session() + def _new_session(self): + """Return an HTTP session carrying this client's auth and TLS settings.""" + session = requests.Session() # v2 tokens start with "nbt_" prefix (format: nbt_.); # v1 tokens are plain 40-char hex strings using legacy Token auth. auth_scheme = "Bearer" if self.token.startswith("nbt_") else "Token" - self._session.headers.update( + session.headers.update( { "Authorization": f"{auth_scheme} {self.token}", "Content-Type": "application/json", } ) - self._session.verify = not self.ignore_ssl + session.verify = not self.ignore_ssl if self.ignore_ssl: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + return session @property def handle(self): @@ -178,8 +191,35 @@ def handle(self): return self._handle def clone(self): - """Return an independent client with the same connection settings.""" - return type(self)(self.url, self.token, self.ignore_ssl, self.handle, self.DEFAULT_PAGE_SIZE) + """Return an independent client with the same settings and its own HTTP session. + + Copied rather than reconstructed, so a setting added to ``__init__`` travels with + the clone instead of quietly reverting to its default. The session is the one + thing a worker must not share, so it is the one thing rebuilt here. + """ + clone = copy.copy(self) + clone._session = self._new_session() + return clone + + def detect_module_bay_type_support(self): + """Ask NetBox for its version and record whether the relation can be selected. + + The importer learns this from its pynetbox client; the export entry point has no + such client, so it asks here. Both sides decide from core.compat, so the line + cannot drift between them. + """ + status_url = f"{self.url}/api/status/" + try: + response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) + response.raise_for_status() + version = response.json().get("netbox-version", "") + except requests.RequestException as exc: + raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc + except ValueError as exc: + # A proxy error page answers 200 with HTML, so the body is not JSON. + raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + self.supports_module_bay_types = supports_module_bay_types(version) + return self.supports_module_bay_types def close(self): """Close the underlying HTTP session.""" @@ -477,6 +517,12 @@ def get_module_types(self, manufacturer_slugs=None): sequence of non-blank strings. """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) + module_bay_type_selection = ( + "module_bay_types {\n id\n name\n slug\n" + " manufacturer {\n slug\n }\n }\n " + if self.supports_module_bay_types + else "" + ) query = f""" query($pagination: OffsetPaginationInput{var_decl}) {{ @@ -490,7 +536,7 @@ def get_module_types(self, manufacturer_slugs=None): weight weight_unit last_updated - manufacturer {{ + {module_bay_type_selection}manufacturer {{ id name slug @@ -801,6 +847,9 @@ def get_component_templates(self, endpoint_name, manufacturer_slug=None, on_page raise ValueError("manufacturer_slug must be None or a non-empty string") fields = component.graphql_fields + if not self.supports_module_bay_types: + # Selecting a field the server's schema lacks fails the whole query. + fields = [f for f in fields if f not in component.graphql_relation_fields] list_key = component.list_key parent_fields = "device_type { id }" diff --git a/core/module_bay_types.py b/core/module_bay_types.py new file mode 100644 index 00000000..68a23fbc --- /dev/null +++ b/core/module_bay_types.py @@ -0,0 +1,241 @@ +"""Resolve module-bay-type reference names to NetBox ids. + +A device type's bay says which classes of module it accepts, and a module type says +which classes it belongs to. Both sides name those classes as plain strings, so +something has to turn a name into the id of one specific NetBox object, creating it +when the target instance has never seen it. + +That is this module. Callers pass the owning manufacturer and the names; they get +ids back. The scope rule, the catalog files, the lookup, the creation, and the cache +stay in here, so the four call sites (device-type create, module-type create, change +detection, export) do not each restate them. +""" + +import os +import re + +import pynetbox +import requests +import yaml + +from core.errors import FatalError + +# Directory in the devicetype-library holding the catalog, one directory per manufacturer. +CATALOG_DIRNAME = "module-bay-types" + +# Second resolution scope, for a class one vendor defines and another fills. +FALLBACK_MANUFACTURER = "Generic" + + +def manufacturer_slug(name): + """Slugify a manufacturer name the way the library loader does. + + ``core.repo`` reduces every YAML ``manufacturer`` to this slug, because upstream data + disagrees on case (RuggedCOM vs RuggedCom). The catalog keys on the same slug so both + sides meet on one value. + """ + return re.sub(r"\W+", "-", (name or "").lower()) + + +class ModuleBayTypeError(FatalError): + """A reference could not be resolved to exactly one NetBox object. + + Raised per definition so the caller can log it and continue with the next one. + Never raised for a difference this module could paper over: an unresolved name and + a conflicting identity are both reported rather than guessed at. + """ + + +class ModuleBayTypeCatalog: + """Turn module-bay-type names into NetBox ids, creating what is missing. + + ``ids_for(manufacturer, names)`` resolves each name in the owning manufacturer's + scope, then in ``Generic``, and raises :class:`ModuleBayTypeError` if neither has + it. Resolution creates the NetBox object, and the manufacturer that owns it, when + they do not exist yet. Returned order is not meaningful; callers compare as sets. + + Export does not use this class: it reads the names off the records NetBox returned, + because an export run resolves nothing and so has no id-to-name mapping of its own. + """ + + def __init__(self, netbox, repo_path, handle): + """Store the NetBox client, the library checkout to read the catalog from, and the log handle.""" + self._netbox = netbox + self._catalog_dir = os.path.join(repo_path, CATALOG_DIRNAME) + self._handle = handle + self._entries = None + self._load_error = None + self._ids = {} + self._manufacturer_ids = {} + + def ids_for(self, manufacturer, names): + """Return the NetBox id for each name, resolved against *manufacturer* then Generic. + + *manufacturer* is a manufacturer slug, as produced by :func:`manufacturer_slug`. + An empty list is a valid instruction to hold no classes; anything that is not a + list of non-empty strings is refused rather than read as empty, because reading a + malformed value as empty would drop a restriction the author asked for. + """ + self._validate(names) + # Duplicates collapse: the relationship is a set. + return sorted({self._id_for(manufacturer, name) for name in names}) + + def identities_for(self, manufacturer, names): + """Return the identity each name resolves to, without touching NetBox. + + The identity is ``(manufacturer slug, slug)``: the object a reference means, not + the name it is written as. Two manufacturers may both define a class called + ``X``, so a name alone cannot say which object is intended. + + This is the query half of the module. :meth:`ids_for` is the command half and + creates what is missing; this one reads only the catalog files, so change + detection can ask what a reference means without causing anything to exist. + """ + self._validate(names) + return frozenset( + (manufacturer_slug(entry["manufacturer"]), entry["slug"]) + for entry in (self._lookup(manufacturer, name) for name in names) + ) + + @staticmethod + def _validate(names): + """Reject anything that is not a list of non-empty names.""" + if names is None or not isinstance(names, list): + raise ModuleBayTypeError(f"module_bay_types must be a list of names, got {names!r}") + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ModuleBayTypeError(f"module_bay_types entries must be non-empty names, got {name!r}") + + def _id_for(self, manufacturer, name): + entry = self._lookup(manufacturer, name) + cache_key = (manufacturer_slug(entry["manufacturer"]), entry["name"]) + if cache_key not in self._ids: + self._ids[cache_key] = self._netbox_id(entry) + return self._ids[cache_key] + + def _lookup(self, manufacturer, name): + """Find the catalog entry for *name*, owner scope first, then Generic.""" + entries = self._load_catalog() + for scope in (manufacturer, manufacturer_slug(FALLBACK_MANUFACTURER)): + entry = entries.get((scope, name)) + if entry is not None: + return entry + raise ModuleBayTypeError( + f"Module bay type {name!r} is not in the catalog for manufacturer " + f"{manufacturer!r} or {FALLBACK_MANUFACTURER!r}" + ) + + def _load_catalog(self): + """Read every catalog file once, indexed by (manufacturer, name). + + A failure is cached too: it is terminal for the run, and re-walking the tree for + every later lookup only repeats the same error more slowly. + """ + if self._entries is not None: + return self._entries + if self._load_error is not None: + raise self._load_error + try: + self._entries = self._read_catalog() + except ModuleBayTypeError as exc: + self._load_error = exc + raise + return self._entries + + def _read_catalog(self): + """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + entries = {} + for root, _dirs, files in os.walk(self._catalog_dir): + for filename in sorted(files): + if not filename.endswith((".yaml", ".yml")): + continue + path = os.path.join(root, filename) + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + except (OSError, yaml.YAMLError) as exc: + raise ModuleBayTypeError(f"Module bay type catalog file {path!r} could not be read: {exc}") from exc + if not isinstance(data, dict): + continue + # Reject a half-written entry here; the readers index on name and dereference slug. + invalid = [ + field + for field in ("name", "slug", "manufacturer") + if not isinstance(data.get(field), str) or not data[field].strip() + ] + if invalid: + raise ModuleBayTypeError( + f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" + ) + key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) + if key in entries: + raise ModuleBayTypeError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + entries[key] = data + return entries + + @staticmethod + def _request(action, description): + """Run one NetBox request, reporting a rejection as a catalog error. + + Callers resolve one definition at a time and recover from ModuleBayTypeError. A + raw RequestError or a dropped connection escapes that recovery and ends the run, + which can leave a parent half created and every later definition unprocessed. + """ + try: + return action() + except pynetbox.RequestError as exc: + raise ModuleBayTypeError(f"NetBox rejected {description}: {exc}") from exc + except requests.exceptions.RequestException as exc: + raise ModuleBayTypeError(f"NetBox could not be reached for {description}: {exc}") from exc + + def _netbox_id(self, entry): + """Return the id of the NetBox object for *entry*, creating it if absent.""" + manufacturer_id = self._manufacturer_id(entry["manufacturer"]) + existing = self._request( + lambda: list( + self._netbox.dcim.module_bay_types.filter(manufacturer_id=manufacturer_id, name=entry["name"]) + ), + f"the lookup of module bay type {entry['name']!r}", + ) + for record in existing: + if record.slug != entry["slug"]: + raise ModuleBayTypeError( + f"NetBox already has module bay type {entry['name']!r} for " + f"{entry['manufacturer']!r} with slug {record.slug!r}, but the catalog " + f"says {entry['slug']!r}. Resolve the conflict in NetBox; this import " + f"will not rename it." + ) + return record.id + + payload = {"name": entry["name"], "slug": entry["slug"], "manufacturer": manufacturer_id} + if entry.get("description"): + payload["description"] = entry["description"] + created = self._request( + lambda: self._netbox.dcim.module_bay_types.create(payload), + f"creating module bay type {entry['name']!r}", + ) + self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") + return created.id + + def _manufacturer_id(self, name): + """Return the id of *name*, creating the manufacturer when the catalog needs it. + + A Generic-scoped class is reachable from any vendor, so an import filtered to one + manufacturer still has to create the manufacturer that owns the class. + """ + if name in self._manufacturer_ids: + return self._manufacturer_ids[name] + found = self._request( + lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), + f"the lookup of manufacturer {name!r}", + ) + if found: + self._manufacturer_ids[name] = found[0].id + else: + created = self._request( + lambda: self._netbox.dcim.manufacturers.create({"name": name, "slug": manufacturer_slug(name)}), + f"creating manufacturer {name!r}", + ) + self._handle.verbose_log(f"Manufacturer Created: {name} - {created.id}") + self._manufacturer_ids[name] = created.id + return self._manufacturer_ids[name] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index c6535506..5ba5b2ca 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -7,7 +7,7 @@ import warnings from typing import Any, Sequence -from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES +from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS # Row order sets the component key order of the serialized YAML. COMPONENT_ENDPOINT_NAMES = [component.endpoint for component in COMPONENT_TYPES] @@ -125,6 +125,26 @@ def _serialize_component(record: Any, fields: Sequence[str]) -> dict: return result +def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: + """Return the catalog name of each related object, which is how YAML names them. + + The names come off the record the query returned, not from the import-side catalog: + an export run resolves nothing, so it has no id-to-name mapping of its own. + + An empty relation writes no key. Emitting an empty list instead would add the key to + every bay in the library and make _repo_supersedes report every existing definition as + differing, and it tells a fresh import nothing that omitting it does not. + """ + result = {} + for relation in relations: + names = sorted( + name for name in (getattr(item, "name", None) for item in getattr(record, relation, None) or []) if name + ) + if names: + result[relation] = names + return result + + def _serialize_front_port(record: Any) -> dict: """Serialize a front port template, including rear_port mapping.""" result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) @@ -161,12 +181,15 @@ def _serialize_front_port(record: Any) -> dict: def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" + component = BY_ENDPOINT[endpoint_name] out = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): if endpoint_name == "front_port_templates": - out.append(_serialize_front_port(record)) + serialized = _serialize_front_port(record) else: - out.append(_serialize_component(record, BY_ENDPOINT[endpoint_name].fields)) + serialized = _serialize_component(record, component.fields) + serialized.update(_serialize_relations(record, component.relations)) + out.append(serialized) return out @@ -236,6 +259,7 @@ def serialize_module_type(nb_record: Any, components_by_mt_id: dict) -> dict: if _should_include(field, val): result[field] = val + result.update(_serialize_relations(nb_record, MODULE_TYPE_RELATIONS)) _add_components(result, nb_record.id, components_by_mt_id) return result diff --git a/core/netbox_api.py b/core/netbox_api.py index 220efae1..979e704d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -5,7 +5,6 @@ from functools import lru_cache import hashlib import json -import re import tempfile import time import pynetbox @@ -24,7 +23,9 @@ LINK_POWER_PORT, LINK_REAR_PORTS, MODULE_TYPE_COMPONENTS, + MODULE_TYPE_RELATIONS, ) +from core.compat import parse_netbox_version, supports_module_bay_types from core.formatting import log_property_diffs from core.errors import FatalError, UnknownError from core.graphql_client import GraphQLError, NetBoxGraphQLClient @@ -50,6 +51,35 @@ def __init__(self, ignore_ssl_errors: bool, cause=None): ) +# Oldest NetBox this importer is tested against; see the CI matrix and the README. +MINIMUM_NETBOX_VERSION = (4, 3) + + +def _relation_identities_differ(catalog, manufacturer, declared, related, *, names_differ): + """Return True when the assigned objects are not the ones the references resolve to. + + Comparing names alone reports equality when NetBox holds a same-named class from + another manufacturer's scope, so the identity is compared where both sides can supply + one. Where they cannot, the name comparison the caller already made stands. + """ + from core.module_bay_types import ModuleBayTypeError, manufacturer_slug + + if not catalog or not manufacturer: + return names_differ + current = set() + for item in related: + slug = getattr(item, "slug", None) + owner = getattr(getattr(item, "manufacturer", None), "slug", None) + if not slug or not owner: + return names_differ + current.add((manufacturer_slug(owner), slug)) + try: + return catalog.identities_for(manufacturer, declared) != frozenset(current) + except ModuleBayTypeError: + # A matching name must not make an unresolvable reference look applied. + return True + + class NetBoxError(FatalError): """A fatal error reported by the NetBox integration.""" @@ -464,6 +494,7 @@ def __init__(self, config, handle): self.ignore_ssl = config.ignore_ssl_errors self.modules = False self.new_filters = False + self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts @@ -495,6 +526,7 @@ def __init__(self, config, handle): self.ignore_ssl, handle=self.handle, page_size=config.graphql_page_size, + supports_module_bay_types=self.module_bay_types, ) try: self.existing_manufacturers = self.get_manufacturers() @@ -509,6 +541,7 @@ def __init__(self, config, handle): self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, + module_bay_types_supported=self.module_bay_types, repo_path=config.repo_path, max_threads=config.preload_threads, ) @@ -601,8 +634,15 @@ def verify_compatibility(self): msg += f"\nResponse body (may be from an intermediate proxy):\n{body}" msg += f"\nHint: Verify that {self.url} is reachable and not blocked by a proxy." raise NetBoxError(msg) from e - _raw = [int(re.sub(r"\D.*", "", x.strip()) or "0") for x in nb_version.split(".")] - version_split = (_raw + [0, 0])[:2] # pad to (major, minor) to guard against single-component strings + version_split = parse_netbox_version(nb_version) + + # Below the floor the run dies later naming a schema field, not the real cause. + if tuple(version_split) < MINIMUM_NETBOX_VERSION: + minimum = ".".join(str(part) for part in MINIMUM_NETBOX_VERSION) + raise NetBoxError( + f"NetBox {nb_version} is not supported: this importer requires NetBox {minimum} or later. " + f"Older releases fail part way through with a GraphQL schema error rather than here." + ) # Later than 3.2 # Might want to check for the module-types entry as well? @@ -622,6 +662,10 @@ def verify_compatibility(self): self.m2m_front_ports = True self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + if supports_module_bay_types(nb_version): + self.module_bay_types = True + self.handle.log(f"Netbox version {self.netbox.version} found. Module bay types are supported.") + def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" return self.graphql.get_manufacturers() @@ -1162,7 +1206,13 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima continue if yaml_key == "module-bays" and not self.modules: continue - self.device_types.create_components(yaml_key, device_type[yaml_key], dt_id, context=src_file) + self.device_types.create_components( + yaml_key, + device_type[yaml_key], + dt_id, + context=src_file, + manufacturer=device_type.get("manufacturer"), + ) if component_errors: # The type exists but not all of its components do. self.outcomes.record( @@ -1522,6 +1572,8 @@ def filter_actionable_module_types(self, module_types, all_module_types, only_ne if not values_equal(module_type[f], nb_val): changed_fields_info.append((f, nb_val, module_type[f])) + changed_fields_info += self._type_relation_changes(module_type, existing_module) + component_changes = detector._compare_components(module_type, existing_module.id, parent_type="module") if changed_fields_info or component_changes: @@ -1574,6 +1626,57 @@ def _fetch_module_type_existing_images(self): ) return module_type_existing_images + def _type_relation_changes(self, module_type, existing_module): + """Return (field, current, wanted) for each of the type's own relations that differs. + + A relation is a list, so the scalar comparison loop never sees it. A field the + query did not return is skipped rather than read as empty, which would otherwise + report a change on every run. + """ + changes: list[tuple[str, list[str], list[str]]] = [] + if not self.module_bay_types: + return changes + for field in MODULE_TYPE_RELATIONS: + if field not in module_type: + continue + netbox_value = getattr(existing_module, field, _MISSING) + if netbox_value is _MISSING: + continue + related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] + declared = module_type[field] + if not isinstance(declared, list) or any(not isinstance(x, str) or not x for x in declared): + # Malformed or bare key: leave the relation unmanaged rather than clear it. + continue + wanted = sorted(set(declared)) + current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) + if _relation_identities_differ( + self.device_types.module_bay_types, + self.device_types._manufacturer_slug(module_type.get("manufacturer")), + declared, + related, + names_differ=wanted != current, + ): + changes.append((field, current, wanted)) + return changes + + def _resolve_type_relations(self, payload): + """Return *payload* with its own relation fields resolved from names to ids. + + Raises ModuleBayTypeError if a name does not resolve, so the caller can report the + module type rather than write it with the restriction silently dropped. + """ + names = {field: payload[field] for field in MODULE_TYPE_RELATIONS if field in payload} + if not names: + return payload + if not self.module_bay_types: + # The server predates ModuleBayType; sending the field would be rejected. + return {k: v for k, v in payload.items() if k not in names} + manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) + resolved = { + field: self.device_types.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + } + return {**payload, **resolved} + def _try_update_module_type(self, curr_mt, module_type_res, src_file): """Apply pending field updates to an existing module type in NetBox. @@ -1590,6 +1693,18 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + if self.module_bay_types: + from core.module_bay_types import ModuleBayTypeError + + for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): + try: + updates[field] = self.device_types.module_bay_types.ids_for( + self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted + ) + except ModuleBayTypeError as exc: + # Not fatal to the run, but not a success either; the caller records it. + self.handle.log(f"Skipped {field} on {curr_mt.get('model')}: {exc} (Context: {src_file})") + return False, False if not updates: return True, False try: @@ -1622,7 +1737,12 @@ def _create_module_type_components(self, curr_mt, module_type_id, src_file): yaml_key = component.yaml_key if yaml_key in curr_mt: self.device_types.create_components( - yaml_key, curr_mt[yaml_key], module_type_id, parent_type="module", context=src_file + yaml_key, + curr_mt[yaml_key], + module_type_id, + parent_type="module", + context=src_file, + manufacturer=curr_mt.get("manufacturer"), ) if component_errors: # The module type exists but not all of its components do. @@ -1776,7 +1896,20 @@ def _process_single_module_type( ) else: try: - module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, curr_mt) + from core.module_bay_types import ModuleBayTypeError + + try: + payload = self._resolve_type_relations(curr_mt) + except ModuleBayTypeError as exc: + self.handle.log(f"Error creating Module Type: {exc} (Context: {src_file})") + self._record_failure( + EntityKind.MODULE_TYPE, + self._yaml_identity(curr_mt), + str(exc), + src_file, + ) + return False + module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, payload) self.counter["module_added"] += 1 is_new = True manufacturer_slug = curr_mt["manufacturer"]["slug"] @@ -2192,6 +2325,7 @@ def __init__( graphql, repo_path, m2m_front_ports=False, + module_bay_types_supported=False, max_threads=8, ): """Initialize empty DeviceTypes cache structures; no data is fetched at construction time. @@ -2206,6 +2340,7 @@ def __init__( ignore_ssl (bool): Whether SSL certificate verification is disabled. new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. + module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. m2m_front_ports (bool): Whether NetBox uses the 4.5+ M2M port mapping model. max_threads (int): Maximum number of concurrent threads for component preloading. @@ -2218,6 +2353,7 @@ def __init__( self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports + self.module_bay_types_supported = module_bay_types_supported self.max_threads = max_threads self.components = ComponentCache( netbox, @@ -2228,6 +2364,7 @@ def __init__( wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None + self._module_bay_types = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2486,6 +2623,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, if change.component_name in existing: comp = existing[change.component_name] update_data = {"id": comp.id} + unresolved = False for pc in change.property_changes: if comp_type == "front-ports" and pc.property_name == "_mappings": yaml_front_port = next( @@ -2501,8 +2639,21 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, parent_type, ) continue + if pc.property_name in component.relations: + # The comparison works in names; NetBox wants ids. + from core.module_bay_types import ModuleBayTypeError + + try: + update_data[pc.property_name] = self.module_bay_types.ids_for( + self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value + ) + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{change.component_name}': {exc}") + unresolved = True + break + continue update_data[pc.property_name] = pc.new_value - if len(update_data) > 1: # has fields beyond just "id" + if not unresolved and len(update_data) > 1: # has fields beyond just "id" updates.append(update_data) success_count = 0 @@ -2547,7 +2698,13 @@ def _apply_additions_for_type(self, comp_type, changes, yaml_data, device_type_i if not components_to_add: return - self.create_components(comp_type, components_to_add, device_type_id, parent_type=parent_type) + self.create_components( + comp_type, + components_to_add, + device_type_id, + parent_type=parent_type, + manufacturer=yaml_data.get("manufacturer"), + ) def update_components(self, yaml_data, device_type_id, component_changes, parent_type="device"): """Update existing components and add new components based on detected changes. @@ -2837,7 +2994,66 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): f"Connection error bridging interfaces after {_MAX_RETRIES} retries: {e} (Context: {context})" ) - def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None): + @property + def module_bay_types(self): + """The module-bay-type catalog, built once per run from the library checkout.""" + if self._module_bay_types is None: + from core.module_bay_types import ModuleBayTypeCatalog + + self._module_bay_types = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_types + + @staticmethod + def _manufacturer_slug(manufacturer): + """Return the manufacturer slug, whatever shape the caller happens to hold. + + ``core.repo`` rewrites every YAML ``manufacturer`` to ``{"slug": ...}`` before the + importer sees it, so that is the usual shape; a plain name is slugified here. + """ + from core.module_bay_types import manufacturer_slug + + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or manufacturer_slug(manufacturer.get("name")) + return manufacturer_slug(getattr(manufacturer, "name", manufacturer)) + + def _resolve_relations(self, component, items, manufacturer): + """Turn each relation field's names into NetBox ids, dropping items that cannot resolve. + + A component whose restriction cannot be resolved is skipped and logged rather than + created without it: creating the bay anyway would silently discard the restriction. + """ + if not component.relations: + return items + if not self.module_bay_types_supported: + # The server predates ModuleBayType; drop the field rather than have it rejected. + return [{k: v for k, v in item.items() if k not in component.relations} for item in items] + from core.module_bay_types import ModuleBayTypeError + + manufacturer = self._manufacturer_slug(manufacturer) + resolved = [] + for item in items: + names = {field: item[field] for field in component.relations if field in item} + if not names: + resolved.append(item) + continue + if not manufacturer: + # No scope to resolve in, and NetBox wants ids: sending the names would fail. + self._log_component_error( + f"Skipped {component.label} '{item.get('name', 'Unknown')}': no manufacturer to " + f"resolve {', '.join(sorted(names))} in" + ) + continue + try: + replacements = { + field: self.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + } + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") + continue + resolved.append({**item, **replacements}) + return resolved + + def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None, manufacturer=None): """Create component templates of one kind for one parent, skipping those that exist. The registry row for *yaml_key* supplies the endpoint, the cache name, the log label @@ -2850,6 +3066,8 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co parent_id (int): NetBox ID of the parent device or module type. parent_type (str): ``"device"`` or ``"module"``. context (str | None): Optional context string appended to log messages. + manufacturer (dict | str | None): Owning manufacturer, used to resolve any + relation fields the registry row declares. """ component = BY_YAML_KEY[yaml_key] label = component.create_label(parent_type) @@ -2869,7 +3087,7 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co self._create_generic( component, - items, + self._resolve_relations(component, items, manufacturer), parent_id, parent_type=parent_type, post_process=post_process, diff --git a/tests/conftest.py b/tests/conftest.py index 364bd128..a1df5ff4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,14 @@ def mock_git_repo(request): @pytest.fixture def mock_pynetbox(): - """Mock pynetbox to prevent API calls.""" + """Mock pynetbox to prevent API calls. + + Defaults the reported version to the oldest supported release, so a test that does not + care about version gating still constructs a NetBox; the importer refuses anything + older. Individual tests override it to exercise a specific release. + """ with patch("core.netbox_api.pynetbox") as mock_nb: + mock_nb.api.return_value.version = "4.3" yield mock_nb diff --git a/tests/helpers.py b/tests/helpers.py index 9bbad389..c6f0f381 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,13 @@ """Shared test utilities for the NetBox device-type importer test suite.""" +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlparse + +import pynetbox def paginate_dispatch(data_dict): @@ -49,3 +56,195 @@ def recording_handle(): console = RecordingConsole() handle.set_console(console) return handle, console + + +class FakeNetBox: + """A local HTTP server answering the slice of the NetBox REST API the importer uses. + + Collections are addressed by their URL segment with dashes turned into underscores, + so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates and PATCH + bulk-updates behave the way pynetbox expects, so a real client drives it and the + serialising, filtering and paginating under test are the production ones. + + ``/graphql/`` answers every query with an empty list, which is enough to bring a real + :class:`~core.netbox_api.NetBox` up; the REST half is where the assertions live. + + A test using this must carry the ``real_http`` marker. Without it the suite patches + ``requests.Session`` and no request ever leaves the client. + """ + + _IGNORED_FILTERS = ("limit", "offset", "brief", "exclude") + + # GraphQL collections are named "_list"; the client asks for one per query. + _LIST_KEY = re.compile(r"\b(\w+_list)\b") + + def __init__(self, errors=None, netbox_version="4.7.0", **collections): + """Start the server with *collections* seeded, keyed by collection name. + + *errors* maps a collection name to an HTTP status the server answers it with, so a + test can drive a rejection the importer has to survive. *netbox_version* is what + ``/api/status/`` reports, which is how the export side decides what it may select. + """ + self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} + self.errors = dict(errors or {}) + self.netbox_version = netbox_version + self.requests = [] + self._server = HTTPServer(("127.0.0.1", 0), self._handler()) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def url(self): + """Return the base URL a client should talk to.""" + return f"http://127.0.0.1:{self._server.server_port}" + + def api(self, token="test-token"): + """Return a real pynetbox client pointed at this server.""" + return pynetbox.api(self.url, token=token) + + def close(self): + """Stop serving and release the listening socket.""" + self._server.shutdown() + self._server.server_close() + + def collection(self, name): + """Return the stored records for a collection, creating it empty when unseen.""" + return self.collections.setdefault(name, []) + + def sent(self, verb, name): + """Return the payload of each *verb* request made to collection *name*. + + A bulk create or update sends a list; its entries are returned individually, so a + caller asserting on what was written does not have to care which shape was used. + """ + out = [] + for method, collection, payload in self.requests: + if method != verb or collection != name: + continue + out.extend(payload) if isinstance(payload, list) else out.append(payload) + return out + + def matches(self, name, query): + """Filter a collection the way the NetBox REST API filters it.""" + out = [] + for record in self.collection(name): + ok = True + for key, values in query.items(): + if key in self._IGNORED_FILTERS: + continue + field = key[:-3] if key.endswith("_id") else key + actual = record.get(field) + if isinstance(actual, dict): + actual = actual.get("id") + if str(actual) not in values: + ok = False + break + if ok: + out.append(record) + return out + + def _create(self, name, payload): + """Store the new record(s) and echo them back the way NetBox does. + + A list payload is a bulk create, which is how the importer adds components, and it + answers with a list. + """ + if isinstance(payload, list): + return [self._create_one(name, item) for item in payload] + return self._create_one(name, payload) + + def _create_one(self, name, payload): + """Store one new record and return it as NetBox would echo it back.""" + collection = self.collection(name) + record = {"id": 1000 + len(collection), **payload} + # NetBox echoes a foreign key back as a nested object, not the id it was given. + if isinstance(record.get("manufacturer"), int): + owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) + record["manufacturer"] = {"id": record["manufacturer"], "name": owner.get("name")} + collection.append(record) + return record + + def _patch(self, name, payload): + """Merge a bulk update into the stored records and return the updated ones.""" + updated = [] + for entry in payload if isinstance(payload, list) else [payload]: + record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) + if record is None: + continue + record.update(entry) + updated.append(record) + return updated + + def _handler(self): + """Build the request handler class bound to this server's state.""" + state = self + + class Handler(BaseHTTPRequestHandler): + def _collection_name(self): + return urlparse(self.path).path.rstrip("/").rsplit("/", 1)[-1].replace("-", "_") + + def _body(self): + return json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + + def _reply(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _refused(self, name): + """Answer with the configured status for *name*, if the test set one.""" + status = state.errors.get(name) + if status is None: + return False + self._reply(status, {"detail": f"{name} refused with {status}"}) + return True + + def do_GET(self): + parsed = urlparse(self.path) + name = self._collection_name() + state.requests.append(("GET", name, parsed.query)) + if name == "status": + self._reply(200, {"netbox-version": state.netbox_version}) + return + if self._refused(name): + return + results = state.matches(name, parse_qs(parsed.query)) + self._reply(200, {"count": len(results), "next": None, "previous": None, "results": results}) + + def do_POST(self): + name = self._collection_name() + payload = self._body() + if name == "graphql": + state.requests.append(("POST", name, payload)) + key = state._LIST_KEY.search(payload.get("query", "")) + self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) + return + state.requests.append(("POST", name, payload)) + if self._refused(name): + return + self._reply(201, state._create(name, payload)) + + def do_PATCH(self): + name = self._collection_name() + payload = self._body() + state.requests.append(("PATCH", name, payload)) + if self._refused(name): + return + self._reply(200, state._patch(name, payload)) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + return Handler + + +def write_module_bay_type(root, manufacturer, slug, name, description=None): + """Write one module-bay-type catalog file where the devicetype-library puts it.""" + directory = root / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + body = f"name: {name}\nslug: {slug}\nmanufacturer: {manufacturer}\n" + if description: + body += f"description: {description}\n" + (directory / f"{slug}.yaml").write_text(body, encoding="utf-8") diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 2b19c784..7e536f53 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -131,7 +131,14 @@ def test_the_query_selects_exactly_these_fields(self): _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], - "module_bay_templates": ["id", "name", "position", "label", "description"], + "module_bay_templates": [ + "id", + "name", + "position", + "label", + "description", + "module_bay_types { id name slug manufacturer { slug } }", + ], } def test_only_the_front_port_query_selects_a_nested_block(self): @@ -146,13 +153,17 @@ class TestDerivedComparisonAndExport: @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_every_compared_property_is_fetched(self, component): """A property compared but never queried reads as missing and is skipped in silence.""" - queried = set(component.graphql_fields) | {"_mappings"} + # A relation is selected as a nested block, so take the field name each selection + # opens with. Adding component.relations here instead would make the assertion + # hold even if the query stopped selecting them. + queried = {selection.split(None, 1)[0] for selection in component.graphql_fields} | {"_mappings"} assert set(component.compare_properties) <= queried @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_the_export_writes_every_scalar_the_query_reads(self, component): """Export fields are the query's scalars: an unexported scalar drops out of a round trip.""" - scalars = [name for name in component.graphql_fields if name != "id" and name not in component.graphql_extra] + non_scalar = set(component.graphql_extra) | set(component.graphql_relation_fields) + scalars = [name for name in component.graphql_fields if name != "id" and name not in non_scalar] assert list(component.fields) == scalars def test_front_ports_compare_the_mapping_the_query_selects(self): diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 18c06bf2..d9eebe29 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -239,6 +239,26 @@ def test_yaml_equal_normalizes_component_order_and_numbers(self): class TestRepoSupersedes: """Tests for _repo_supersedes / _is_subset (asymmetric containment).""" + def test_an_empty_relation_does_not_make_every_definition_differ(self): + """The serializer omits an empty relation, and this is why it has to. + + _is_subset requires every NetBox leaf to be present in the repo YAML, and + _normalize_for_compare does not drop empty lists. A serialized + "module_bay_types: []" would therefore be absent from every library definition + and re-export the whole library on a NetBox 4.7 server. + """ + from core.nb_serializer import _serialize_relations + + bay = type("Bay", (), {"name": "FPC 0", "module_bay_types": []})() + assert _serialize_relations(bay, ("module_bay_types",)) == {} + + repo = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} + as_serialized = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} + with_empty_key = {"model": "MX304", "module-bays": [{"name": "FPC 0", "module_bay_types": []}]} + + assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" + assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -853,11 +873,17 @@ def test_fetch_vendor_components_groups_device_and_module_records(self, tmp_path def _side_effect(endpoint_name, manufacturer_slug=None): return [dt_rec, mt_rec] if endpoint_name == "interface_templates" else [] + # A worker must fetch through its own clone, so only the clone answers. + worker = MagicMock() + worker.get_component_templates.side_effect = _side_effect mock_client = MagicMock() - mock_client.get_component_templates.side_effect = _side_effect - with patch("core.export.NetBoxGraphQLClient", return_value=mock_client): - dt_result, mt_result = exporter._fetch_vendor_components("nokia") + mock_client.get_component_templates.side_effect = AssertionError("worker must use clone()") + mock_client.clone.return_value = worker + exporter.graphql = mock_client + + dt_result, mt_result = exporter._fetch_vendor_components("nokia") + assert mock_client.clone.called assert dt_result[11]["interface_templates"] == [dt_rec] assert mt_result[22]["interface_templates"] == [mt_rec] diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 6c5b22b5..b72ddae6 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -118,22 +118,115 @@ def test_logging_dependency_has_a_public_read_only_handle(self): client.handle = LogHandler(True) def test_clone_uses_an_independent_session_with_the_same_settings(self): + """Every stored setting is compared, so a new one cannot be dropped unnoticed. + + clone() re-lists constructor arguments by hand. Listing the settings here by hand + too let supports_module_bay_types default back to False in every cloned worker. + """ handle = MagicMock() sessions = [MagicMock(), MagicMock()] with patch("core.graphql_client.requests.Session", side_effect=sessions): - client = NetBoxGraphQLClient("http://netbox.local", "token", True, handle, 250) + client = NetBoxGraphQLClient( + "http://netbox.local", "token", True, handle, 250, supports_module_bay_types=True + ) clone = client.clone() assert clone is not client assert clone._session is sessions[1] - assert (clone.url, clone.token, clone.ignore_ssl, clone.handle, clone.DEFAULT_PAGE_SIZE) == ( - client.url, - client.token, - client.ignore_ssl, - client.handle, - client.DEFAULT_PAGE_SIZE, - ) + settings = lambda c: {k: v for k, v in vars(c).items() if k != "_session"} # noqa: E731 + assert settings(clone) == settings(client) + + @pytest.mark.real_http + def test_the_cloned_session_is_configured_not_merely_new(self): + """A bare requests.Session() would be independent and completely unauthenticated. + + Marked real_http only to get real Session objects; nothing here sends a request. + """ + client = NetBoxGraphQLClient("http://netbox.local", "nbt_key.secret", ignore_ssl=True) + + clone = client.clone() + + assert clone._session is not client._session + assert clone._session.headers["Authorization"] == "Bearer nbt_key.secret" + assert clone._session.headers["Content-Type"] == "application/json" + assert clone._session.verify is False + + @pytest.mark.real_http + def test_a_v1_token_clone_keeps_the_legacy_auth_scheme(self): + client = NetBoxGraphQLClient("http://netbox.local", "0123456789abcdef") + + assert client.clone()._session.headers["Authorization"] == "Token 0123456789abcdef" + + @pytest.mark.real_http + def test_an_unreachable_server_fails_the_probe_as_a_graphql_error(self): + """It is the export's first request, so a raw transport error escapes as a traceback.""" + from core.graphql_client import GraphQLError + from helpers import FakeNetBox + + server = FakeNetBox() + url = server.url + server.close() # nothing is listening on that port any more + + with pytest.raises(GraphQLError): + NetBoxGraphQLClient(url, "tok").detect_module_bay_type_support() + + @pytest.mark.real_http + def test_a_non_json_status_body_fails_the_probe_as_a_graphql_error(self): + """A proxy error page answers 200 with HTML; json() then raises ValueError.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"gateway" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): + """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" + from helpers import FakeNetBox + + for version, expected in (("4.7.0", True), ("4.7.0-beta2", True), ("4.6.9", False), ("4.3.7", False)): + server = FakeNetBox(netbox_version=version) + try: + client = NetBoxGraphQLClient(server.url, "tok") + assert client.detect_module_bay_type_support() is expected, version + assert client.supports_module_bay_types is expected + finally: + server.close() + + def test_clone_carries_a_setting_it_does_not_name(self): + """clone() must not enumerate settings: the one it forgets is the one that breaks. + + supports_module_bay_types was lost exactly that way, and adding it to the argument + list only fixes the setting that was already missed. + """ + with patch("core.graphql_client.requests.Session"): + client = NetBoxGraphQLClient("http://netbox.local", "token") + client.added_after_this_test_was_written = "carried" + clone = client.clone() + + assert clone.added_after_this_test_was_written == "carried" def test_init_stores_config(self): from core.graphql_client import NetBoxGraphQLClient @@ -1025,6 +1118,40 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): + """The prefetch runs on clones, not on the client it was cloned from. + + A clone that drops the relation reads every bay without it, and the comparison + then skips a field it never received. + """ + from core.graphql_client import NetBoxGraphQLClient + + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=True) + mock_post.side_effect = _make_paged_responses({"module_bay_template_list": []}, "module_bay_template_list") + + client.clone().get_component_templates("module_bay_templates") + + queries = [call.kwargs["json"]["query"] for call in mock_post.call_args_list] + assert queries and all("module_bay_types" in q for q in queries) + + def test_the_module_type_query_selects_the_relation_only_when_supported(self, mock_post): + """The selection has to track the server, in both directions. + + An unselected field reads as missing and is skipped by the comparison, and + selecting it on a server below 4.7 fails the whole query. + """ + from core.graphql_client import NetBoxGraphQLClient + + def _query_for(supported): + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=supported) + mock_post.reset_mock() + mock_post.side_effect = _make_paged_responses({"module_type_list": []}, "module_type_list") + client.get_module_types() + return mock_post.call_args_list[0].kwargs["json"]["query"] + + assert "module_bay_types" in _query_for(True) + assert "module_bay_types" not in _query_for(False) + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py new file mode 100644 index 00000000..ed2a27e1 --- /dev/null +++ b/tests/test_module_bay_type_sync.py @@ -0,0 +1,577 @@ +"""Tests for keeping module bay type assignments in sync with NetBox. + +Resolving a reference name to an id is :mod:`core.module_bay_types` and is tested there. +This file covers the wiring around it: the relation a module type carries itself, the +component update path, and the gate that keeps the field off a server that predates it. + +Both sides are real. The catalog is written to disk and read by a real +``ModuleBayTypeCatalog``; a local HTTP server answers a real ``pynetbox`` client and the +real GraphQL client, so the payloads asserted on here are the ones that would go over the +wire. Only the version handshake is stood in for, to fix the server release under test. +""" + +import pynetbox +import pytest + +from core.change_detector import ChangeType, ComponentChange, PropertyChange +from core.component_registry import BY_YAML_KEY +from core.graphql_client import NetBoxGraphQLClient +from core.module_bay_types import ModuleBayTypeCatalog +from core.netbox_api import DeviceTypes, NetBox +from core.outcomes import EntityKind, Outcome +from helpers import FakeNetBox, recording_handle, write_module_bay_type + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +JUNIPER = {"id": 1, "name": "Juniper", "slug": "juniper"} + + +def stub(**attrs): + """Return an object carrying exactly *attrs*, the way NetBox nests a related object.""" + return type("Stub", (), attrs)() + + +def created_id(server, slug): + """Return the id the server gave the module bay type it created for *slug*.""" + return next(record["id"] for record in server.collection("module_bay_types") if record["slug"] == slug) + + +def related(name, slug=None, manufacturer_slug=None): + """Return a module bay type as NetBox returns it nested inside a relation.""" + return stub(name=name, slug=slug, manufacturer=stub(slug=manufacturer_slug)) + + +class NetBoxRecord: + """An existing NetBox object as the importer holds it: an id, plus what was fetched. + + A field the query did not ask for is genuinely absent, which is the difference the + comparison turns on, so this carries only what a test hands it. + """ + + def __init__(self, record_id, **fields): + """Store the id and the fields this record is meant to have.""" + self.id = record_id + for name, value in fields.items(): + setattr(self, name, value) + + +@pytest.fixture +def catalog_root(tmp_path): + """Write the catalog these tests resolve against and return its root.""" + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot") + return root + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server seeded with the manufacturers the catalog needs.""" + fake = FakeNetBox(manufacturers=[JUNIPER]) + yield fake + fake.close() + + +@pytest.fixture +def make_device_types(server, catalog_root): + """Build a real DeviceTypes talking to the local server, with its cache already primed.""" + + def _make(module_bay_types_supported=True): + handle, console = recording_handle() + device_types = DeviceTypes( + server.api(), + handle, + {}, + False, + True, + graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), + repo_path=str(catalog_root), + module_bay_types_supported=module_bay_types_supported, + ) + device_types.components.ensure_ready() + return device_types, console + + return _make + + +@pytest.fixture +def netbox(make_config, mock_pynetbox, server, catalog_root): + """Build a real NetBox against the local server, reporting the release that has the feature.""" + mock_pynetbox.api.return_value.version = "4.7" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, console = recording_handle() + config = make_config(netbox_url=server.url, repo_path=str(catalog_root)) + nb = NetBox(config, handle) + assert nb.module_bay_types, "a 4.7 server supports module bay types; the rest of this file assumes it" + + # connect_api ran against the patched pynetbox; every call under test uses a real client. + api = server.api() + nb.netbox = api + nb.device_types.netbox = api + nb.device_types.components.netbox = api + nb.device_types.components.ensure_ready() + return nb, console + + +class TestModuleTypeOwnRelation: + """A module type says which classes it belongs to, and that has to stay in sync.""" + + def test_a_missing_class_is_reported_as_a_change(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", [], ["MX304-RE"])] + + def test_the_right_class_is_left_alone(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): + """Juniper owns MX304-RE. A Generic object of that name is not the one referenced.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "generic")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + + def test_names_are_compared_when_netbox_returned_no_identity(self, netbox): + """A record carrying only a name cannot answer the scope question; names still can.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_unresolvable_reference_is_reported_even_when_the_name_matches(self, netbox): + """A name that happens to match must not make an unresolvable reference look applied. + + Comparison cannot say which object the name means, so the write path has to try, + fail, and refuse. Reporting no change here skips that entirely. + """ + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["NO-SUCH-CLASS"]} + + assert nb._type_relation_changes(module_type, existing) == [ + ("module_bay_types", ["NO-SUCH-CLASS"], ["NO-SUCH-CLASS"]) + ] + + def test_a_matching_but_unresolvable_name_does_not_read_as_updated(self, netbox, server): + """The end of that path: the module type is refused, not patched with its scalars.""" + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + assert nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") == (False, False) + assert not server.sent("PATCH", "module_types") + + def test_a_field_the_query_did_not_return_is_skipped(self, netbox): + """Reading an absent field as empty would report a change on every run.""" + nb, _ = netbox + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, NetBoxRecord(7)) == [] + + def test_an_omitted_key_leaves_the_relation_unmanaged(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + + assert nb._type_relation_changes({"model": "JNP304-RE"}, existing) == [] + + def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox): + """A bare key parses as None; clearing on it would drop a restriction nobody removed.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": None} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_names_are_compared_when_the_definition_names_no_manufacturer(self, netbox): + """Without an owning manufacturer there is no scope, so the name comparison stands.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC", "mx304-lmic", "juniper")]) + module_type = {"model": "JNP304-RE", "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_older_server_reports_no_relation_changes(self, netbox): + """Below 4.7 the relation does not exist, so there is nothing to compare.""" + nb, _ = netbox + nb.module_bay_types = False + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + +class TestModuleTypeCreatePayload: + """What the create path sends for a module type's own relation.""" + + def test_names_are_replaced_by_ids(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + resolved = nb._resolve_type_relations(payload) + + created = server.sent("POST", "module_bay_types") + assert [p["slug"] for p in created] == ["mx304-re"] + assert resolved["module_bay_types"] == [created_id(server, "mx304-re")] + assert resolved["model"] == "JNP304-RE" + + def test_a_payload_without_the_relation_is_untouched(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + + assert nb._resolve_type_relations(payload) == payload + assert not server.sent("POST", "module_bay_types") + + def test_an_older_server_never_sees_the_field(self, netbox, server): + """Sending a field the server does not have would fail the whole create.""" + nb, _ = netbox + nb.module_bay_types = False + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._resolve_type_relations(payload) == {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + assert not server.sent("POST", "module_bay_types") + + +class TestModuleTypeUpdate: + """The update path patches the relation, and survives one it cannot resolve.""" + + def test_a_changed_relation_is_patched_as_ids(self, netbox, server): + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (True, True) + assert server.sent("PATCH", "module_types") == [{"id": 7, "module_bay_types": [created_id(server, "mx304-re")]}] + + def test_an_unresolvable_reference_is_logged_and_the_run_goes_on(self, netbox, server): + """One bad module type must not end the run, and must not be written without it.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False), "an unapplied relation must not read as success" + assert not server.sent("PATCH", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + def test_a_scalar_change_is_held_back_when_the_relation_cannot_resolve(self, netbox, server): + """Patching the description while dropping the restriction is a half-applied write.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False) + assert not server.sent("PATCH", "module_types"), "the scalar PATCH must not go out alone" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestModuleTypeCreateRefusal: + """A module type whose relation cannot be resolved is reported, not written without it.""" + + def test_an_unresolvable_reference_skips_the_module_type(self, netbox, server): + nb, console = netbox + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + created = nb._process_single_module_type(curr_mt, "juniper/jnp304-re.yaml", {}, {}, only_new=False) + + assert created is False + assert not server.sent("POST", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + # The registry is the only tally of failures; a path that merely logs is absent + # from the run summary and from the itemised report. + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [(r.kind, r.identity) for r in failed] == [(EntityKind.MODULE_TYPE, "juniper/JNP304-RE")] + assert "NO-SUCH-CLASS" in failed[0].reason + + +class TestModuleTypeUpdateOutcome: + """The run summary must show the module type whose relation could not be applied.""" + + def test_an_unresolvable_relation_is_recorded_as_a_failure(self, netbox, server): + """Driven through the parent operation, because that is where the outcome is recorded.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[], description="old" + ) + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + nb._process_single_module_type( + curr_mt, "juniper/jnp304-re.yaml", {"juniper": {"JNP304-RE": existing}}, {}, only_new=False + ) + + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [r.kind for r in failed] == [EntityKind.MODULE_TYPE], "the failure must reach the run summary" + assert not server.sent("PATCH", "module_types"), "nothing may be written for a type it could not apply" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestComponentCreatePayload: + """A module bay is created with the restriction its definition asked for, or not at all.""" + + def test_names_are_replaced_by_ids(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert resolved == [{"name": "FPC 0", "module_bay_types": [created_id(server, "mx304-lmic")]}] + + def test_an_item_without_the_field_passes_through(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == items + assert not server.sent("POST", "module_bay_types") + + def test_an_unresolvable_restriction_drops_the_bay_rather_than_relaxing_it(self, make_device_types, server): + """Creating the bay without its restriction would silently accept any module.""" + device_types, console = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [ + {"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}, + {"name": "FPC 1", "module_bay_types": ["MX304-LMIC"]}, + ] + + with device_types.collect_component_errors() as collected: + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert [item["name"] for item in resolved] == ["FPC 1"] + # Collected, not just printed: the parent's outcome reason is built from these. + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + assert any("FPC 0" in line for line in console.lines) + + def test_an_older_server_never_sees_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == [{"name": "FPC 0"}] + assert not server.sent("POST", "module_bay_types") + + def test_a_netbox_rejection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A 403 while resolving must not escape the per-component recovery. + + The callers recover from ModuleBayTypeError only, so a raw RequestError ends the + run with a traceback and leaves a partly created parent behind. + """ + device_types, _ = make_device_types() + server.errors["module_bay_types"] = 403 + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e and "403" in e] + + def test_a_lost_connection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A dropped connection is not a RequestError, so it escaped the same recovery.""" + device_types, _ = make_device_types() + server.close() # the port stops answering; resolution now hits a refused connection + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e] + + def test_a_component_kind_with_no_relations_is_untouched(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "xe-0/0/0", "type": "100gbase-x-qsfp28"}] + + assert device_types._resolve_relations(BY_YAML_KEY["interfaces"], items, {"slug": "juniper"}) == items + + def test_an_unknown_manufacturer_drops_the_bay_rather_than_sending_names(self, make_device_types): + """Without a scope the names cannot become ids, and NetBox rejects raw names.""" + device_types, console = make_device_types() + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [] + assert any("FPC 0" in line for line in console.lines) + + def test_an_unknown_manufacturer_still_passes_through_a_bay_with_no_restriction(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == items + + def test_an_older_server_strips_the_field_even_without_a_manufacturer(self, make_device_types): + """The manufacturer question must not decide whether an unsupported field is sent.""" + device_types, _ = make_device_types(module_bay_types_supported=False) + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [{"name": "FPC 0"}] + + +class TestComponentCreateWiring: + """Through create_components(), so the resolution is proved to be wired in, not just present.""" + + def test_a_created_bay_carries_resolved_ids(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert [p["name"] for p in posted] == ["FPC 0"] + assert posted[0]["module_bay_types"] == [created_id(server, "mx304-lmic")] + + def test_an_older_server_creates_the_bay_without_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert posted and "module_bay_types" not in posted[0] + + def test_an_unresolvable_bay_is_never_posted(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + with device_types.collect_component_errors() as collected: + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + assert not server.sent("POST", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e] + + +class TestComponentUpdatePayload: + """An existing bay whose restriction changed is patched with ids, not names.""" + + @staticmethod + def _change(name, new_value): + return ComponentChange( + component_type="module-bays", + component_name=name, + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange(property_name="module_bay_types", old_value=[], new_value=new_value)], + ) + + def test_a_changed_restriction_is_patched_as_ids(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + device_types._apply_updates_for_type( + "module-bays", [self._change("FPC 0", ["MX304-LMIC"])], {"manufacturer": {"slug": "juniper"}}, 3, "device" + ) + + assert server.sent("PATCH", "module_bay_templates") == [ + {"id": 55, "module_bay_types": [created_id(server, "mx304-lmic")]} + ] + + def test_a_rejected_patch_is_collected_rather_than_raised(self, make_device_types, server): + """The bay resolves; NetBox refuses the write. That must land in the entity's report.""" + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + server.errors["module_bay_templates"] = 400 + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["MX304-LMIC"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert [e for e in collected if "55" in e] + + def test_an_unresolvable_restriction_is_logged_and_nothing_is_patched(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["NO-SUCH-CLASS"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert not server.sent("PATCH", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + + +class TestCatalogWiring: + """The catalog is built once per run, from the library checkout the run is using.""" + + def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): + device_types, _ = make_device_types() + + catalog = device_types.module_bay_types + + assert isinstance(catalog, ModuleBayTypeCatalog) + assert device_types.module_bay_types is catalog + assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py new file mode 100644 index 00000000..bdf1581d --- /dev/null +++ b/tests/test_module_bay_types.py @@ -0,0 +1,250 @@ +"""Tests for module bay type reference resolution. + +Driven through the public interface (``ids_for`` / ``identities_for``) against catalog files +the tests write themselves and a real ``pynetbox`` client talking HTTP to a local server. +Nothing here is mocked: the client serialises, filters and paginates for real, which is +where the semantics that matter actually live. +""" + +import pytest + +from core.module_bay_types import ModuleBayTypeCatalog, ModuleBayTypeError +from helpers import FakeNetBox, write_module_bay_type + + +class Handle: + """Capture what the catalog logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +DEFAULT_MANUFACTURERS = ( + {"id": 1, "name": "Juniper", "slug": "juniper"}, + {"id": 2, "name": "Cisco", "slug": "cisco"}, + {"id": 3, "name": "Nokia", "slug": "nokia"}, +) + + +@pytest.fixture +def library(tmp_path): + """Write the catalog the resolution tests resolve against, and return its root. + + QSFP-DD is defined twice on purpose, by Juniper and by Generic, so the owner-scope + rule has two candidates to choose between. + """ + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot compatibility") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot compatibility") + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "Juniper MX304 QSFP-DD cage") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "QSFP-DD pluggable transceiver form factor") + return root + + +@pytest.fixture +def catalog(library): + """Build a catalog wired to a real pynetbox client and a local NetBox-shaped server.""" + servers = [] + + def _make(module_bay_types=(), manufacturers=None, root=None): + server = FakeNetBox( + manufacturers=DEFAULT_MANUFACTURERS if manufacturers is None else manufacturers, + module_bay_types=module_bay_types, + ) + servers.append(server) + return ModuleBayTypeCatalog(server.api(), str(root or library), Handle()), server + + yield _make + for server in servers: + server.close() + + +@pytest.mark.real_http +class TestResolution: + """Names resolve to ids, in the owning manufacturer's scope and then in Generic.""" + + def test_resolves_in_owner_manufacturer_scope(self, catalog): + cat, server = catalog() + ids = cat.ids_for("juniper", ["MX304-RE"]) + assert len(ids) == 1 + created = server.sent("POST", "module_bay_types") + assert created == [ + { + "name": "MX304-RE", + "slug": "mx304-re", + "manufacturer": 1, + "description": "Juniper MX304 routing-engine slot compatibility", + } + ] + + def test_falls_back_to_generic_for_another_manufacturer(self, catalog): + """A Cisco optic reaches the Generic form factor, and Generic is created on demand.""" + cat, server = catalog() + assert len(cat.ids_for("cisco", ["QSFP-DD"])) == 1 + made = server.sent("POST", "manufacturers") + assert made == [{"name": "Generic", "slug": "generic"}] + + def test_owner_scope_wins_over_generic(self, catalog): + """Juniper and Generic both define QSFP-DD, and a Juniper reference means Juniper's.""" + cat, server = catalog() + cat.ids_for("juniper", ["QSFP-DD"]) + created = server.sent("POST", "module_bay_types") + assert [p["manufacturer"] for p in created] == [1] + assert not server.sent("POST", "manufacturers") + + def test_existing_object_is_reused_not_recreated(self, catalog): + cat, server = catalog( + module_bay_types=[{"id": 77, "name": "MX304-RE", "slug": "mx304-re", "manufacturer": {"id": 1}}] + ) + assert cat.ids_for("juniper", ["MX304-RE"]) == [77] + assert not server.sent("POST", "module_bay_types") + + def test_repeated_resolution_is_cached(self, catalog): + cat, server = catalog() + first = cat.ids_for("juniper", ["MX304-RE"]) + before = len(server.requests) + + assert cat.ids_for("juniper", ["MX304-RE"]) == first + assert len(first) == 1, "an empty result would make the request count meaningless" + assert len(server.requests) == before + + def test_order_is_not_meaningful(self, catalog): + cat, _ = catalog() + a = cat.ids_for("juniper", ["MX304-RE", "MX304-LMIC"]) + b = cat.ids_for("juniper", ["MX304-LMIC", "MX304-RE"]) + assert len(a) == 2 + assert sorted(a) == sorted(b) + + +@pytest.mark.real_http +class TestRefusals: + """An unresolved name and a conflicting identity are reported, never papered over.""" + + def test_unresolved_name_raises_rather_than_dropping(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["NO-SUCH-CLASS"]) + assert "NO-SUCH-CLASS" in str(exc.value) + + def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): + """The catalog says mx304-re; NetBox holds mx304_re. Never silently rename.""" + cat, server = catalog( + module_bay_types=[{"id": 88, "name": "MX304-RE", "slug": "mx304_re", "manufacturer": {"id": 1}}] + ) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["MX304-RE"]) + assert "mx304_re" in str(exc.value) and "mx304-re" in str(exc.value) + assert not server.sent("POST", "module_bay_types") + + +@pytest.mark.real_http +class TestCatalogReading: + """The catalog is read from real files on disk, including the shapes that are rejected.""" + + def test_entry_without_a_description_is_created_without_one(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + cat, server = catalog(root=tmp_path) + cat.ids_for("generic", ["SFP"]) + created = server.sent("POST", "module_bay_types") + generic = next(m for m in server.collection("manufacturers") if m["slug"] == "generic") + assert created == [{"name": "SFP", "slug": "sfp", "manufacturer": generic["id"]}] + + def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + (tmp_path / "module-bay-types" / "Generic" / "README.md").write_text("not a catalog entry\n") + (tmp_path / "module-bay-types" / "Generic" / "blank.yaml").write_text("# only a comment\n") + cat, _ = catalog(root=tmp_path) + assert len(cat.ids_for("generic", ["SFP"])) == 1 + + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): + """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayTypeError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) + + def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog): + """A parser error escaping the boundary ends the run before it reports anything.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayTypeError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "sfp.yaml" in str(exc.value) + + def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catalog, monkeypatch): + """The load caches only on success, so a bad catalog re-walked the tree every time.""" + import core.module_bay_types as module + + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + + walks = [] + real_walk = module.os.walk + monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) + + for _ in range(3): + with pytest.raises(ModuleBayTypeError): + cat.identities_for("generic", ["SFP"]) + assert len(walks) == 1 + + def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): + """Two files claiming the same (manufacturer, name) would make a reference ambiguous.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("generic", ["SFP"]) + assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) + + +@pytest.mark.real_http +class TestMalformedReferences: + """A reference list that is not a list of names is refused, never read as empty.""" + + def test_a_bare_key_is_refused_rather_than_clearing(self, catalog): + """`module_bay_types:` with no value parses as None; treating it as [] would clear.""" + cat, server = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", None) + assert "list of names" in str(exc.value) + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_a_non_string_entry_is_refused(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [{"name": "MX304-RE"}]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", ["MX304-RE", 7]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [" "]) + + def test_an_empty_list_is_allowed_and_holds_no_classes(self, catalog): + """`module_bay_types: []` is an explicit instruction, not a malformed value.""" + cat, server = catalog() + assert cat.ids_for("juniper", []) == [] + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_duplicate_names_collapse(self, catalog): + """The relationship is a set, so a repeated name must not produce a repeated id.""" + cat, _ = catalog() + once = cat.ids_for("juniper", ["MX304-RE"]) + + assert len(once) == 1 + assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 31245d0b..a358c6ca 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -2190,7 +2190,7 @@ class TestExportDiffVendorFilterOverRealHTTP: """Same run against a local HTTP server, so the filter is asserted as it is serialized on the wire.""" @staticmethod - def _serve(): + def _serve(netbox_version="4.7.0"): """Serve empty GraphQL pages and record every decoded request body.""" import json import threading @@ -2199,16 +2199,22 @@ def _serve(): bodies = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers["Content-Length"]) - bodies.append(json.loads(self.rfile.read(length))) - payload = b'{"data": {}}' + def _reply(self, payload): self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) + def do_GET(self): + """Answer the version probe the export runs before its first query.""" + self._reply(json.dumps({"netbox-version": netbox_version}).encode()) + + def do_POST(self): + length = int(self.headers["Content-Length"]) + bodies.append(json.loads(self.rfile.read(length))) + self._reply(b'{"data": {}}') + def log_message(self, *args): """Silence the default stderr access log.""" @@ -2229,3 +2235,33 @@ def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypa assert set(filters) == set(_LIST_FIELDS) for field, variables in filters.items(): assert variables["manufacturer_slugs"] == ["cisco", "juniper"], field + + def _queries_for_version(self, nb_dt_import, monkeypatch, tmp_path, library_root, version): + """Run one whole export against a server reporting *version* and return its queries.""" + url, server, bodies = self._serve(version) + monkeypatch.setenv("NETBOX_URL", url) + try: + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, library_root, "Juniper") + finally: + server.shutdown() + server.server_close() + return [payload["query"] for payload in bodies] + + def test_a_47_server_is_asked_for_the_module_bay_type_relation( + self, nb_dt_import, monkeypatch, tmp_path, _real_library_root + ): + """Not selecting it exports every bay without its restriction, silently. + + Asserted on the module-type query, which this run always issues; the component + queries only run once there is something to export. + """ + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.7.0") + + module_types = [q for q in queries if "module_type_list(" in q] + assert module_types and all("module_bay_types" in q for q in module_types) + + def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + """Selecting a field the schema lacks fails the whole query.""" + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.6.9") + + assert not [q for q in queries if "module_bay_types" in q] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 9250234d..3d5edf70 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -677,3 +677,69 @@ def test_components_sorted_by_name(self): result = serialize_device_type(record, components) names = [i["name"] for i in result["interfaces"]] assert names == sorted(names) + + +class TestRelationSerialization: + """A module bay's restriction has to survive the trip back out to YAML.""" + + @staticmethod + def _bay(name, module_bay_types=None, **extra): + """Build a module bay template as the GraphQL query returns it.""" + return _dotdict( + name=name, + position=None, + label="", + description="", + module_bay_types=module_bay_types, + **extra, + ) + + def test_a_bay_exports_the_names_of_its_classes(self): + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = self._bay("FPC 0", [_dotdict(id=9, name="MX304-LMIC"), _dotdict(id=8, name="QSFP-DD")]) + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC", "QSFP-DD"]}] + + def test_a_bay_with_no_classes_writes_no_key(self): + """An empty list here would add the key to every bay in the library. + + See test_an_empty_relation_does_not_make_every_definition_differ for the effect + that has on the export diff. + """ + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + + result = serialize_module_type(record, {1: {"module_bay_templates": [self._bay("FPC 0", [])]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_server_that_never_returned_the_field_omits_the_key(self): + """Below 4.7 the relation is not selected, and absent must not become empty.""" + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = _dotdict(name="FPC 0", position=None, label="", description="") + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_module_type_exports_the_classes_it_belongs_to(self): + record = _dotdict( + id=5, + model="JNP304-RE", + manufacturer=_make_mfr(name="Juniper", slug="juniper"), + part_number=None, + module_bay_types=[_dotdict(id=9, name="MX304-RE")], + ) + + assert serialize_module_type(record, {})["module_bay_types"] == ["MX304-RE"] + + def test_a_device_type_bay_exports_its_classes_too(self): + record = _dotdict( + id=2, model="MX304", slug="mx304", manufacturer=_make_mfr(), u_height=None, is_full_depth=None + ) + bay = self._bay("RE0", [_dotdict(id=9, name="MX304-RE")]) + + result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 3f7b3315..eb535191 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -112,7 +112,7 @@ def _mark_cache_ready(device_types): def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): # Mock api call - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.url == "http://mock-netbox" @@ -128,7 +128,7 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc remove_unmanaged_types=True, verify_images=True, ) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" netbox = NetBox(config, mock_handle) @@ -138,24 +138,16 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): - # Test 5.0 - mock_pynetbox.api.return_value.version = "5.0" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters - - # Test 4.0 - mock_pynetbox.api.return_value.version = "4.0" - nb = NetBox(mock_settings, mock_handle) - assert not nb.new_filters - - # Test 4.1 - mock_pynetbox.api.return_value.version = "4.1" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters + """Every supported release uses the new filter names; 4.7 adds module bay types.""" + for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): + mock_pynetbox.api.return_value.version = version + nb = NetBox(mock_settings, mock_handle) + assert nb.new_filters, version + assert nb.module_bay_types is module_bay_types, version def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] nb = NetBox(mock_settings, mock_handle) @@ -168,7 +160,7 @@ def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): def test_create_manufacturers_no_new_is_verbose_only(mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -233,7 +225,7 @@ def test_create_generic_counts_the_created_list_at_the_caller(mock_pynetbox, mak def test_redundant_image_upload(mock_settings, mock_pynetbox, mock_handle): # Setup # Ensure modules check doesn't fail - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = MagicMock() @@ -611,7 +603,7 @@ class TestNetBoxConnectApi: def test_ssl_ignore_sets_verify_false(self, mock_pynetbox, mock_handle, make_config): mock_settings = make_config(ignore_ssl_errors=True) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.netbox.http_session.verify is False @@ -622,7 +614,7 @@ class TestCreateManufacturersError: def test_request_error_logged(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" # Make pynetbox.RequestError in the module under test be the real exception class mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -930,7 +922,7 @@ def test_creates_new_device_type_with_components( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) @@ -1118,14 +1110,14 @@ class TestCreateModuleTypes: """Tests for TestCreateModuleTypes.""" def test_empty_module_types_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) # Should not raise and should not call create nb.create_module_types([]) nb.netbox.dcim.module_types.create.assert_not_called() def test_creates_new_module_type(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1250,7 +1242,7 @@ class TestCreateModuleTypesBody: """Tests for TestCreateModuleTypesBody.""" def test_cached_module_type_skips_creation(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1285,7 +1277,7 @@ def test_create_module_type_request_error_logged( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1310,7 +1302,7 @@ def test_create_module_type_request_error_logged( def test_creates_module_type_with_components( self, mock_settings, mock_pynetbox, mock_graphql_requests, graphql_client, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1606,7 +1598,7 @@ class TestCreateManufacturersSuccessLog: def test_verbose_log_per_created_manufacturer(self, mock_settings, mock_pynetbox, mock_handle): """verbose_log should be called for each created manufacturer.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_m = MagicMock() @@ -2333,7 +2325,7 @@ def test_create_failure_reaches_the_end_of_run_failure_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b"{\"manufacturer\":[\"Related object not found using the provided attributes: {'slug': 'ribbon'}\"]}" ) @@ -2574,7 +2566,7 @@ class TestFilterActionableModuleTypesEdge: def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """Empty module_types list returns [], {} immediately.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) result, images, _ = nb.filter_actionable_module_types([], {}, only_new=False) assert result == [] @@ -2582,7 +2574,7 @@ def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mo def test_only_new_delegates_to_filter_new(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """only_new=True returns only genuinely new module types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() all_mts = {"cisco": {"LC": existing_mt}} @@ -2599,7 +2591,7 @@ def test_new_module_type_added_to_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """Module type not in all_module_types is added to actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2622,7 +2614,7 @@ def test_existing_module_with_new_image_is_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """Existing module type with an image not yet in NetBox is actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2673,7 +2665,7 @@ def test_existing_module_with_changed_property_is_actionable( """Existing module type with a changed scalar property (e.g. part_number) is actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2726,7 +2718,7 @@ def test_existing_module_with_missing_image_and_property_change_logs_both( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2782,7 +2774,7 @@ def test_existing_module_with_only_missing_image_is_actionable_but_not_logged( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2827,7 +2819,7 @@ def test_existing_module_with_unchanged_property_is_not_actionable( """Existing module type whose properties all match NetBox is not actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2868,7 +2860,7 @@ class TestCreateModuleTypesEdge: def test_existing_module_type_verbose_logged(self, mock_settings, mock_pynetbox, mock_handle): """When a module type already exists, verbose_log is called with 'Cached'.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() @@ -2894,7 +2886,7 @@ def test_only_new_skips_existing_module_component_creation( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """only_new=True + existing module → skip component creation.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("interface_templates", "module", 5, {}) @@ -2923,7 +2915,7 @@ def test_creates_module_type_with_power_outlets_console_server_ports_front_ports self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """power-outlets, console-server-ports, front-ports branches in create_module_types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) for endpoint_name in ("power_outlet_templates", "console_server_port_templates", "front_port_templates"): @@ -2955,7 +2947,7 @@ def test_a_new_module_type_gets_its_module_bays( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The DTL module-type schema allows module-bays, so creation must not skip them.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("module_bay_templates", "module", 5, {}) @@ -2981,7 +2973,7 @@ def test_existing_module_type_property_update_calls_api(self, mock_settings, moc """Existing module type with changed part_number calls module_types.update and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3017,7 +3009,7 @@ def test_existing_module_type_property_unchanged_no_api_call(self, mock_settings """Existing module type with matching part_number does not call module_types.update.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3048,7 +3040,7 @@ def test_existing_module_type_only_new_skips_property_update(self, mock_settings """only_new=True skips property update even when part_number differs.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3082,7 +3074,7 @@ def test_existing_module_type_component_update_calls_update_components( """Existing module type with changed component property calls update_components and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3146,7 +3138,7 @@ def test_existing_module_type_property_and_component_update_increments_once( """Both property and component change → module_updated incremented only once.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3214,7 +3206,7 @@ def test_existing_module_type_removal_only_no_counter_increment( """COMPONENT_REMOVED-only changes call update_components but do NOT increment module_updated.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3275,7 +3267,7 @@ def test_property_update_plus_removal_only_remove_false_counts_as_updated( """Properties changed + removal-only diff with remove_components=False → module_updated incremented.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.update_components = MagicMock() @@ -3358,7 +3350,7 @@ def test_existing_image_is_skipped( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """If the image name is already in module_type_existing_images, upload is skipped.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -3388,7 +3380,7 @@ def test_new_image_is_uploaded_and_tracked( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """When image is not yet in existing_images, upload_image_attachment is called.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -4218,7 +4210,7 @@ class TestCreateModuleTypesCornerCases: def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_mt = MagicMock() @@ -4252,7 +4244,7 @@ def tracking_iter(): def test_all_module_types_fetched_when_none(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """all_module_types is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4277,7 +4269,7 @@ def test_module_type_existing_images_fetched_when_none( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """module_type_existing_images is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4421,7 +4413,7 @@ class TestGetExistingRackTypes: def test_delegates_to_graphql(self, mock_settings, mock_pynetbox, graphql_client, mock_handle): """get_existing_rack_types() returns whatever graphql.get_rack_types() returns.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.graphql = graphql_client expected = {"apc": {"AR1300": MagicMock()}} @@ -4442,7 +4434,7 @@ class TestCreateRackTypes: """Tests for NetBox.create_rack_types().""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_empty_list_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): @@ -4510,7 +4502,7 @@ def test_existing_rack_type_fields_differ_calls_update(self, mock_settings, mock def test_new_rack_type_calls_create(self, mock_settings, mock_pynetbox, mock_handle): """Non-existing rack type: create called, counter incremented, added to cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 99 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4532,7 +4524,7 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet """RequestError during create is logged; processing continues.""" import pynetbox - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4553,7 +4545,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet import pynetbox from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4573,7 +4565,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, mock_handle): """When all_rack_types=None, get_existing_rack_types() is called to populate the cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) nb.get_existing_rack_types = MagicMock(return_value={}) rack_type = { @@ -4587,7 +4579,7 @@ def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When a progress wrapper is provided, it is used as the iterator.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 1 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4615,16 +4607,15 @@ class TestVerifyCompatibility: @pytest.mark.parametrize( "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", [ - ("3.1", False, False, False, False), - ("3.2", True, False, False, False), - ("4.0", True, False, False, False), - ("4.1", True, True, True, False), + ("4.3", True, True, True, False), ("4.4", True, True, True, False), ("4.5", True, True, True, True), ("4.6", True, True, True, True), + ("4.7", True, True, True, True), + ("5.0", True, True, True, True), # Version strings with non-numeric suffixes ("4.5-beta", True, True, True, True), - ("4.1.0", True, True, True, False), + ("4.3.0", True, True, True, False), ], ) def test_version_thresholds( @@ -4646,16 +4637,19 @@ def test_version_thresholds( assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): - """Version string with only major component (e.g. '4') does not crash.""" - mock_pynetbox.api.return_value.version = "4" + """A version string with only a major component (e.g. '5') does not crash.""" + mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is False # 4.0 → no new filters + assert nb.new_filters is True - def test_version_42_enables_new_filters_not_m2m(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "4.2" + def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( + self, mock_settings, mock_pynetbox, mock_handle + ): + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.new_filters is True assert nb.m2m_front_ports is False + assert nb.module_bay_types is False # ============================================================ @@ -4855,7 +4849,7 @@ class TestLogModuleTypeChanges: def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, mock_handle): """A non-empty changed_property_log triggers verbose logging.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4868,7 +4862,7 @@ def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, def test_empty_log_emits_nothing(self, mock_settings, mock_pynetbox, mock_handle): """An empty changed_property_log does not trigger any logging calls.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4887,7 +4881,7 @@ class TestTryUpdateModuleTypeErrors: """Tests for RequestError and retryable-exception handlers in _try_update_module_type.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def _make_module_type_res(self): @@ -4901,7 +4895,7 @@ def test_request_error_returns_false_and_logs(self, mock_settings, mock_pynetbox """pynetbox.RequestError during update causes (False, False) return and log.""" import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) @@ -4926,7 +4920,7 @@ def test_retryable_exception_returns_false_and_logs(self, mock_settings, mock_py import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) mock_handle.log.reset_mock() @@ -4969,7 +4963,7 @@ def test_retryable_exception_on_create_returns_false( import requests mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -5777,7 +5771,7 @@ class TestUploadModuleTypeImagesVerify: """Tests for _upload_module_type_images with verify_images=True.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types.upload_image_attachment = MagicMock(return_value=True) nb.verify_images = True @@ -5936,7 +5930,7 @@ def test_load_image_hash_cache_returns_empty_dict_on_bad_json(self, tmp_path): def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_settings, mock_pynetbox, mock_handle): from core.graphql_client import GraphQLError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with patch.object(NetBox, "get_manufacturers", side_effect=GraphQLError("bad query")): with pytest.raises(NetBoxError, match="GraphQL error: bad query"): @@ -5945,7 +5939,7 @@ def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_setti def test_init_raises_typed_error_when_device_types_initialization_fails( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with ( patch.object(NetBox, "get_manufacturers", return_value=[]), @@ -6022,7 +6016,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.manufacturers.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6032,7 +6026,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock assert any("Connection error creating manufacturers" in str(c) for c in mock_handle.log.call_args_list) def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") @@ -6049,7 +6043,7 @@ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pyn from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") resolution = SimpleNamespace( @@ -6072,7 +6066,7 @@ def test_try_resolve_update_logs_auto_resolve_failure(self, mock_settings, mock_ from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6105,7 +6099,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( from core.update_failure_resolver import FailureKind mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6132,7 +6126,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( def test_log_device_type_change_outcome_partial_success_mentions_property_failure( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6151,7 +6145,7 @@ def test_log_device_type_change_outcome_partial_success_mentions_property_failur def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6168,7 +6162,7 @@ def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( assert any("Device Type Cached" in str(c) for c in mock_handle.verbose_log.call_args_list) def test_filter_images_for_upload_keeps_changed_image(self, mock_settings, mock_pynetbox, tmp_path, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True @@ -6194,7 +6188,7 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( from core.change_detector import PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.update.side_effect = requests.exceptions.ConnectionError("offline") nb._log_device_type_change_outcome = MagicMock() @@ -6217,7 +6211,7 @@ def test_create_new_device_type_logs_retryable_error(self, mock_settings, mock_p import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6238,7 +6232,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( ): from core.change_detector import ChangeType, ComponentChange, PropertyChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) changes = [ ComponentChange("interfaces", "xe-0", ChangeType.COMPONENT_ADDED), @@ -6261,7 +6255,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True details = {7: {"linecard.front": {"att_id": 5, "url": "/media/linecard.front.jpg"}}} @@ -6273,7 +6267,7 @@ def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( assert nb._module_image_details == details def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(spec=["id", "manufacturer", "model"]) module_type_res.id = 1 @@ -6290,7 +6284,7 @@ def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, def test_filter_actionable_module_types_marks_verify_images_module_actionable( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True _mark_cache_ready(nb.device_types) @@ -6314,7 +6308,7 @@ class TestAdditionalModuleTypeCoverage: def test_apply_module_type_component_updates_records_failed_no_actionable_changes( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6334,7 +6328,7 @@ def test_apply_module_type_component_updates_marks_partial_on_partial_component_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6362,7 +6356,7 @@ def test_apply_module_type_component_updates_marks_partial_when_properties_only_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6383,7 +6377,7 @@ def test_apply_module_type_component_updates_records_failed_when_no_changes_appl ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6510,7 +6504,7 @@ def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock from core.graphql_client import DotDict mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6530,7 +6524,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6547,7 +6541,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock def test_upload_module_type_images_discards_missing_attachment_before_failed_upload( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=False) @@ -6575,7 +6569,7 @@ def test_upload_module_type_images_discards_missing_attachment_before_failed_upl def test_upload_module_type_images_skips_changed_image_when_delete_fails( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=True) @@ -6825,7 +6819,7 @@ def test_filter_actionable_module_types_skips_unchanged_existing_module( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6874,7 +6868,7 @@ def test_filter_actionable_module_types_includes_module_with_missing_component( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6922,7 +6916,7 @@ def test_missing_netbox_field_is_not_treated_as_change( ): """When existing module lacks an attribute, it's skipped — no false positive change.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6979,7 +6973,7 @@ def test_remove_components_is_called_when_flag_set( ): """When remove_components=True and there are component changes, remove_components is called.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7037,7 +7031,7 @@ def test_component_reconciliation_continues_when_scalar_patch_fails( skipped entirely. """ mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7106,7 +7100,7 @@ def _netbox(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_rack_type_create_failure_is_reported(self, mock_settings, mock_pynetbox, mock_handle): @@ -7190,7 +7184,7 @@ def test_device_type_create_failure_is_not_called_an_update_failure( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b'{"manufacturer":["Related object not found using the provided attributes: slug ribbon"]}' ) @@ -7223,7 +7217,7 @@ def test_module_type_create_failure_is_not_called_an_update_failure( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.module_types.create.side_effect = _request_error( b'{"model":["This field may not be blank."]}' ) @@ -7247,7 +7241,7 @@ class TestSkippedComponentReasonReachesTheReport: def _dt(self, mock_pynetbox, make_device_types, parent_id=1): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) return dt @@ -7260,7 +7254,7 @@ def test_unresolvable_power_port_reaches_the_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) inlet = MagicMock() @@ -7351,7 +7345,7 @@ def test_component_update_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) existing_iface = MagicMock() @@ -7410,7 +7404,7 @@ def test_component_removal_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) stale = MagicMock() @@ -7439,7 +7433,7 @@ def test_failed_component_create_reports_the_netbox_message( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 6119, {}) @@ -7484,7 +7478,7 @@ def _nb(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_new_device_type_with_failing_component_is_partial( @@ -7495,7 +7489,7 @@ def test_new_device_type_with_failing_component_is_partial( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 900, {}) mock_nb_api.dcim.interface_templates.create.side_effect = _request_error( @@ -7532,7 +7526,7 @@ def test_new_device_type_with_failing_component_is_partial( def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, graphql_client, make_device_types): """Errors buffered for one entity must not surface in the next entity's reason.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt._log_component_error("stale error from an earlier entity") @@ -7547,7 +7541,7 @@ def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, grap def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_client, make_device_types): """The buffer is empty after a scope closes, so the next scope starts clean.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) with dt.collect_component_errors() as first: @@ -7557,3 +7551,16 @@ def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_cl with dt.collect_component_errors() as second: pass assert second == [] + + +def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_settings, mock_pynetbox, mock_handle): + """A GraphQL schema error is a poor way to learn the server is too old.""" + from core.netbox_api import NetBoxError + + for version in ("4.1", "4.2", "3.5"): + mock_pynetbox.api.return_value.version = version + with pytest.raises(NetBoxError) as exc: + NetBox(mock_settings, mock_handle) + message = str(exc.value) + assert "4.3" in message, f"{version}: the message must name the minimum" + assert version in message, f"{version}: the message must name what was found" diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py new file mode 100644 index 00000000..7dac8f0f --- /dev/null +++ b/tests/test_relation_scope.py @@ -0,0 +1,188 @@ +"""A relation assigned from the wrong manufacturer scope must be detected as a change. + +The catalog resolves a reference name in the owning manufacturer's scope first and only +then in Generic. If NetBox already holds the Generic object where the owner has one of +its own, comparing names alone reports equality and the wrong object survives. +""" + +import pytest + +from core.change_detector import ChangeDetector +from core.module_bay_types import ModuleBayTypeCatalog + + +class Handle: + """Capture what the detector logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +class Related: + """A module bay type as NetBox returns it inside a relation.""" + + def __init__(self, name, slug, manufacturer_slug): + """Store the identity fields the comparison needs.""" + self.name = name + self.slug = slug + self.manufacturer = type("M", (), {"slug": manufacturer_slug})() + + +class NetBoxBay: + """A module bay template as the cache hands it to the detector.""" + + def __init__(self, name, module_bay_types): + """Store the bay name and its assigned classes.""" + self.name = name + self.module_bay_types = module_bay_types + + +@pytest.fixture +def two_scope_catalog(tmp_path): + """Build a catalog where the same class name exists under Acme and under Generic.""" + for manufacturer, slug in (("Acme", "acme-x"), ("Generic", "generic-x")): + directory = tmp_path / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{slug}.yaml").write_text( + f"name: X\nslug: {slug}\nmanufacturer: {manufacturer}\n", encoding="utf-8" + ) + return str(tmp_path) + + +def _detector(catalog, handle=None): + """Build a detector whose device_types exposes the catalog, as the real one does.""" + device_types = type("DeviceTypes", (), {"module_bay_types": catalog, "module_bay_types_supported": True})() + return ChangeDetector(device_types, handle or Handle()) + + +def test_wrong_scope_assignment_is_detected(two_scope_catalog): + """Acme owns X, but NetBox assigned Generic's X. The names match; the objects do not.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert [c.property_name for c in changes] == ["module_bay_types"], ( + "a bay owned by Acme holding Generic's X must be corrected to Acme's X" + ) + + +def test_right_scope_assignment_is_left_alone(two_scope_catalog): + """The same bay already holding Acme's X is correct and must not be rewritten.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert changes == [] + + +def test_generic_fallback_is_correct_when_the_owner_has_no_such_class(two_scope_catalog): + """A Nokia bay resolves X to Generic's X, so holding Generic's X is right.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="nokia" + ) + assert changes == [] + + +def _changes(detector, yaml_comp, netbox_comp, manufacturer="acme"): + """Run the real property comparison for the relation and return what it found.""" + return detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer=manufacturer + ) + + +class TestUnmanagedRelations: + """A relation is only rewritten when both sides say something about it.""" + + def test_an_omitted_key_leaves_the_relation_alone(self, two_scope_catalog): + """A definition that never mentions the relation is not asking for it to be cleared.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0"}, netbox_comp) == [] + + def test_a_bare_key_leaves_the_relation_alone_but_says_so(self, two_scope_catalog): + """Parses as None, so nothing is cleared; a typo must still not be invisible.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": None}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): + """Reading an absent field as empty would report a change on every run.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = type("Bay", (), {"name": "Slot 0"})() + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_an_unresolvable_reference_reaches_the_write_path(self, two_scope_catalog): + """Reporting "no change" leaves the bay unrestricted in silence. + + The write path is the only thing that logs and records an unresolvable name, and it + only ever sees a component the detector reported as changed. + """ + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["NO-SUCH-CLASS"]}, netbox_comp) + + assert [(c.property_name, c.new_value) for c in changes] == [("module_bay_types", ["NO-SUCH-CLASS"])] + + +class TestNameComparisonFallback: + """Where an identity cannot be had, comparing names is still better than doing nothing.""" + + def test_names_are_compared_when_netbox_returned_no_slug(self, two_scope_catalog): + """A read path returning only id and name cannot answer the scope question.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", None, None)]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.property_name, c.old_value, c.new_value) for c in changes] == [("module_bay_types", ["Y"], ["X"])] + + def test_matching_names_are_left_alone_when_netbox_returned_no_slug(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", None, None)]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_names_are_compared_without_a_catalog(self): + """A caller that has not wired a catalog still gets the name comparison.""" + detector = _detector(None) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", "generic-y", "generic")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["Y"], ["X"])] + + def test_an_empty_list_clears_the_relation(self, two_scope_catalog): + """`module_bay_types: []` is an explicit instruction to hold no classes.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": []}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["X"], [])] diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index dd8a46bc..e85d411e 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -82,6 +82,25 @@ def test_integration_collection_reads_credentials_from_a_local_env_file(tmp_path assert "1 passed" in result.stdout, result.stdout + result.stderr +def test_the_fake_netbox_releases_its_listening_socket(): + """shutdown() only stops the serve loop. One server per test leaks a descriptor each.""" + import socket + + from helpers import FakeNetBox + + server = FakeNetBox(manufacturers=[{"id": 1, "name": "Juniper", "slug": "juniper"}]) + port = server._server.server_port + server.close() + + probe = socket.socket() + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: # pragma: no cover - only reached when close() leaks + raise AssertionError(f"FakeNetBox.close() left port {port} bound: {exc}") from exc + finally: + probe.close() + + def test_no_test_function_contains_an_orphaned_docstring(): """A dropped ``def`` header silently merges two tests into one. From a0ee29b5c7751bcb0ad9a0b4ac882e49ea920fc1 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:07:07 +0200 Subject: [PATCH 04/39] feat(export): write the NetBox 4.5 port-mappings stanza (#134) Closes #78. NetBox 4.5 replaced FrontPortTemplate's rear_port foreign key and rear_port_position integer with a through table, so one front port can occupy several positions across several rear ports. The export wrote only the first mapping and warned about the rest, and it wrote them inline on the front-port entry. The device-type library has since taken the same model: a front-port entry carries no rear port at all, the linkage lives in a top-level port-mappings stanza, and additionalProperties is false, so the old shape no longer validates. The export now writes every mapping into that stanza with all four fields. A server below 4.5 answers with the rear_port and rear_port_position scalars instead; those describe one mapping, so they are written as one entry rather than dropped, because the output format follows the library schema and not the server release. The library also made positions required on front and rear ports, and writes it even when it is 1, so it is no longer omitted at its default and the front port row selects it. positions arrived with the mapping model, so the pre-4.5 tiers of the front-port query fallback must not ask for it: leaving it in made every tier fail the same way and took the whole front-port preload down on a server that would have answered the older shape. Verified the output against the real schema/devicetype.json from the library, and round-tripped it back through normalize_port_mappings, which reads the stanza and rebuilds the same per-port mappings. --- core/component_registry.py | 3 +- core/graphql_client.py | 5 +- core/nb_serializer.py | 81 ++++---- tests/test_component_registry.py | 1 + tests/test_graphql_client.py | 18 ++ tests/test_nb_serializer.py | 314 ++++++++++++------------------- 6 files changed, 198 insertions(+), 224 deletions(-) diff --git a/core/component_registry.py b/core/component_registry.py index a1fc18e7..fe414278 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -119,7 +119,8 @@ def create_label(self, parent_type): yaml_key="front-ports", endpoint="front_port_templates", label="Front Port", - fields=("name", "type", "label", "description", "color"), + # positions arrived with the 4.5 mapping model; the query drops it on older servers. + fields=("name", "type", "label", "description", "color", "positions"), graphql_extra=("mappings { id front_port_position rear_port_position rear_port { id name } }",), compare_extra=("_mappings",), link=LINK_REAR_PORTS, diff --git a/core/graphql_client.py b/core/graphql_client.py index 4a8fe786..88cd53e3 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -744,7 +744,7 @@ def get_module_type_image_details(self): def _front_port_field_variants(fields): """Yield successive field-list tiers for the front_port_templates fallback. - Tier 1: mappings block (NetBox 4.5+) + Tier 1: mappings block and positions (NetBox 4.5+) Tier 2: rear_port_position scalar (<4.5) Tier 3: neither (field removed entirely) """ @@ -753,7 +753,8 @@ def _front_port_field_variants(fields): for f in fields: if "mappings" in f: fallback.extend(["rear_port_position", "rear_port { id name }"]) - else: + elif f != "positions": + # positions arrived with the mapping model, so no pre-4.5 tier may ask for it. fallback.append(f) yield fallback stripped = [f for f in fallback if f != "rear_port_position" and "rear_port" not in f] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 5ba5b2ca..5e899758 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -4,7 +4,6 @@ comparison against existing repo YAML files. """ -import warnings from typing import Any, Sequence from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS @@ -26,7 +25,6 @@ "feed_leg": None, "maximum_draw": None, "allocated_draw": None, - "positions": 1, # rear port default; include only when > 1 } # Device type scalar field order for output. @@ -146,37 +144,53 @@ def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: def _serialize_front_port(record: Any) -> dict: - """Serialize a front port template, including rear_port mapping.""" - result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) - mappings = getattr(record, "mappings", None) or [] - if mappings: - if len(mappings) > 1: - port_name = getattr(record, "name", "") - warnings.warn( - f"Front port '{port_name}' has {len(mappings)} mappings; " - "only the first will be exported. " - "Full multi-mapping support requires DTL schema update (see issue #78).", - UserWarning, - stacklevel=4, + """Serialize a front port template's own fields. + + The rear-port linkage is no longer written here: NetBox 4.5 moved it to a through + table and the library schema follows, carrying it in a top-level ``port-mappings`` + stanza built by :func:`_port_mappings`. + """ + return _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + + +def _port_mappings(records: list) -> list: + """Return the ``port-mappings`` stanza for a type's front port templates. + + Every mapping is written, not just the first: one front port may occupy several + positions across rear ports, which is what the through table exists to express. + + A server below 4.5 has no through table and answers with ``rear_port`` and + ``rear_port_position`` scalars instead. Those describe one mapping, so they are + written as one entry rather than dropped. + """ + stanza = [] + for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): + name = getattr(record, "name", None) + for mapping in getattr(record, "mappings", None) or []: + rear_port = getattr(mapping, "rear_port", None) + if not rear_port: + continue + stanza.append( + { + "front_port": name, + "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, + "rear_port": rear_port.name, + "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, + } ) - m = mappings[0] - rear_port = getattr(m, "rear_port", None) - if rear_port: - result["rear_port"] = rear_port.name - rear_pos = getattr(m, "rear_port_position", None) - rear_pos = _coerce_numeric(rear_pos) - if rear_pos is not None and rear_pos > 1: - result["rear_port_position"] = rear_pos - else: - # Legacy: pre-4.5 NetBox returns rear_port / rear_port_position as direct scalar fields - legacy_rp = getattr(record, "rear_port", None) - if legacy_rp: - result["rear_port"] = legacy_rp.name - legacy_pos = getattr(record, "rear_port_position", None) - legacy_pos = _coerce_numeric(legacy_pos) - if legacy_pos is not None and legacy_pos > 1: - result["rear_port_position"] = legacy_pos - return result + if getattr(record, "mappings", None): + continue + legacy = getattr(record, "rear_port", None) + if legacy: + stanza.append( + { + "front_port": name, + "front_port_position": 1, + "rear_port": legacy.name, + "rear_port_position": _coerce_numeric(getattr(record, "rear_port_position", None)) or 1, + } + ) + return stanza def _serialize_component_list(endpoint_name: str, records: list) -> list: @@ -200,6 +214,9 @@ def _add_components(result: dict, type_id: int, components_by_id: dict) -> None: records = type_components.get(component.endpoint, []) if records: result[component.yaml_key] = _serialize_component_list(component.endpoint, records) + mappings = _port_mappings(type_components.get("front_port_templates", [])) + if mappings: + result["port-mappings"] = mappings def serialize_device_type(nb_record: Any, components_by_dt_id: dict) -> dict: diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 7e536f53..f1d05dbe 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -128,6 +128,7 @@ def test_the_query_selects_exactly_these_fields(self): "label", "description", "color", + "positions", _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index b72ddae6..5cfb8004 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -1152,6 +1152,24 @@ def _query_for(supported): assert "module_bay_types" in _query_for(True) assert "module_bay_types" not in _query_for(False) + def test_the_front_port_fallback_drops_every_45_only_field(self, mock_post): + """Positions arrived with the mapping model, so a pre-4.5 tier must not ask for it. + + Leaving it in a fallback tier makes every tier fail the same way, and the whole + front-port preload dies on a server that would answer the older shape. + """ + from core.component_registry import BY_ENDPOINT + from core.graphql_client import NetBoxGraphQLClient + + fields = list(BY_ENDPOINT["front_port_templates"].graphql_fields) + assert "positions" in fields, "guard: the registry is expected to select positions" + + tiers = list(NetBoxGraphQLClient._front_port_field_variants(fields)) + + assert any("positions" in f for f in tiers[0]), "the 4.5 tier keeps it" + for tier in tiers[1:]: + assert not any("positions" in f for f in tier), f"pre-4.5 tier still asks for it: {tier}" + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 3d5edf70..fce437b1 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -441,195 +441,7 @@ def test_rack_type_manufacturer_as_name_string(self): class TestFrontPortSerialization: - """Tests for front port rear_port extraction.""" - - def test_front_port_rear_port_extracted_from_mapping(self): - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] - - def test_front_port_rear_port_position_included_when_gt_1(self): - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=3) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port_position"] == 3 - - def test_front_port_rear_port_position_zero_omitted(self): - """Position 0 is not a valid DTL value and should be omitted.""" - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=0) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert "rear_port_position" not in result["front-ports"][0] - - def test_front_port_multiple_mappings_warns_and_uses_first(self): - """When a front port has >1 mappings a UserWarning is raised and only the first is used.""" - import warnings - from types import SimpleNamespace - - m1 = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - m2 = SimpleNamespace(rear_port=SimpleNamespace(name="RP2"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[m1, m2]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert len(caught) == 1 - assert issubclass(caught[0].category, UserWarning) - assert "FP1" in str(caught[0].message) - assert "2 mappings" in str(caught[0].message) - assert "issue #78" in str(caught[0].message) - - def test_front_port_legacy_rear_port_scalars(self): - """pre-4.5 NetBox: record has rear_port/rear_port_position as direct attrs (no mappings).""" - from types import SimpleNamespace - - fp = SimpleNamespace( - name="FP1", - type="8p8c", - label="", - description="", - color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=3, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert result["front-ports"][0]["rear_port_position"] == 3 - - def test_front_port_legacy_rear_port_position_1_omitted(self): - """pre-4.5: rear_port_position == 1 should be omitted (same as mappings path).""" - from types import SimpleNamespace - - fp = SimpleNamespace( - name="FP1", - type="8p8c", - label="", - description="", - color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=1, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] + """Front port scalars. The rear-port linkage lives in TestPortMappingsStanza.""" def test_components_sorted_by_name(self): from types import SimpleNamespace @@ -743,3 +555,127 @@ def test_a_device_type_bay_exports_its_classes_too(self): result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] + + +class TestPortMappingsStanza: + """Export writes the NetBox 4.5 port-mappings stanza the DTL schema now requires.""" + + @staticmethod + def _mapping(rear_port, rear_position=1, front_position=1): + from types import SimpleNamespace + + return SimpleNamespace( + rear_port=SimpleNamespace(name=rear_port), + rear_port_position=rear_position, + front_port_position=front_position, + ) + + @staticmethod + def _front_port(name, mappings=None, positions=1, **extra): + from types import SimpleNamespace + + return SimpleNamespace( + name=name, + type="lc-upc", + label="", + description="", + color="", + positions=positions, + mappings=mappings or [], + **extra, + ) + + def _device(self): + return _dotdict( + id=1, + model="X", + slug="acme-x", + manufacturer=_make_mfr(), + u_height=1, + is_full_depth=True, + part_number=None, + airflow=None, + weight=None, + weight_unit=None, + description="", + comments="", + subdevice_role=None, + front_image=None, + rear_image=None, + ) + + def test_a_front_port_carries_positions_and_no_inline_rear_port(self): + """The schema dropped rear_port from front-port entries and made positions required.""" + fp = self._front_port("FP1", [self._mapping("RP1")], positions=1) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "lc-upc", "positions": 1}] + + def test_every_mapping_reaches_the_stanza_not_just_the_first(self): + """The issue: a crossover front port mapped to two rear ports lost the second.""" + fp = self._front_port("FP1", [self._mapping("RP1", 1), self._mapping("RP2", 3, front_position=2)], positions=2) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 1}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2", "rear_port_position": 3}, + ] + + def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): + """The shape the library actually carries: many front ports onto one MPO rear port.""" + ports = [self._front_port(f"FP{i}", [self._mapping("MPO1", i)]) for i in (1, 2, 3)] + + result = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert [(m["front_port"], m["rear_port_position"]) for m in result["port-mappings"]] == [ + ("FP1", 1), + ("FP2", 2), + ("FP3", 3), + ] + + def test_a_pre_45_server_still_exports_its_mappings(self): + """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" + from types import SimpleNamespace + + fp = SimpleNamespace( + name="FP1", + type="8p8c", + label="", + description="", + color="", + rear_port=SimpleNamespace(name="RP1"), + rear_port_position=4, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} + ] + + def test_a_front_port_with_no_mapping_adds_no_stanza(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) + + assert "port-mappings" not in result + + def test_a_type_without_front_ports_adds_no_stanza(self): + assert "port-mappings" not in serialize_device_type(self._device(), {1: {}}) + + def test_the_importer_reads_back_what_the_export_wrote(self): + """Serializer to normalizer, both real: the stanza is the seam between them.""" + from core.repo import normalize_port_mappings + + ports = [ + self._front_port("1", [self._mapping("MPO1", 1)]), + self._front_port("2", [self._mapping("MPO1", 2)]), + ] + exported = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert normalize_port_mappings(exported) is None + assert [fp["_mappings"] for fp in exported["front-ports"]] == [ + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 1}], + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 2}], + ] + assert "port-mappings" not in exported, "the normalizer consumes the stanza" From 43480e70e63237d15820f0aa964eb2a9ac9748ab Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 09:25:37 +0200 Subject: [PATCH 05/39] fix(export): default front-port positions for pre-4.5 servers The DTL front-port schema makes positions required, but the field arrived with the NetBox 4.5 mapping model. The pre-4.5 query tiers do not select it, so _serialize_component found no value and omitted the key, and every front-port entry exported from an older server failed schema validation. Default an absent positions to 1, which is what a pre-4.5 front port occupies: one rear-port position. positions is the last registry field, so the key lands in the same slot the 4.5 path produces and the YAML order does not change with the server version. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/134#discussion_r3965186646 --- core/nb_serializer.py | 5 ++++- tests/test_nb_serializer.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 5e899758..9b89619e 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -150,7 +150,10 @@ def _serialize_front_port(record: Any) -> dict: table and the library schema follows, carrying it in a top-level ``port-mappings`` stanza built by :func:`_port_mappings`. """ - return _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + # positions is schema-required but arrived in 4.5, so a pre-4.5 record has none. + result.setdefault("positions", 1) + return result def _port_mappings(records: list) -> list: diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index fce437b1..fdbbf119 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -654,6 +654,17 @@ def test_a_pre_45_server_still_exports_its_mappings(self): assert result["port-mappings"] == [ {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} ] + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_legacy_front_port_carries_the_schema_required_positions(self): + """The schema requires positions, but it arrived in 4.5, so a pre-4.5 record needs the default.""" + from types import SimpleNamespace + + fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] def test_a_front_port_with_no_mapping_adds_no_stanza(self): result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) From a9fed7212a8a6c82a4976ee3a80376dfc9d4e426 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 09:45:57 +0200 Subject: [PATCH 06/39] feat(export): report module bays exported without a position The library schema requires a position on every module bay, but NetBox leaves the field blank when a bay names no physical slot. A blank string writes no key, so the exported entry silently omits a required field and the file fails validation only later, in the library CI. Name the offending bays as each file is written, so the operator sees which type needs a position before opening a library PR. The export still writes the file: NetBox is the source of truth here, and there is no correct value to invent, unlike front-port positions where one position is implied. module_bays_missing_position keeps the schema fact next to the serializer that produces the entries, and the exporter owns the logging as it does for the other write-time warnings. --- core/export.py | 7 +++++ core/nb_serializer.py | 9 +++++++ tests/test_exporter.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/core/export.py b/core/export.py index f0911ee6..b14ebc7b 100644 --- a/core/export.py +++ b/core/export.py @@ -23,6 +23,7 @@ from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.nb_serializer import ( COMPONENT_ENDPOINT_NAMES, + module_bays_missing_position, serialize_device_type, serialize_module_type, serialize_rack_type, @@ -466,6 +467,12 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: continue written_count += 1 + bays = module_bays_missing_position(to_write) + if bays: + self.handle.log( + f"[yellow]{item.mfr_name}/{item.filename}: module bay(s) {', '.join(bays)} have " + f"no position, which the library schema requires[/yellow]" + ) images_ok = self._download_type_images(item) if images_ok: update_entry(manifest, f"{item.kind}s", item.manifest_key, item.nb_record.last_updated) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 9b89619e..a822a130 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -196,6 +196,15 @@ def _port_mappings(records: list) -> list: return stanza +def module_bays_missing_position(serialized: dict) -> list: + """Return the names of module bays the library schema would reject. + + NetBox leaves ``position`` blank on a bay that names no physical slot, and a blank + string writes no key, but the schema requires one on every module bay. + """ + return [bay.get("name", "?") for bay in serialized.get("module-bays", []) if "position" not in bay] + + def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" component = BY_ENDPOINT[endpoint_name] diff --git a/tests/test_exporter.py b/tests/test_exporter.py index d9eebe29..343d9c20 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1565,3 +1565,62 @@ def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_ with pytest.raises(FileNotFoundError, match="No device-type library found"): self._exporter(tmp_path, repo)._verify_repo_available() + + +class TestModuleBayPositionWarning: + """NetBox allows a blank module-bay position; the DTL schema requires one.""" + + @staticmethod + def _item(module_bays, kind="device-type"): + return ExportItem( + kind=kind, + nb_record=_make_dt(), + repo_yaml=None, + serialized={"model": "7750-SR-7s", "module-bays": module_bays}, + reason="absent", + mfr_name="Nokia", + filename="nokia-7750-sr-7s.yaml", + manifest_key="Nokia/nokia-7750-sr-7s", + ) + + @staticmethod + def _write(tmp_path, item): + """Drive the real write path with a real LogHandler, which prints to stdout.""" + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + exporter._get_module_image_details = lambda: {} + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): + item = self._item([{"name": "Slot 0"}, {"name": "Slot 1", "position": "1"}]) + + self._write(tmp_path, item) + + out = capsys.readouterr().out + assert "Slot 0" in out + assert "position" in out + assert "nokia-7750-sr-7s.yaml" in out + assert "Slot 1" not in out, "a bay that has a position is not a problem" + + def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): + """'0' is a real position: the MX304 PSU bays use '0' and '1'.""" + item = self._item([{"name": "Slot 0", "position": "0"}]) + + self._write(tmp_path, item) + + assert "no position" not in capsys.readouterr().out + + def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + item = replace(self._item([]), serialized={"model": "7750-SR-7s"}) + + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + assert "no position" not in capsys.readouterr().out + + def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): + """Device types and module types share the module-bay schema definition.""" + item = self._item([{"name": "Sub 0"}], kind="module-type") + + self._write(tmp_path, item) + + assert "Sub 0" in capsys.readouterr().out From c20153d772c24e8a6d9e5c6629b25284ab00f55c Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:10:26 +0200 Subject: [PATCH 07/39] refactor: drop version branches unreachable at the NetBox 4.3 floor (#135) Closes #133. The importer refuses anything below NetBox 4.3, so three of the five feature flags can no longer be false on a server it will talk to: modules is a 3.2 threshold, new_filters and rack_types are 4.1 thresholds. The branches behind them were still carried, threaded through three constructors and asserted on. Removed the three flags and kept the true side of every branch they guarded. m2m_front_ports and module_bay_types still vary across supported releases and are untouched. core/compat.py keeps its version helpers, which #132 added and both entry points use; only the four filter-key helpers go, because the devicetype_id and moduletype_id spelling they chose between is unreachable at 4.1 and later. The issue predates those helpers and called for deleting the module outright. Tests that asserted the removed behaviour are deleted rather than adapted: a test that pins the legacy filter name or the rack-types skip is asserting what no supported deployment executes. Four filter-key tests collapse into two that say the supported key is the only one ever sent, which still catches the failure that mattered, since the wrong key does not raise but silently returns every row. --- core/compat.py | 58 +---------------- core/component_cache.py | 20 ++---- core/import_run.py | 93 +++++++++++---------------- core/netbox_api.py | 28 +------- core/update_failure_resolver.py | 24 ++----- tests/test_change_detector.py | 2 +- tests/test_component_cache.py | 9 --- tests/test_import_run.py | 7 +- tests/test_module_bay_type_sync.py | 1 - tests/test_nb_dt_import.py | 33 ++-------- tests/test_netbox_api.py | 70 ++++---------------- tests/test_update_failure_resolver.py | 53 ++------------- 12 files changed, 83 insertions(+), 315 deletions(-) diff --git a/core/compat.py b/core/compat.py index 97c59bf5..8ebcbf03 100644 --- a/core/compat.py +++ b/core/compat.py @@ -1,15 +1,7 @@ """NetBox version-compatibility helpers. -Centralises filter parameter names that changed between NetBox releases so -that every caller uses the same logic and drift is impossible. - -NetBox 4.1 renamed several filter keys on the DCIM endpoints: - - devicetype_id → device_type_id - moduletype_id → module_type_id - -Any code that constructs endpoint filter kwargs should call the helpers -here rather than inlining ``"device_type_id" if new_filters else "devicetype_id"``. +Centralises the version line each feature sits behind so that every caller uses the +same logic and drift is impossible. """ from __future__ import annotations @@ -33,49 +25,3 @@ def parse_netbox_version(version) -> tuple[int, int]: def supports_module_bay_types(version) -> bool: """Return True when this NetBox release exposes the module bay type relation.""" return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION - - -def device_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for device-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"device_type_id"``); - ``False`` for older releases (returns ``"devicetype_id"``). - """ - return "device_type_id" if new_filters else "devicetype_id" - - -def module_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for module-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"module_type_id"``); - ``False`` for older releases (returns ``"moduletype_id"``). - """ - return "module_type_id" if new_filters else "moduletype_id" - - -def device_type_filter_kwargs(device_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a device type. - - Args: - device_type_id: NetBox ID of the device type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {device_type_filter_key(new_filters): device_type_id} - - -def module_type_filter_kwargs(module_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a module type. - - Args: - module_type_id: NetBox ID of the module type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {module_type_filter_key(new_filters): module_type_id} diff --git a/core/component_cache.py b/core/component_cache.py index d8dc4a22..f314ecf7 100644 --- a/core/component_cache.py +++ b/core/component_cache.py @@ -14,12 +14,6 @@ import threading from typing import Any -from core.compat import ( - device_type_filter_key, - device_type_filter_kwargs, - module_type_filter_key, - module_type_filter_kwargs, -) from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError @@ -118,14 +112,13 @@ class ComponentCache: filter, which happens for types created during this run and after an invalidation. """ - def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_record=None): + def __init__(self, netbox, graphql, handle, max_threads, wrap_record=None): """Build a cache over the *netbox* REST client and the *graphql* client. Args: netbox: pynetbox API object, used for REST fallbacks and count checks. graphql: GraphQL client used for the bulk fetch. handle: Log handler. - new_filters (bool): Whether this NetBox takes the newer filter parameter names. max_threads (int): Upper bound on concurrent endpoint fetches. wrap_record (callable | None): Applied to every front-port record, so change detection sees one mappings shape across NetBox versions. @@ -133,7 +126,6 @@ def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_recor self.netbox = netbox self.graphql = graphql self.handle = handle - self.new_filters = new_filters self.max_threads = max_threads self._wrap_record = wrap_record or (lambda record: record) @@ -313,9 +305,9 @@ def get(self, endpoint_name, parent_type, parent_id, endpoint): return cached[key] if parent_type == "device": - filter_kwargs = device_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"device_type_id": parent_id} else: - filter_kwargs = module_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"module_type_id": parent_id} result = {item.name: item for item in endpoint.filter(**filter_kwargs)} self.record(endpoint_name, parent_type, parent_id, result) return result @@ -479,8 +471,6 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc Raises: GraphQLCountMismatchError: When an endpoint holds fewer records than REST reports. """ - dt_filter_key = device_type_filter_key(self.new_filters) - mt_filter_key = module_type_filter_key(self.new_filters) dt_ids = list(device_type_ids) mt_ids = list(module_type_ids) @@ -495,9 +485,9 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc rest_endpoint = getattr(self.netbox.dcim, endpoint_name) rest_count = 0 if dt_ids: - rest_count += self._rest_count(rest_endpoint, dt_filter_key, dt_ids) + rest_count += self._rest_count(rest_endpoint, "device_type_id", dt_ids) if mt_ids and component.module_types: - rest_count += self._rest_count(rest_endpoint, mt_filter_key, mt_ids) + rest_count += self._rest_count(rest_endpoint, "module_type_id", mt_ids) if cached_count != rest_count: if vendor_scope_valid: diff --git a/core/import_run.py b/core/import_run.py index f81fd7bf..cdac353b 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -49,8 +49,6 @@ class RunSummary: """Snapshot of the result of one completed import run.""" counter: Counter - modules: bool - rack_types: bool outcome_counts: dict failure_lines: tuple duplicate_definitions: tuple @@ -66,8 +64,6 @@ def capture(cls, netbox, repo, started_at): """ return cls( counter=Counter(netbox.counter), - modules=netbox.modules, - rack_types=netbox.rack_types, outcome_counts=netbox.outcomes.summary_by_kind(), failure_lines=tuple(netbox.outcomes.render_failure_report()), duplicate_definitions=tuple(repo.duplicate_definitions), @@ -455,10 +451,6 @@ def _process_rack_types(config, netbox, handle, progress, rack_types, vendor_nam if not rack_types: return - if not netbox.rack_types: - handle.log("Rack types require NetBox >= 4.1. Skipping rack type import.") - return - handle.verbose_log(f"{len(rack_types)} Rack-Types Found") all_rack_types = netbox.get_existing_rack_types() @@ -521,21 +513,19 @@ def _log_run_summary(handle, summary): handle.log(f"{counter['components_removed']} components removed") handle.verbose_log(f"{counter['images']} images uploaded") handle.log(f"{counter['manufacturer']} manufacturers created") - if summary.modules: - handle.log(f"{counter['module_added']} modules created") - handle.log(f"{counter['module_updated']} modules updated") - module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) - if module_failed: - handle.log(f"{module_failed} modules failed to create or update") - module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) - if module_partial: - handle.log(f"{module_partial} modules partially updated") - if summary.rack_types: - handle.log(f"{counter['rack_type_added']} rack types created") - handle.log(f"{counter['rack_type_updated']} rack types updated") - rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) - if rack_failed: - handle.log(f"{rack_failed} rack types failed") + handle.log(f"{counter['module_added']} modules created") + handle.log(f"{counter['module_updated']} modules updated") + module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) + if module_failed: + handle.log(f"{module_failed} modules failed to create or update") + module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) + if module_partial: + handle.log(f"{module_partial} modules partially updated") + handle.log(f"{counter['rack_type_added']} rack types created") + handle.log(f"{counter['rack_type_updated']} rack types updated") + rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) + if rack_failed: + handle.log(f"{rack_failed} rack types failed") for line in summary.failure_lines: handle.log(line) @@ -673,30 +663,24 @@ def plan_vendor(self, selection, vendor): self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] ) - if self.netbox.modules: - module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None - if module_hint is not None and vendor["slug"] not in module_hint: - module_types = [] - else: - module_types = _parse_vendor_files( - self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] - ) - else: + module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None + if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] - - if self.netbox.rack_types: - rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None - if rack_hint is not None and vendor["slug"] not in rack_hint: - rack_types = [] - else: - rack_types = _parse_vendor_files( - self.repo, - selection.racks_path, - vendor["name"], - self.config.slugs or [], - ) else: + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) + + rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None + if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] + else: + rack_types = _parse_vendor_files( + self.repo, + selection.racks_path, + vendor["name"], + self.config.slugs or [], + ) return VendorPlan( vendor=vendor, @@ -738,17 +722,16 @@ def apply(self, plan): ) cache.pump() - if self.netbox.modules: - _process_module_types( - self.config, - self.netbox, - self.reporter, - self.progress, - plan.module_types, - vendor_name=plan.vendor["name"], - task_registry=self.task_registry, - ) - cache.pump() + _process_module_types( + self.config, + self.netbox, + self.reporter, + self.progress, + plan.module_types, + vendor_name=plan.vendor["name"], + task_registry=self.task_registry, + ) + cache.pump() _process_rack_types( self.config, diff --git a/core/netbox_api.py b/core/netbox_api.py index 979e704d..7bdacf92 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -492,11 +492,8 @@ def __init__(self, config, handle): self.handle = handle self.netbox: Any = None self.ignore_ssl = config.ignore_ssl_errors - self.modules = False - self.new_filters = False self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) - self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts self.remove_unmanaged_types = config.remove_unmanaged_types self.verify_images = config.verify_images @@ -538,7 +535,6 @@ def __init__(self, config, handle): self.handle, self.counter, self.ignore_ssl, - self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, module_bay_types_supported=self.module_bay_types, @@ -603,10 +599,10 @@ def connect_api(self): raise UnknownError("NetBox API Error", cause=e) from e def verify_compatibility(self): - """Check the connected NetBox version and configure feature flags accordingly. + """Refuse a server below the supported floor and set the feature flags above it. - Sets ``self.modules = True`` for NetBox >= 3.2 and ``self.new_filters = True`` - for >= 4.1. Logs the detected version when the new-filter flag is enabled. + Only the flags that still vary across supported releases are set here: the 4.3 + floor makes everything introduced at or below 4.1 unconditional. """ # nb.version should be the version in the form '3.2' # Strip non-numeric suffixes (e.g. "4.1-beta") before converting to int. @@ -644,17 +640,6 @@ def verify_compatibility(self): f"Older releases fail part way through with a GraphQL schema error rather than here." ) - # Later than 3.2 - # Might want to check for the module-types entry as well? - if version_split[0] > 3 or (version_split[0] == 3 and version_split[1] >= 2): - self.modules = True - - # check if version >= 4.1 in order to use new filter names (https://github.com/netbox-community/netbox/issues/15410) - if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 1): - self.new_filters = True - self.rack_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using new filters.") - # NetBox 4.5 replaced FrontPortTemplate.rear_port (FK) + rear_port_position (int) # with a ManyToMany through table (PortMapping). The creation and read APIs differ. # https://github.com/netbox-community/netbox/issues/20564 @@ -777,7 +762,6 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er netbox=self.netbox, device_type_id=dt.id, device_type_yaml=device_type, - new_filters=self.new_filters, ) except Exception as exc: # defensive: classifier must never break the run self.handle.verbose_log(f"Failure classifier raised {type(exc).__name__}: {exc}") @@ -1204,8 +1188,6 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima yaml_key = component.yaml_key if yaml_key not in device_type: continue - if yaml_key == "module-bays" and not self.modules: - continue self.device_types.create_components( yaml_key, device_type[yaml_key], @@ -2320,7 +2302,6 @@ def __init__( handle, counter, ignore_ssl, - new_filters, *, graphql, repo_path, @@ -2338,7 +2319,6 @@ def __init__( handle (LogHandler): Sink for creation and error messages. counter (Counter): Shared operation counter updated during creation. ignore_ssl (bool): Whether SSL certificate verification is disabled. - new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. @@ -2349,7 +2329,6 @@ def __init__( self.handle = handle self.counter = counter self.ignore_ssl = ignore_ssl - self.new_filters = new_filters self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports @@ -2359,7 +2338,6 @@ def __init__( netbox, graphql, handle, - new_filters, max_threads, wrap_record=_FrontPortRecordWithMappings, ) diff --git a/core/update_failure_resolver.py b/core/update_failure_resolver.py index 7e2eb1f6..82543a5d 100644 --- a/core/update_failure_resolver.py +++ b/core/update_failure_resolver.py @@ -24,8 +24,6 @@ from enum import Enum from typing import Any, Callable, List, Optional -from core.compat import device_type_filter_kwargs - class FailureKind(str, Enum): """High-level classification of a NetBox update failure.""" @@ -122,7 +120,7 @@ def _matches_subdevice_role_constraint(payload: Any) -> bool: return False -def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> tuple[int, List[str]]: +def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, List[str]]: """Query NetBox for devices using *device_type_id*. Returns ``(count, sample_names)`` where ``sample_names`` is up to 5 names @@ -133,10 +131,8 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ - filter_kwargs = device_type_filter_kwargs(device_type_id, new_filters=new_filters) + filter_kwargs = {"device_type_id": device_type_id} try: devices = list(netbox.dcim.devices.filter(**filter_kwargs, limit=5)) except Exception: @@ -152,7 +148,7 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b return total, sample -def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> Optional[List[Any]]: +def _list_device_bay_templates(netbox: Any, device_type_id: int) -> Optional[List[Any]]: """Return all ``DeviceBayTemplate`` records attached to *device_type_id*. Returns ``None`` when the NetBox query itself fails (network error, 5xx, etc.) @@ -161,15 +157,9 @@ def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ try: - return list( - netbox.dcim.device_bay_templates.filter( - **device_type_filter_kwargs(device_type_id, new_filters=new_filters) - ) - ) + return list(netbox.dcim.device_bay_templates.filter(device_type_id=device_type_id)) except Exception: return None @@ -180,7 +170,6 @@ def classify_device_type_update_failure( netbox: Any, device_type_id: int, device_type_yaml: dict, - new_filters: bool = False, ) -> FailureResolution: """Classify a ``pynetbox.RequestError`` raised while updating a device type. @@ -192,7 +181,6 @@ def classify_device_type_update_failure( device_type_yaml: Parsed YAML dict for this device-type (used to detect whether the YAML *also* lists device bays — in which case we cannot blindly delete them). - new_filters: When True, use updated filter parameter names (NetBox ≥ 4.1). Returns: A :class:`FailureResolution` describing the constraint and (when safe) @@ -207,7 +195,7 @@ def classify_device_type_update_failure( ) # SUBDEVICE_ROLE_FLIP path ------------------------------------------------- - blocking_templates = _list_device_bay_templates(netbox, device_type_id, new_filters=new_filters) + blocking_templates = _list_device_bay_templates(netbox, device_type_id) if blocking_templates is None: return FailureResolution( kind=FailureKind.MANUAL_REQUIRED, @@ -216,7 +204,7 @@ def classify_device_type_update_failure( ) blocking_names = [getattr(t, "name", str(getattr(t, "id", "?"))) for t in blocking_templates] - dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id, new_filters=new_filters) + dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id) # YAML must NOT redefine device-bays — otherwise deleting them would just # cause our own component-creation step to fail or thrash. This catches diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index 3a9b6434..bf9d6a92 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -19,7 +19,7 @@ def _cache(**records): Populating through the real object means these tests read the same index the importer builds, rather than a dict that merely looks like it. """ - cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), new_filters=True, max_threads=1) + cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), max_threads=1) for endpoint_name, items in records.items(): cache.populate(endpoint_name, items) return cache diff --git a/tests/test_component_cache.py b/tests/test_component_cache.py index a6d9e330..a915936f 100644 --- a/tests/test_component_cache.py +++ b/tests/test_component_cache.py @@ -187,7 +187,6 @@ def make_cache(netbox=None, graphql=None, handle=None, **kwargs): netbox or FakeNetBox(), graphql or FakeGraphQL(), handle or FakeHandle(), - kwargs.pop("new_filters", True), kwargs.pop("max_threads", 4), **kwargs, ) @@ -279,14 +278,6 @@ def test_a_miss_filters_by_module_type_for_a_module_parent(self): assert endpoint.filter_calls == [{"module_type_id": 5}] - def test_old_netbox_filter_names_are_used_when_asked(self): - cache = make_cache(new_filters=False) - endpoint = FakeEndpoint() - - cache.get("interface_templates", "device", 1, endpoint) - - assert endpoint.filter_calls == [{"devicetype_id": 1}] - def test_an_empty_result_still_becomes_a_hit(self): """Otherwise every parent with no components is re-read on each lookup.""" cache = make_cache() diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 1aa14152..9c9d5694 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -59,7 +59,6 @@ class _NetBoxBoundary: """Expose run state and fail if planning starts an import.""" def __init__(self): - self.modules = True self.rack_types = True self.device_types = _DeviceTypes() self.outcomes = _Outcomes() @@ -271,8 +270,6 @@ def test_execute_returns_snapshot_and_owns_console_lifecycle(make_config, tmp_pa assert isinstance(summary, RunSummary) assert summary.counter["added"] == 2 - assert summary.modules is True - assert summary.rack_types is True assert progress_factory.entered is True assert progress_factory.exited is True assert handle.console is None @@ -434,11 +431,9 @@ def test_banners_omit_the_separator_when_no_vendor_is_given(make_config): class _OutcomeNetBox: """Run-state stand-in carrying a real OutcomeRegistry.""" - def __init__(self, *, modules=True, rack_types=True, counter=None): + def __init__(self, *, counter=None): from core.outcomes import OutcomeRegistry - self.modules = modules - self.rack_types = rack_types self.outcomes = OutcomeRegistry() base = dict( added=0, diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index ed2a27e1..6030b764 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -84,7 +84,6 @@ def _make(module_bay_types_supported=True): handle, {}, False, - True, graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), repo_path=str(catalog_root), module_bay_types_supported=module_bay_types_supported, diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index a358c6ca..fea59e41 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -335,13 +335,11 @@ def _make_mock_repo(device_types=None): return mock_repo -def _make_mock_netbox(modules=False, rack_types=False): +def _make_mock_netbox(): """Return a pre-configured NetBox mock.""" from collections import Counter mock_nb = MagicMock() - mock_nb.modules = modules - mock_nb.rack_types = rack_types mock_nb.device_types.existing_device_types = {} mock_nb.device_types.existing_device_types_by_slug = {} mock_nb.count_device_type_images.return_value = 0 @@ -884,7 +882,7 @@ def test_modules_with_types_to_process(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([module_type], {}, []) MockNetBox.return_value = mock_nb MockNetBox.filter_new_module_types.return_value = [] @@ -912,7 +910,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([], {}, change_log) mock_nb.filter_new_module_types.return_value = [] MockNetBox.return_value = mock_nb @@ -930,7 +928,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): mock_nb.log_module_type_changes.assert_called_once_with(change_log) def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): - """When netbox.modules is True, module_added/updated counters are logged.""" + """Module counters are always logged: every supported release has module types.""" with ( patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), patch("nb_dt_import.DTLRepo") as MockRepo, @@ -938,7 +936,7 @@ def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): patch("nb_dt_import.LogHandler") as MockLogHandler, ): MockRepo.return_value = _make_mock_repo() - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() MockNetBox.return_value = mock_nb nb_dt_import.main() @@ -987,19 +985,6 @@ class TestProcessRackTypes: def _make_args(self, only_new=False): return SimpleNamespace(only_new=only_new) - def test_rack_types_disabled_logs_warning_and_returns(self, nb_dt_import): - """netbox.rack_types=False with actual rack types: warning logged, no further processing.""" - handle = MagicMock() - netbox = MagicMock() - netbox.rack_types = False - - rack_type = {"manufacturer": {"slug": "apc"}, "model": "AR1300", "slug": "apc-ar1300"} - import_run_module._process_rack_types(self._make_args(), netbox, handle, None, [rack_type]) - - handle.log.assert_called_once() - assert "4.1" in handle.log.call_args[0][0] - netbox.get_existing_rack_types.assert_not_called() - def test_empty_rack_types_returns_early(self, nb_dt_import): """rack_types=[]: returns immediately without any logging or API calls.""" handle = MagicMock() @@ -1252,7 +1237,7 @@ def test_module_type_only_vendor_uses_scoped_preload(self, nb_dt_import): """ mt = {"manufacturer": {"slug": "acbel"}, "model": "M1", "slug": "acbel-m1"} - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Acbel", "slug": "acbel"}] @@ -1371,8 +1356,6 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = True from collections import Counter mock_nb.counter = Counter( @@ -1404,8 +1387,6 @@ def test_duplicate_definitions_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = False from collections import Counter mock_nb.counter = Counter( @@ -1988,7 +1969,7 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c if files == ["cisco-module-types.yaml"] else [] ) - netbox = _make_mock_netbox(modules=True) + netbox = _make_mock_netbox() slug_resolved = { "device_files": {"empty": [], "cisco": ["resolved.yaml"]}, "module_vendors": {"cisco"}, diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index eb535191..68549282 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -80,7 +80,6 @@ def _factory(nb_api=None, handle=None, counter=None, **kwargs): handle if handle is not None else mock_handle, counter if counter is not None else MagicMock(), False, - False, graphql=kwargs.pop("graphql", graphql_client), repo_path=kwargs.pop("repo_path", mock_settings.repo_path), **kwargs, @@ -118,7 +117,6 @@ def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): assert nb.url == "http://mock-netbox" assert nb.token == "mock-token" # Verify module support detection - assert nb.modules def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, mock_handle): @@ -142,7 +140,6 @@ def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): mock_pynetbox.api.return_value.version = version nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters, version assert nb.module_bay_types is module_bay_types, version @@ -2479,7 +2476,6 @@ def test_creates_all_component_types( nb = NetBox(mock_settings, mock_handle) nb.device_types = dt - nb.modules = True created_dt = MagicMock() created_dt.id = 1 @@ -4174,40 +4170,6 @@ def test_image_file_not_found_logs_error( nb.create_device_types([device_type]) assert any("Error locating image file" in str(c) for c in mock_handle.log.call_args_list) - def test_module_bays_not_created_when_modules_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle - ): - """module-bays are only created when self.modules is True.""" - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - dt.existing_device_types = {} - dt.existing_device_types_by_slug = {} - dt.components.record("module_bay_templates", "device", 1, {}) - - nb = NetBox(mock_settings, mock_handle) - nb.device_types = dt - nb.modules = False # explicitly disabled - - created_dt = MagicMock() - created_dt.id = 1 - created_dt.manufacturer.name = "Cisco" - created_dt.model = "TestSwitch" - mock_nb_api.dcim.device_types.create.return_value = created_dt - - device_type = { - "manufacturer": {"slug": "cisco"}, - "model": "TestSwitch", - "slug": "testswitch", - "module-bays": [{"name": "MB1"}], - "src": "/tmp/device-types/cisco/testswitch.yaml", - } - nb.create_device_types([device_type]) - mock_nb_api.dcim.module_bay_templates.create.assert_not_called() - - -class TestCreateModuleTypesCornerCases: - """Corner-case tests for create_module_types (cognitive complexity 16).""" - def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" mock_pynetbox.api.return_value.version = "4.3" @@ -4605,49 +4567,45 @@ class TestVerifyCompatibility: """Tests for NetBox.verify_compatibility() version thresholds.""" @pytest.mark.parametrize( - "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", + "version_str, expected_m2m, expected_module_bay_types", [ - ("4.3", True, True, True, False), - ("4.4", True, True, True, False), - ("4.5", True, True, True, True), - ("4.6", True, True, True, True), - ("4.7", True, True, True, True), - ("5.0", True, True, True, True), + ("4.3", False, False), + ("4.4", False, False), + ("4.5", True, False), + ("4.6", True, False), + ("4.7", True, True), + ("5.0", True, True), # Version strings with non-numeric suffixes - ("4.5-beta", True, True, True, True), - ("4.3.0", True, True, True, False), + ("4.5-beta", True, False), + ("4.3.0", False, False), ], ) def test_version_thresholds( self, version_str, - expected_modules, - expected_new_filters, - expected_rack_types, expected_m2m, + expected_module_bay_types, mock_settings, mock_pynetbox, mock_handle, ): + """Only the flags that still vary above the 4.3 floor are set.""" mock_pynetbox.api.return_value.version = version_str nb = NetBox(mock_settings, mock_handle) - assert nb.modules == expected_modules, f"modules mismatch for {version_str}" - assert nb.new_filters == expected_new_filters, f"new_filters mismatch for {version_str}" - assert nb.rack_types == expected_rack_types, f"rack_types mismatch for {version_str}" assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" + assert nb.module_bay_types == expected_module_bay_types, f"module_bay_types mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): """A version string with only a major component (e.g. '5') does not crash.""" mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True + assert nb.m2m_front_ports is True def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( self, mock_settings, mock_pynetbox, mock_handle ): mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True assert nb.m2m_front_ports is False assert nb.module_bay_types is False @@ -6808,7 +6766,6 @@ def _device_types(url, handle): handle, MagicMock(), False, - False, graphql=NetBoxGraphQLClient(url, "token", page_size=10), repo_path="/tmp/repo", max_threads=2, @@ -7222,7 +7179,6 @@ def test_module_type_create_failure_is_not_called_an_update_failure( b'{"model":["This field may not be blank."]}' ) nb = NetBox(mock_settings, mock_handle) - nb.modules = True nb._process_single_module_type( {"manufacturer": {"slug": "panduit"}, "model": "FAP6WBUSC", "slug": "fap6wbusc"}, "/repo/module-types/Panduit/FAP6WBUSC.yaml", diff --git a/tests/test_update_failure_resolver.py b/tests/test_update_failure_resolver.py index a6135626..93a4bf2f 100644 --- a/tests/test_update_failure_resolver.py +++ b/tests/test_update_failure_resolver.py @@ -314,57 +314,18 @@ def test_classifier_count_fallback_when_count_query_fails(): assert res.dependent_devices_count == 5 -def test_new_filters_uses_device_type_id_key(): - """new_filters=True must call filter(device_type_id=...) not devicetype_id=... +def test_the_device_bay_template_lookup_uses_the_supported_filter_key(): + """The 4.1 rename is below the supported floor, so only device_type_id is ever sent. - This matters because NetBox >= 4.1 changed the query param name. - Passing the wrong key causes pynetbox to silently return ALL templates. + The wrong key does not raise: pynetbox silently returns every template, so the + classifier would report an unrelated device type as the blocker. """ nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=99, device_type_yaml={}) nb.dcim.device_bay_templates.filter.assert_called_once_with(device_type_id=99) -def test_old_filters_uses_devicetype_id_key(): - """new_filters=False (default) must call filter(devicetype_id=...) for NetBox < 4.1.""" - nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.device_bay_templates.filter.assert_called_once_with(devicetype_id=99) - - -def test_count_dependent_devices_uses_new_filter_key(): - """When new_filters=True, dcim.devices must be queried with device_type_id= not devicetype_id=.""" +def test_the_dependent_device_count_uses_the_supported_filter_key(): nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=77, device_type_yaml={}) nb.dcim.devices.filter.assert_called_once_with(device_type_id=77, limit=5) - - -def test_count_dependent_devices_uses_legacy_filter_key(): - """When new_filters=False (default), dcim.devices must be queried with devicetype_id=.""" - nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.devices.filter.assert_called_once_with(devicetype_id=77, limit=5) From 7b3c9e5b673fad5017ee28aea0c937393b9b408e Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 14:07:20 +0200 Subject: [PATCH 08/39] fix(export): treat an omitted front-port positions as the default The export now always writes positions, because the library schema requires it. _is_subset needs every NetBox leaf to be present in the repo YAML, so a library entry that omits positions no longer matched the serialized positions: 1 and the whole definition read as differing. On a library that omits the field, an export would rewrite every such file. Fill an absent front-port positions with 1 on both sides before comparing. The default is what an omitted value means, so this restores the old comparison without giving up the field the schema requires. A non-default value still differs: the merge puts the default first so an explicit value wins. Rejected the alternative of adding positions to _OMIT_IF_EQUAL: that would drop the key from the output again and reintroduce the schema violation fixed in 43480e7. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3967814233 --- core/export.py | 13 +++++++++++-- tests/test_exporter.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/core/export.py b/core/export.py index b14ebc7b..b01ede70 100644 --- a/core/export.py +++ b/core/export.py @@ -162,8 +162,17 @@ def _norm_mfr(d: dict) -> dict: return d return {**d, "manufacturer": _canon_mfr_slug(d["manufacturer"])} - nrepo = _normalize_for_compare(_norm_mfr(repo_yaml)) - nnb = _normalize_for_compare(_norm_mfr(nb_serialized)) + # The serializer always writes the schema-required positions; an entry that omits it + # means the default, so filling it here keeps those definitions from reading as differing. + def _default_positions(d: dict) -> dict: + ports = d.get("front-ports") + if not isinstance(ports, list): + return d + filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] + return {**d, "front-ports": filled} + + nrepo = _normalize_for_compare(_default_positions(_norm_mfr(repo_yaml))) + nnb = _normalize_for_compare(_default_positions(_norm_mfr(nb_serialized))) return _is_subset(nnb, nrepo) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 343d9c20..e5184fb5 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -259,6 +259,32 @@ def test_an_empty_relation_does_not_make_every_definition_differ(self): assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + def test_a_default_positions_does_not_make_every_front_port_differ(self): + """The export writes the schema-required positions; it must not re-export the library. + + A library entry that omits positions means the default, 1. _is_subset requires every + NetBox leaf to be present in the repo YAML, so a serialized positions: 1 compared + against an entry that omits it would report every such definition as differing. + """ + from types import SimpleNamespace + from core.nb_serializer import _serialize_front_port + + legacy = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + serialized = _serialize_front_port(legacy) + assert serialized == {"name": "FP1", "type": "8p8c", "positions": 1} + + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [serialized]} + + assert _repo_supersedes(repo, nb), "an omitted positions is the default, not a difference" + + def test_a_non_default_positions_still_differs(self): + """Only the default may be treated as absent, or a real change would be suppressed.""" + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c", "positions": 4}]} + + assert _repo_supersedes(repo, nb) is False + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} From ac319ceaa9f39112f727c692b1bba4b799421a15 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 14:08:36 +0200 Subject: [PATCH 09/39] fix(config): report a NETBOX_URL that sends the token in cleartext The API token travels in an Authorization header on every request, including the /api/status/ probe added for module bay type detection. An http:// NETBOX_URL pointing off this machine puts that token on the wire in cleartext (CWE-319). Add a startup notice through the existing notices channel, so it covers the importer and the export alike from the one place NETBOX_URL is resolved. Warn rather than reject. Plain HTTP to NetBox on a trusted network is a common deployment, the tool already carries IGNORE_SSL_ERRORS for imperfect transport, and a hard failure would break those users with no upgrade path. Loopback is exempt: it never leaves the host, and the integration tests run against it. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3967814227 --- core/config.py | 24 +++++++++++++++++++++++- tests/test_config.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/core/config.py b/core/config.py index d37778e6..fe7fa307 100644 --- a/core/config.py +++ b/core/config.py @@ -1,9 +1,11 @@ """Resolution of one run's configuration from the command line and the environment.""" +import ipaddress import os import re from argparse import ArgumentParser from dataclasses import dataclass, field +from urllib.parse import urlparse from dotenv import load_dotenv @@ -22,6 +24,20 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def _sends_token_in_cleartext(url): + """Return True when *url* would send the API token over plain HTTP off this host.""" + parsed = urlparse(str(url or "").strip()) + if parsed.scheme != "http": + return False + host = (parsed.hostname or "").casefold() + if host in {"", "localhost"}: + return False + try: + return not ipaddress.ip_address(host).is_loopback + except ValueError: + return True + + def is_local_repo_url(url): """Return True when *url* is the sentinel that turns off every git operation.""" return str(url or "").strip().casefold() == LOCAL_REPO_URL @@ -271,6 +287,12 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + netbox_url = _text(env, "NETBOX_URL") + if _sends_token_in_cleartext(netbox_url): + notices.append( + "NETBOX_URL uses http:// on a remote host, so the API token is sent in cleartext. " + "Use https:// unless NetBox is on this machine." + ) if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: notices.append( f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " @@ -278,7 +300,7 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: ) return RunConfig( - netbox_url=_text(env, "NETBOX_URL"), + netbox_url=netbox_url, netbox_token=_text(env, "NETBOX_TOKEN"), ignore_ssl_errors=(_text(env, "IGNORE_SSL_ERRORS", "False") or "False").casefold() in {"true", "1", "yes"}, graphql_page_size=_positive_int(env, "GRAPHQL_PAGE_SIZE", DEFAULT_GRAPHQL_PAGE_SIZE), diff --git a/tests/test_config.py b/tests/test_config.py index 1aae1838..d9db0907 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -139,3 +139,39 @@ def test_a_branch_set_beside_a_real_url_needs_no_notice(self): config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + +class TestInsecureNetboxUrlIsReported: + """The API token travels in an Authorization header, so cleartext transport leaks it.""" + + @pytest.mark.parametrize( + "url", + [ + "http://netbox.example.com", + "http://netbox.example.com:8000/", + "http://10.0.0.5:8000", + ], + ) + def test_http_to_a_remote_host_is_reported(self, url): + config = _resolve(NETBOX_URL=url) + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + @pytest.mark.parametrize( + "url", + [ + "http://localhost:8000", + "http://127.0.0.1:8000/", + "http://[::1]:8000", + ], + ) + def test_http_to_loopback_needs_no_notice(self, url): + """A loopback URL never leaves the host, and the integration tests rely on it.""" + config = _resolve(NETBOX_URL=url) + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_https_needs_no_notice(self): + config = _resolve(NETBOX_URL="https://netbox.example.com") + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices From 2f0bd8016ed7b0d28a024cf235f4227097134894 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 18:12:05 +0200 Subject: [PATCH 10/39] fix(module-bay-types): separate a catalog load failure from a name miss ModuleBayTypeError carried two failures of different severity. A name that does not resolve is per definition, and every caller recovers from it to report that one definition and continue. An unreadable, malformed or duplicated catalog file is terminal: nothing in the catalog can be trusted. Because both raised the same type, those per-definition handlers caught the terminal one too. A single broken catalog file was reported as one skipped component after another, and _load_catalog's own docstring already claimed the failure "is terminal for the run" while the code made it anything but. ModuleBayCatalogError is a sibling of ModuleBayTypeError under FatalError, never a subclass, so the existing `except ModuleBayTypeError` handlers let it through and the run ends once with the real reason. The three load failures in _read_catalog raise it; per-name lookups and per-definition NetBox rejections keep ModuleBayTypeError. The four catalog-reading tests that asserted the old shared type now assert the new one, which is the behaviour change this makes. Also reuse the already-fetched nb_version in the two version log lines. Each pynetbox `version` access is another HTTP request, and both of these sat outside the try that maps transport failures to NetBoxError, so a connection drop there escaped unmapped. Review findings: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3970510857 https://github.com/marcinpsk/Device-Type-Library-Import/pull/136 (review body, core/netbox_api.py:652) --- core/module_bay_types.py | 19 ++++++++--- core/netbox_api.py | 4 +-- tests/test_module_bay_types.py | 62 +++++++++++++++++++++++++++++++--- tests/test_netbox_api.py | 15 ++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 68a23fbc..308c999e 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -46,6 +46,15 @@ class ModuleBayTypeError(FatalError): """ +class ModuleBayCatalogError(FatalError): + """The catalog itself could not be read, so no name in it can be trusted. + + A sibling of :class:`ModuleBayTypeError`, never a subclass: every caller recovers from + that one per definition, which would turn a single unreadable catalog into one skipped + component after another instead of ending the run once. + """ + + class ModuleBayTypeCatalog: """Turn module-bay-type names into NetBox ids, creating what is missing. @@ -137,7 +146,7 @@ def _load_catalog(self): raise self._load_error try: self._entries = self._read_catalog() - except ModuleBayTypeError as exc: + except ModuleBayCatalogError as exc: self._load_error = exc raise return self._entries @@ -154,7 +163,9 @@ def _read_catalog(self): with open(path, encoding="utf-8") as handle: data = yaml.safe_load(handle) except (OSError, yaml.YAMLError) as exc: - raise ModuleBayTypeError(f"Module bay type catalog file {path!r} could not be read: {exc}") from exc + raise ModuleBayCatalogError( + f"Module bay type catalog file {path!r} could not be read: {exc}" + ) from exc if not isinstance(data, dict): continue # Reject a half-written entry here; the readers index on name and dereference slug. @@ -164,12 +175,12 @@ def _read_catalog(self): if not isinstance(data.get(field), str) or not data[field].strip() ] if invalid: - raise ModuleBayTypeError( + raise ModuleBayCatalogError( f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" ) key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) if key in entries: - raise ModuleBayTypeError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + raise ModuleBayCatalogError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") entries[key] = data return entries diff --git a/core/netbox_api.py b/core/netbox_api.py index 7bdacf92..bf2ab197 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -645,11 +645,11 @@ def verify_compatibility(self): # https://github.com/netbox-community/netbox/issues/20564 if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 5): self.m2m_front_ports = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + self.handle.log(f"Netbox version {nb_version} found. Using M2M front/rear port mappings.") if supports_module_bay_types(nb_version): self.module_bay_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Module bay types are supported.") + self.handle.log(f"Netbox version {nb_version} found. Module bay types are supported.") def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index bdf1581d..7fa40c05 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -8,7 +8,7 @@ import pytest -from core.module_bay_types import ModuleBayTypeCatalog, ModuleBayTypeError +from core.module_bay_types import ModuleBayCatalogError, ModuleBayTypeCatalog, ModuleBayTypeError from helpers import FakeNetBox, write_module_bay_type @@ -172,7 +172,7 @@ def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, ca (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.identities_for("generic", ["SFP"]) assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) @@ -183,7 +183,7 @@ def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog) (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.identities_for("generic", ["SFP"]) assert "sfp.yaml" in str(exc.value) @@ -200,7 +200,7 @@ def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catal monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) for _ in range(3): - with pytest.raises(ModuleBayTypeError): + with pytest.raises(ModuleBayCatalogError): cat.identities_for("generic", ["SFP"]) assert len(walks) == 1 @@ -209,7 +209,7 @@ def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.ids_for("generic", ["SFP"]) assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) @@ -248,3 +248,55 @@ def test_duplicate_names_collapse(self, catalog): assert len(once) == 1 assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once + + +@pytest.mark.real_http +class TestACatalogFailureIsNotAPerComponentSkip: + """A broken catalog is terminal for the run, but every caller recovers per component.""" + + @staticmethod + def _broken_catalog(tmp_path, catalog): + root = tmp_path / "broken" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "fine") + # A half-written entry: the readers index on name and dereference slug. + (root / "module-bay-types" / "Juniper" / "bad.yaml").write_text( + "name: 123\nslug: bad\nmanufacturer: Juniper\n", encoding="utf-8" + ) + return catalog(root=root)[0] + + def test_a_malformed_entry_is_not_reported_as_a_relation_change(self, tmp_path, catalog): + """_relation_change recovers from ModuleBayTypeError, so a load failure must not be one.""" + from types import SimpleNamespace + + from core.change_detector import _relation_change + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + netbox_comp = SimpleNamespace(name="RE0", module_bay_types=[]) + + with pytest.raises(ModuleBayCatalogError): + _relation_change( + "module_bay_types", + {"name": "RE0", "module_bay_types": ["MX304-RE"]}, + netbox_comp, + catalog=broken, + manufacturer="Juniper", + ) + + def test_the_catalog_failure_is_not_a_module_bay_type_error(self, tmp_path, catalog): + """Sibling, not subclass: an `except ModuleBayTypeError` must not swallow it.""" + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + + with pytest.raises(ModuleBayCatalogError) as caught: + broken.identities_for("Juniper", ["MX304-RE"]) + + assert not isinstance(caught.value, ModuleBayTypeError), "per-name catches would swallow it" + + def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, catalog): + """The split must not promote a per-name miss into a run-ending failure.""" + resolver, _server = catalog() + + with pytest.raises(ModuleBayTypeError): + resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 68549282..895c2c62 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -143,6 +143,21 @@ def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): assert nb.module_bay_types is module_bay_types, version +def test_the_netbox_version_is_fetched_once(mock_settings, mock_pynetbox, mock_handle): + """Each pynetbox `version` access is another HTTP request, and only the first is error-mapped.""" + from unittest.mock import PropertyMock + + api_type = type(mock_pynetbox.api.return_value) + version = PropertyMock(return_value="4.7.0") + api_type.version = version + try: + NetBox(mock_settings, mock_handle) + + assert version.call_count == 1, "the log lines must reuse the version already fetched" + finally: + del api_type.version + + def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] From 0b12aca23142736a041ef550bd09d2557ee16111 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 10:10:33 +0200 Subject: [PATCH 11/39] fix: close the paths that resolve a short catalog to the wrong entry Six findings from review, five of them variations on the same shape: a boundary that accepts or skips something quietly, so the failure surfaces later as wrong data rather than as an error. - os.walk suppresses a directory it cannot read. A manufacturer directory that fails to open produced a short catalog, and a same-named Generic entry then answered in its place, writing the wrong module_bay_types relation. An onerror callback makes the traversal error terminal. - A catalog document that parses to a list or a scalar was skipped in the same silent way. Only an empty document is skipped now; any other non-mapping is refused. - The /api/status/ probe read a missing netbox-version as "", which answers "module bay types unsupported" for a 4.7 server and exports without the relation. A non-mapping body raised a bare AttributeError. Both are now GraphQLError. - _is_relation_list and _type_relation_changes accepted a whitespace-only name that the catalog rejects later, which skipped the whole component's update. Both strip before testing, matching the catalog. - The export manifest recorded only last_updated. An unchanged NetBox record keeps that timestamp forever, so an old manifest skipped exactly the types a new output shape was added for. Entries now carry EXPORT_SCHEMA_REVISION and an older revision is not fresh. - The cleartext-token notice parsed the URL with urlparse, which reads "http://10.0.0.1\@localhost" as loopback where requests targets 10.0.0.1. Backslashes are normalized before parsing. The GraphQL session fixture never configured session.get, so the status probe read a synthesized MagicMock and any shape check on the payload saw something no NetBox would return. It now serves a real status body, which is what surfaced the probe finding in eight existing tests. --- core/change_detector.py | 8 +++++-- core/config.py | 4 +++- core/export_manifest.py | 11 ++++++++-- core/graphql_client.py | 7 ++++++- core/module_bay_types.py | 16 ++++++++++++-- core/netbox_api.py | 2 +- tests/conftest.py | 7 +++++++ tests/test_config.py | 10 +++++++++ tests/test_export_manifest.py | 38 +++++++++++++++++++++++++++++----- tests/test_graphql_client.py | 38 ++++++++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 36 ++++++++++++++++++++++++++++++++ tests/test_relation_scope.py | 9 ++++++++ 12 files changed, 172 insertions(+), 14 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 12ea6f0d..e232de17 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -45,8 +45,12 @@ def _relation_properties(comp_type): def _is_relation_list(value): - """Return True when *value* is a list of non-empty strings, the only shape a reference takes.""" - return isinstance(value, list) and all(isinstance(item, str) and item for item in value) + """Return True when *value* is a list of non-empty strings, the only shape a reference takes. + + Blank is checked after stripping, matching the catalog: accepting " " here only defers + the rejection to the write path, where it skips the whole component's update. + """ + return isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value) def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): diff --git a/core/config.py b/core/config.py index fe7fa307..4e1b0068 100644 --- a/core/config.py +++ b/core/config.py @@ -26,7 +26,9 @@ def _sends_token_in_cleartext(url): """Return True when *url* would send the API token over plain HTTP off this host.""" - parsed = urlparse(str(url or "").strip()) + # requests treats a backslash in the authority as a delimiter and urlparse does not, so + # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. + parsed = urlparse(str(url or "").strip().replace("\\", "/")) if parsed.scheme != "http": return False host = (parsed.hostname or "").casefold() diff --git a/core/export_manifest.py b/core/export_manifest.py index b959ddeb..d9ae91c0 100644 --- a/core/export_manifest.py +++ b/core/export_manifest.py @@ -11,6 +11,11 @@ _EMPTY: dict = {"device-types": {}, "module-types": {}, "rack-types": {}} +# Bump whenever the serialized output shape changes (a new stanza, a field that starts or +# stops being written). An unchanged NetBox record has the same last_updated forever, so +# without this an old manifest skips the very types the new shape was added for. +EXPORT_SCHEMA_REVISION = 2 + def load_manifest(path: Path) -> dict: """Load manifest from *path*. Returns an empty manifest on any error.""" @@ -33,13 +38,15 @@ def save_manifest(path: Path, data: dict) -> None: def is_entry_fresh(manifest: dict, kind: str, key: str, last_updated: str) -> bool: - """Return True if the manifest entry for *key* matches *last_updated*.""" + """Return True if *key* was written by this exporter and the record has not changed.""" section = manifest.get(kind) if not isinstance(section, dict): return False entry = section.get(key) if not isinstance(entry, dict): return False + if entry.get("schema") != EXPORT_SCHEMA_REVISION: + return False return entry.get("last_updated") == last_updated @@ -48,4 +55,4 @@ def update_entry(manifest: dict, kind: str, key: str, last_updated: str) -> None section = manifest.get(kind) if not isinstance(section, dict): manifest[kind] = {} - manifest[kind][key] = {"last_updated": last_updated} + manifest[kind][key] = {"last_updated": last_updated, "schema": EXPORT_SCHEMA_REVISION} diff --git a/core/graphql_client.py b/core/graphql_client.py index 88cd53e3..f6e9e8f5 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -212,12 +212,17 @@ def detect_module_bay_type_support(self): try: response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) response.raise_for_status() - version = response.json().get("netbox-version", "") + payload = response.json() except requests.RequestException as exc: raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc except ValueError as exc: # A proxy error page answers 200 with HTML, so the body is not JSON. raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + # Reading an absent version as "" would answer "unsupported" for a 4.7 server and + # export without the relation, so demand the field rather than defaulting it. + version = payload.get("netbox-version") if isinstance(payload, dict) else None + if not isinstance(version, str) or not version.strip(): + raise GraphQLError(f"No netbox-version in the status payload from {status_url}: {payload!r}") self.supports_module_bay_types = supports_module_bay_types(version) return self.supports_module_bay_types diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 308c999e..76697cd6 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -153,8 +153,16 @@ def _load_catalog(self): def _read_catalog(self): """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + + def _unreadable(exc): + # os.walk skips a directory it cannot read; a short catalog resolves to the + # wrong entry rather than failing, so make the traversal error terminal. + raise ModuleBayCatalogError( + f"Module bay type catalog directory {exc.filename!r} could not be read: {exc}" + ) from exc + entries = {} - for root, _dirs, files in os.walk(self._catalog_dir): + for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable): for filename in sorted(files): if not filename.endswith((".yaml", ".yml")): continue @@ -166,8 +174,12 @@ def _read_catalog(self): raise ModuleBayCatalogError( f"Module bay type catalog file {path!r} could not be read: {exc}" ) from exc + if data is None: + continue # an empty or comment-only document is not an entry if not isinstance(data, dict): - continue + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is not a mapping, got {type(data).__name__}" + ) # Reject a half-written entry here; the readers index on name and dereference slug. invalid = [ field diff --git a/core/netbox_api.py b/core/netbox_api.py index bf2ab197..1e09f712 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1626,7 +1626,7 @@ def _type_relation_changes(self, module_type, existing_module): continue related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] declared = module_type[field] - if not isinstance(declared, list) or any(not isinstance(x, str) or not x for x in declared): + if not isinstance(declared, list) or any(not isinstance(x, str) or not x.strip() for x in declared): # Malformed or bare key: leave the relation unmanaged rather than clear it. continue wanted = sorted(set(declared)) diff --git a/tests/conftest.py b/tests/conftest.py index a1df5ff4..a2d55c4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,6 +114,13 @@ def mock_graphql_requests(request): } } mock_session.post.return_value = response + # /api/status/ is a GET; without this the probe reads a synthesized MagicMock and + # any shape check on the payload sees something no NetBox would ever return. + status = MagicMock() + status.status_code = 200 + status.raise_for_status = MagicMock() + status.json.return_value = {"netbox-version": "4.7.0"} + mock_session.get.return_value = status yield mock_session.post diff --git a/tests/test_config.py b/tests/test_config.py index d9db0907..2db730f7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -175,3 +175,13 @@ def test_https_needs_no_notice(self): config = _resolve(NETBOX_URL="https://netbox.example.com") assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + +class TestCleartextCheckMatchesWhatRequestsWillDo: + """The notice is worthless if the URL it parses is not the URL the token is sent to.""" + + def test_a_backslash_authority_is_not_treated_as_loopback(self): + """The host reads as localhost here, but requests targets the address before the backslash.""" + config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_export_manifest.py b/tests/test_export_manifest.py index 8ffeb796..c646a241 100644 --- a/tests/test_export_manifest.py +++ b/tests/test_export_manifest.py @@ -93,11 +93,10 @@ class TestIsEntryFresh: """Tests for is_entry_fresh function.""" def test_fresh_when_last_updated_matches(self): - manifest = { - "device-types": {"Nokia/acme-x": {"last_updated": "2024-01-01T00:00:00Z"}}, - "module-types": {}, - "rack-types": {}, - } + """Built through update_entry: a hand-written entry cannot carry the schema revision.""" + manifest = {"device-types": {}, "module-types": {}, "rack-types": {}} + update_entry(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") + assert is_entry_fresh(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") is True def test_stale_when_last_updated_differs(self): @@ -125,3 +124,32 @@ def test_updates_existing_entry(self): manifest = {"device-types": {"Nokia/acme-x": {"last_updated": "old"}}, "module-types": {}, "rack-types": {}} update_entry(manifest, "device-types", "Nokia/acme-x", "2024-02-01T00:00:00Z") assert manifest["device-types"]["Nokia/acme-x"]["last_updated"] == "2024-02-01T00:00:00Z" + + +class TestExportSchemaRevision: + """last_updated alone cannot see a change in what the exporter writes.""" + + def test_an_entry_from_an_older_exporter_is_not_fresh(self): + """The record did not change, but the serialized shape did, so it must be rewritten.""" + from core.export_manifest import is_entry_fresh + + manifest = {"device-types": {"Acme/x": {"last_updated": "2026-01-01T00:00:00Z"}}} + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is False + + def test_an_entry_this_exporter_wrote_is_fresh(self): + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is True + + def test_a_changed_record_is_still_not_fresh(self): + """The revision must not paper over the timestamp check it sits beside.""" + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-02-02T00:00:00Z") is False diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 5cfb8004..e5ec70c7 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -201,6 +201,44 @@ def log_message(self, *args): server.shutdown() server.server_close() + @pytest.mark.real_http + @pytest.mark.parametrize( + ("body", "why"), + [ + (b"[]", "a JSON list has no .get, so the probe raised a bare AttributeError"), + (b"{}", "no netbox-version read as '' and silently disabled the relation"), + (b'{"netbox-version": ""}', "an empty version is not a version"), + (b'{"netbox-version": null}', "a null version is not a version"), + ], + ) + def test_a_wrong_shaped_status_body_fails_the_probe(self, body, why): + """Silently deciding 'unsupported' on a 4.7 server exports without the relation.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + @pytest.mark.real_http def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index 7fa40c05..ab0a1a32 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -165,6 +165,17 @@ def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog) cat, _ = catalog(root=tmp_path) assert len(cat.ids_for("generic", ["SFP"])) == 1 + def test_a_yaml_document_that_is_not_a_mapping_is_refused(self, tmp_path, catalog): + """Silently skipping it shrinks the catalog, and a Generic entry then answers instead.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + directory = tmp_path / "module-bay-types" / "Juniper" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("- name: SFP\n slug: sfp\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("Juniper", ["SFP"]) + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" directory = tmp_path / "module-bay-types" / "Generic" @@ -300,3 +311,28 @@ def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, c with pytest.raises(ModuleBayTypeError): resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) + + +@pytest.mark.real_http +class TestUnreadableCatalogDirectory: + """os.walk swallows a directory it cannot read, which silently shrinks the catalog.""" + + def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, catalog): + """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" + import os + + if os.geteuid() == 0: + pytest.skip("root ignores the permission bits this test relies on") + + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "the owner-scoped entry") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "the fallback entry") + vendor_dir = root / "module-bay-types" / "Juniper" + os.chmod(vendor_dir, 0o000) + try: + resolver, _server = catalog(root=root) + + with pytest.raises(ModuleBayCatalogError): + resolver.identities_for("Juniper", ["QSFP-DD"]) + finally: + os.chmod(vendor_dir, 0o755) diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py index 7dac8f0f..ca69e25e 100644 --- a/tests/test_relation_scope.py +++ b/tests/test_relation_scope.py @@ -138,6 +138,15 @@ def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + def test_a_whitespace_only_entry_leaves_the_relation_alone(self, two_scope_catalog): + """The catalog rejects it later, which would skip the whole component's update.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [" "]}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): """Reading an absent field as empty would report a change on every run.""" detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) From 51dc2a8a43d04b14316ce39e44c63b83171c51ff Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 10:13:46 +0200 Subject: [PATCH 12/39] test(module-bay-types): restore the vendor directory owner-only The unreadable-directory test only needs the directory readable again so tmp_path cleanup can remove it, so 0o700 does the job. 0o755 hands the group and world access the test never needed, which bandit's S103 flags. --- tests/test_module_bay_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index ab0a1a32..66f52a70 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -335,4 +335,4 @@ def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, with pytest.raises(ModuleBayCatalogError): resolver.identities_for("Juniper", ["QSFP-DD"]) finally: - os.chmod(vendor_dir, 0o755) + os.chmod(vendor_dir, 0o700) From 5d80883b6495070b3adff5db609bd734dbe744f0 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 12:15:41 +0200 Subject: [PATCH 13/39] fix: stop three silent data paths the adversarial review found All three lose or skip data without saying anything. - Adding a mapping to a front port that NetBox 4.5+ reports as unmapped did nothing at all. mappings=[] carries no rear port name, and the comparison inferred the model from the data, so an empty list read as a pre-4.5 record and produced a positions-only tuple. _build_mappings_patch returns None for that, so no PATCH was sent and nothing was logged. The record wrapper now records which model it read instead of leaving it to be guessed. - Exporting from a server below 4.7 stripped module_bay_types from every definition it wrote. The relation is never queried there, so its absence means "not asked", not "cleared", but the writer preserved only absent non-list top-level fields and took component lists wholly from NetBox. Change one description and the restriction was gone. Relations the server cannot answer for are now carried back from the repo, matched by name; a server that does answer stays authoritative, which a second test pins. - The cleartext-token notice could abort the run. urlparse rejects some authorities with ValueError, which escapes the ConfigError handler and reaches the user as a traceback on a configuration that worked before. An authority this tool cannot read is also one it cannot clear as loopback, so it warns rather than raising. Also guards os.geteuid, which does not exist on Windows, so the permission test skips instead of erroring during collection. --- core/change_detector.py | 5 +++- core/config.py | 7 ++++- core/export.py | 43 ++++++++++++++++++++++++++++--- core/netbox_api.py | 6 ++++- tests/test_change_detector.py | 25 ++++++++++++++++++ tests/test_config.py | 6 +++++ tests/test_exporter.py | 47 ++++++++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 2 ++ 8 files changed, 134 insertions(+), 7 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index e232de17..5ea7666f 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -518,7 +518,10 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - has_names = any(m.get("rear_port_name") is not None for m in canonical) + # The wrapper records the model it read; fall back to inference only for a + # caller that did not wrap the record. + m2m = getattr(netbox_comp, "_mappings_m2m", None) + has_names = m2m if m2m is not None else any(m.get("rear_port_name") is not None for m in canonical) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( diff --git a/core/config.py b/core/config.py index 4e1b0068..fd4dd6c8 100644 --- a/core/config.py +++ b/core/config.py @@ -28,7 +28,12 @@ def _sends_token_in_cleartext(url): """Return True when *url* would send the API token over plain HTTP off this host.""" # requests treats a backslash in the authority as a delimiter and urlparse does not, so # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. - parsed = urlparse(str(url or "").strip().replace("\\", "/")) + try: + parsed = urlparse(str(url or "").strip().replace("\\", "/")) + except ValueError: + # urlparse rejects some authorities outright. An authority this tool cannot read is + # also one it cannot clear as loopback, and a notice must never abort the run. + return True if parsed.scheme != "http": return False host = (parsed.hostname or "").casefold() diff --git a/core/export.py b/core/export.py index b01ede70..21670232 100644 --- a/core/export.py +++ b/core/export.py @@ -14,6 +14,7 @@ import requests import yaml +from core.component_registry import COMPONENT_TYPES, MODULE_TYPE_RELATIONS from core.export_manifest import ( is_entry_fresh, load_manifest, @@ -141,6 +142,40 @@ def _yaml_equal(a: dict, b: dict) -> bool: return _normalize_for_compare(a) == _normalize_for_compare(b) +def _relation_names() -> set: + """Every relation key the registry knows, so this does not become a second source.""" + return {relation for component in COMPONENT_TYPES for relation in component.relations} | set(MODULE_TYPE_RELATIONS) + + +def _carry_unqueried_relations(repo_yaml: dict, serialized: dict) -> dict: + """Return *serialized* with relations the server never answered for taken from the repo. + + Below NetBox 4.7 module_bay_types is not queried at all, so its absence means "not asked", + not "cleared". Writing NetBox's answer as it stands would delete the restriction from + every definition the export touches for any other reason. + """ + relations = _relation_names() + result = dict(serialized) + for relation in relations: + if relation in repo_yaml and relation not in result: + result[relation] = repo_yaml[relation] + for key, repo_value in repo_yaml.items(): + if not isinstance(repo_value, list) or not isinstance(result.get(key), list): + continue + by_name = {e.get("name"): e for e in repo_value if isinstance(e, dict)} + merged = [] + for entry in result[key]: + if isinstance(entry, dict): + repo_entry = by_name.get(entry.get("name")) + if isinstance(repo_entry, dict): + carried = {r: repo_entry[r] for r in relations if r in repo_entry and r not in entry} + if carried: + entry = {**entry, **carried} + merged.append(entry) + result[key] = merged + return result + + def _repo_supersedes(repo_yaml: dict, nb_serialized: dict) -> bool: """Return True when *repo_yaml* already contains every field NetBox would write. @@ -456,15 +491,15 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: # top-level fields (e.g. comments, profile) that NetBox does not return # in its serialized output. Component lists are left as NB authoritative. to_write = item.serialized + if item.repo_yaml and not self.graphql.supports_module_bay_types: + to_write = _carry_unqueried_relations(item.repo_yaml, to_write) if item.reason == "differs" and item.repo_yaml: # Only preserve scalar/metadata repo fields not present in the NB output. # Exclude list-valued keys (component sections such as interfaces, power-ports, # console-ports, etc.) so that NB remains authoritative for all components. - extra = { - k: v for k, v in item.repo_yaml.items() if k not in item.serialized and not isinstance(v, list) - } + extra = {k: v for k, v in item.repo_yaml.items() if k not in to_write and not isinstance(v, list)} if extra: - to_write = {**item.serialized, **extra} + to_write = {**to_write, **extra} written = self._write_yaml(dest, to_write) if not written: skipped_overwrite += 1 diff --git a/core/netbox_api.py b/core/netbox_api.py index 1e09f712..63565c6c 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -2241,7 +2241,7 @@ class _FrontPortRecordWithMappings: All other attribute accesses are forwarded to the underlying record. """ - __slots__ = ("_record", "_mappings_canonical") + __slots__ = ("_record", "_mappings_canonical", "_mappings_m2m") def __init__(self, record): """Wrap *record* and pre-compute a canonical mappings list for ChangeDetector compatibility. @@ -2287,6 +2287,10 @@ def __init__(self, record): else None # Both mappings and rear_port_position absent; skip comparison. ) object.__setattr__(self, "_mappings_canonical", canonical) + # Which model the record came from, recorded rather than inferred: an empty M2M list + # carries no names to infer from, and reading it as pre-4.5 drops the rear port name + # the patch needs. + object.__setattr__(self, "_mappings_m2m", mappings_raw is not None) def __getattr__(self, name): """Delegate attribute access to the wrapped record.""" diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index bf9d6a92..d03ffb75 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -428,6 +428,31 @@ def _make_netbox_comp(self, canonical, **attrs): """Build a netbox component with _mappings_canonical and explicit attributes.""" return SimpleNamespace(_mappings_canonical=canonical, **attrs) + def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): + """mappings=[] is 4.5+ saying "none", not a pre-4.5 record without the field. + + Inferring the model from the data cannot tell those apart when the list is empty, and + a positions-only tuple makes _build_mappings_patch return None, so the mapping is + never added and nothing is logged. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + netbox_comp = _FrontPortRecordWithMappings(SimpleNamespace(name="FP1", type="8p8c", mappings=[])) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "adding a mapping is a change" + tup = next(iter(changes[0].new_value)) + assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" + assert tup[0] == "RP1" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { diff --git a/tests/test_config.py b/tests/test_config.py index 2db730f7..e289960c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -185,3 +185,9 @@ def test_a_backslash_authority_is_not_treated_as_loopback(self): config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_an_unparseable_url_is_reported_not_raised(self): + """Urlparse raises on some authorities; an advisory notice must not abort the run.""" + config = _resolve(NETBOX_URL="http://user[foo]@198.18.0.1") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_exporter.py b/tests/test_exporter.py index e5184fb5..456f688c 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1650,3 +1650,50 @@ def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): self._write(tmp_path, item) assert "Sub 0" in capsys.readouterr().out + + +class TestUnqueriedRelationsSurviveTheExport: + """A pre-4.7 server never returns module_bay_types, so the export must not strip it.""" + + @staticmethod + def _item(repo_yaml, serialized): + return ExportItem( + kind="device-type", + nb_record=_make_dt(), + repo_yaml=repo_yaml, + serialized=serialized, + reason="differs", + mfr_name="Juniper", + filename="mx304.yaml", + manifest_key="Juniper/mx304", + ) + + def _write(self, tmp_path, item, supported): + exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = supported + exporter._get_module_image_details = lambda: {} + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) + + def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path): + """Only the description changed; the relation must not be collateral damage.""" + repo = { + "model": "MX304", + "description": "old", + "module-bays": [{"name": "RE0", "position": "0", "module_bay_types": ["MX304-RE"]}], + } + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0", "position": "0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=False) + + assert written["description"] == "new", "the real change still lands" + assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] + + def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): + """On 4.7 an absent relation means NetBox cleared it, so it must not be resurrected.""" + repo = {"model": "MX304", "module-bays": [{"name": "RE0", "module_bay_types": ["MX304-RE"]}]} + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=True) + + assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index 66f52a70..82a2e4d1 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -321,6 +321,8 @@ def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" import os + if not hasattr(os, "geteuid"): + pytest.skip("no POSIX ownership, so the permission bits mean nothing here") if os.geteuid() == 0: pytest.skip("root ignores the permission bits this test relies on") From c4baac89a5d7fb9aed8fda4810ac1b94fbeafdd9 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 13:09:48 +0200 Subject: [PATCH 14/39] fix(mappings): report legacy truncation and detect a legacy rear-port move Three more silent paths from the same review, all on the pre-4.5 mapping model, plus the stanza case that could not express a removal. - A front port moved from RP1 to RP2 produced no change, no PATCH and no warning. The pre-4.5 query does ask for rear_port { id name }, but the record wrapper discarded the name, leaving only positions to compare, and the positions had not changed. The wrapper keeps the name now, and the comparison uses names whenever one is available. - The update path truncated several mappings to one in silence, where the create path already said so. Both now report it through _log_component_error, so the text reaches the outcome reason instead of scrolling past. This does not reclassify the outcome itself. - Which mapping survived that truncation was arbitrary: the winner came from next(iter(frozenset)). It is the sorted first now, so two runs against the same library agree. - port-mappings: [] could not clear anything. An empty stanza fell back to the inline format and no front port received _mappings: [], so a removal was unrepresentable. An explicitly empty stanza is authoritative now, and an empty stanza beside a surviving inline rear_port is a conflict rather than a silent preference. An absent key still means "no opinion", which a test pins, because reading it as empty would clear every file. normalize_port_mappings passed the complexity gate as a result, so the inline collection and the conflict check moved into helpers. Also asserts the written file in two export warning tests. They checked only that no warning appeared, which would hold if the writer never ran. --- core/change_detector.py | 9 ++-- core/netbox_api.py | 18 +++++-- core/repo.py | 99 ++++++++++++++++++++++------------- tests/test_change_detector.py | 23 ++++++++ tests/test_exporter.py | 5 ++ tests/test_netbox_api.py | 34 ++++++++++++ tests/test_repo.py | 41 +++++++++++++++ 7 files changed, 187 insertions(+), 42 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 5ea7666f..afc914f1 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -518,10 +518,11 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - # The wrapper records the model it read; fall back to inference only for a - # caller that did not wrap the record. - m2m = getattr(netbox_comp, "_mappings_m2m", None) - has_names = m2m if m2m is not None else any(m.get("rear_port_name") is not None for m in canonical) + # Compare by name whenever one is available: an empty M2M list has no name to + # infer from but still needs the named path, and the pre-4.5 query returns one. + has_names = bool(getattr(netbox_comp, "_mappings_m2m", False)) or any( + m.get("rear_port_name") is not None for m in canonical + ) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( diff --git a/core/netbox_api.py b/core/netbox_api.py index 63565c6c..e4d9b253 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -2275,10 +2275,18 @@ def __init__(self, record): else: # NetBox < 4.5: rear_port_position is a direct scalar field rp_pos = getattr(record, "rear_port_position", None) + # The pre-4.5 query asks for rear_port { id name }; keeping the name is what lets + # a move to a different rear port be seen at all. + legacy_rp = getattr(record, "rear_port", None) + legacy_name = ( + (legacy_rp.get("name") if isinstance(legacy_rp, dict) else getattr(legacy_rp, "name", None)) + if legacy_rp is not None + else None + ) canonical = ( [ { - "rear_port_name": None, + "rear_port_name": legacy_name, "front_port_position": 1, "rear_port_position": rp_pos, } @@ -2547,7 +2555,11 @@ def _apply_mappings_change(self, comp_name, new_mappings, yaml_mappings, update_ update_data["rear_port"] = None update_data["rear_port_position"] = None return - first = next(iter(new_mappings)) + if len(new_mappings) > 1: + self._log_component_error( + f'Multiple mappings for front port "{comp_name}" on NetBox < 4.5: only first mapping applied' + ) + first = sorted(new_mappings)[0] if len(first) != 3: # Legacy NetBox (<4.5): ChangeDetector emits 2-tuples (fp_pos, rp_pos) # because rear port names are unavailable via the GraphQL API. @@ -2917,7 +2929,7 @@ def link_rear_ports(items, pid): else: if len(resolved) > 1: ctx = f" (Context: {context})" if context else "" - self.handle.log( + self._log_component_error( f'Multiple mappings for {label} "{port["name"]}" on NetBox < 4.5: ' f"only first mapping applied{ctx}" ) diff --git a/core/repo.py b/core/repo.py index 2022cb57..ec7cf0c3 100644 --- a/core/repo.py +++ b/core/repo.py @@ -272,6 +272,51 @@ def validate_repo_path(repo_path): return True, "" +def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): + """Return ``({front_port_name: [mapping, ...]}, error)`` for the pre-4.5 inline format. + + The inline keys are removed from each entry as they are read, so the caller is left with + one representation to reason about. + """ + inline_mappings: dict = {} + for fp in front_ports: + rp_name = fp.get("rear_port") + if rp_name is None: + continue + fp_name = fp.get("name") + if rear_ports_declared and rp_name not in rear_by_name: + return {}, f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" + rp_pos = fp.pop("rear_port_position", 1) + fp.pop("rear_port") + inline_mappings.setdefault(fp_name, []).append( + {"rear_port": rp_name, "front_port_position": 1, "rear_port_position": rp_pos} + ) + return inline_mappings, None + + +def _conflicting_mapping(inline_mappings, stanza_mappings): + """Return an error when both formats describe the same front port differently. + + Carrying both is allowed only while they agree, which is what a half-finished migration + looks like; disagreeing is the case where guessing a winner would silently pick one. + """ + if not (inline_mappings and stanza_mappings): + return None + + def _shape(mappings): + return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) + + for name in set(inline_mappings) | set(stanza_mappings): + inline = _shape(inline_mappings.get(name, [])) + stanza = _shape(stanza_mappings.get(name, [])) + if inline != stanza: + return ( + f"Error: front port '{name}' has conflicting mapping definitions " + f"(inline: {inline}, port-mappings stanza: {stanza})" + ) + return None + + def normalize_port_mappings(data): """Normalize port mapping definitions in a parsed YAML device/module type dict. @@ -320,27 +365,14 @@ def normalize_port_mappings(data): # --- Old inline format --- # Collect rear_port references declared directly on front-port entries. - inline_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - for fp in front_ports: - rp_name = fp.get("rear_port") - if rp_name is None: - continue - fp_name = fp.get("name") - if rear_ports_declared and rp_name not in rear_by_name: - return f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" - rp_pos = fp.pop("rear_port_position", 1) - fp.pop("rear_port") - inline_mappings.setdefault(fp_name, []).append( - { - "rear_port": rp_name, - "front_port_position": 1, - "rear_port_position": rp_pos, - } - ) + inline_mappings, error = _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared) + if error: + return error # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - if "port-mappings" in data: + stanza_present = "port-mappings" in data + if stanza_present: for entry in port_mappings_stanza or []: fp_name = entry.get("front_port") rp_name = entry.get("rear_port") @@ -359,24 +391,21 @@ def normalize_port_mappings(data): ) del data["port-mappings"] - # --- Conflict detection --- - # Accept both formats simultaneously only when they describe identical mappings. - if inline_mappings and stanza_mappings: - all_names = set(inline_mappings) | set(stanza_mappings) - for name in all_names: - inline = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in inline_mappings.get(name, []) - ) - stanza = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in stanza_mappings.get(name, []) + conflict = _conflicting_mapping(inline_mappings, stanza_mappings) + if conflict: + return conflict + + # An explicitly empty stanza states there are no mappings. An absent key states nothing, + # so only the first may clear what NetBox already holds. + if stanza_present and not stanza_mappings: + if inline_mappings: + return ( + "Error: port-mappings is empty but front port(s) " + f"{sorted(inline_mappings)} still declare an inline rear_port" ) - if inline != stanza: - return ( - f"Error: front port '{name}' has conflicting mapping definitions " - f"(inline: {inline}, port-mappings stanza: {stanza})" - ) + for fp in front_ports: + fp["_mappings"] = [] + return None effective = stanza_mappings if stanza_mappings else inline_mappings for fp_name, mappings in effective.items(): diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index d03ffb75..f56450d3 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -453,6 +453,29 @@ def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" assert tup[0] == "RP1" + def test_a_legacy_rear_port_change_is_detected_by_name(self): + """The pre-4.5 query asks for rear_port { id name }, so the name is available. + + Discarding it left only positions to compare, and RP1/1 -> RP2/1 produced no change, + no PATCH and no warning: the definition silently never synced. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + legacy = SimpleNamespace(name="FP1", type="8p8c", rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) + netbox_comp = _FrontPortRecordWithMappings(legacy) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "moving the front port to a different rear port is a change" + assert next(iter(changes[0].new_value))[0] == "RP2" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 456f688c..49d605d3 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1633,6 +1633,9 @@ def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): self._write(tmp_path, item) + # Assert the file was written too: silence alone would also hold if nothing ran. + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] assert "no position" not in capsys.readouterr().out def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): @@ -1641,6 +1644,8 @@ def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written == {"model": "7750-SR-7s"}, "the write path ran, it simply had nothing to warn about" assert "no position" not in capsys.readouterr().out def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 895c2c62..095689bf 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -518,6 +518,40 @@ def test_update_components_legacy_mapping_two_tuple_warns_and_skips( assert any("NetBox < 4.5" in str(c) for c in mock_handle.log.call_args_list) +def test_update_components_legacy_truncation_is_reported( + mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle +): + """The create path says "only first mapping applied"; the update path said nothing at all.""" + from core.change_detector import ChangeType, ComponentChange, PropertyChange + + mock_nb_api = MagicMock() + dt = make_device_types(nb_api=mock_nb_api) + dt.m2m_front_ports = False + + existing_fp = MagicMock(id=10, name="FP1") + rp1 = MagicMock(id=21, name="RP1") + rp2 = MagicMock(id=22, name="RP2") + dt.components.record("front_port_templates", "device", 1, {"FP1": existing_fp}) + dt.components.record("rear_port_templates", "device", 1, {"RP1": rp1, "RP2": rp2}) + + # Two mappings: the legacy model can hold only one of them. + new_mappings_set = frozenset({("RP1", 1, 1), ("RP2", 2, 1)}) + changes = [ + ComponentChange( + component_type="front-ports", + component_name="FP1", + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange("_mappings", frozenset(), new_mappings_set)], + ), + ] + + mock_handle.log.reset_mock() + dt.update_components({}, 1, changes, parent_type="device") + + logged = " ".join(str(c) for c in mock_handle.log.call_args_list) + assert "FP1" in logged and "4.5" in logged, f"truncation must be reported, got: {logged}" + + def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( mock_settings, mock_pynetbox, graphql_client, make_device_types ): diff --git a/tests/test_repo.py b/tests/test_repo.py index ab4e695a..457f1a4f 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1796,3 +1796,44 @@ def test_corrupted_device_json_returns_none(self, tmp_path): repo.cwd = "" assert repo.resolve_slug_files(["nokia"]) is None + + +class TestAnExplicitlyEmptyStanza: + """`port-mappings: []` is an author saying "none", which is not the same as saying nothing.""" + + def test_an_empty_stanza_clears_every_front_port_mapping(self): + """Without _mappings: [] the change detector cannot express removing a mapping.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c"}, {"name": "FP2", "type": "8p8c"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 2}], + "port-mappings": [], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [] + assert data["front-ports"][1]["_mappings"] == [] + + def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): + """Silently preferring the inline linkage ignores the newer, explicit statement.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 1}], + "port-mappings": [], + } + + result = normalize_port_mappings(data) + + assert result is not None and result.startswith("Error:"), result + + def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): + """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" + from core.repo import normalize_port_mappings + + data = {"front-ports": [{"name": "FP1", "type": "8p8c"}], "rear-ports": []} + + assert normalize_port_mappings(data) is None + assert "_mappings" not in data["front-ports"][0] From 26f24e96a57986ecd4a44019a460d2d1710136f5 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 13:14:32 +0200 Subject: [PATCH 15/39] test: split composite assertions and drop a useless lambda Each half of a composite assert reports which condition failed, and dict is the callable the stub wanted. Caught by the widened rule set on the branch that enables PT018 and PIE807. --- tests/test_exporter.py | 4 ++-- tests/test_netbox_api.py | 3 ++- tests/test_repo.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 49d605d3..b2e1fa4e 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1613,7 +1613,7 @@ def _item(module_bays, kind="device-type"): def _write(tmp_path, item): """Drive the real write path with a real LogHandler, which prints to stdout.""" exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) - exporter._get_module_image_details = lambda: {} + exporter._get_module_image_details = dict exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): @@ -1676,7 +1676,7 @@ def _item(repo_yaml, serialized): def _write(self, tmp_path, item, supported): exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) exporter.graphql.supports_module_bay_types = supported - exporter._get_module_image_details = lambda: {} + exporter._get_module_image_details = dict exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 095689bf..b2cdbabc 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -549,7 +549,8 @@ def test_update_components_legacy_truncation_is_reported( dt.update_components({}, 1, changes, parent_type="device") logged = " ".join(str(c) for c in mock_handle.log.call_args_list) - assert "FP1" in logged and "4.5" in logged, f"truncation must be reported, got: {logged}" + assert "FP1" in logged, f"truncation must name the port, got: {logged}" + assert "4.5" in logged, f"truncation must say why, got: {logged}" def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( diff --git a/tests/test_repo.py b/tests/test_repo.py index 457f1a4f..b661dc96 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1827,7 +1827,8 @@ def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): result = normalize_port_mappings(data) - assert result is not None and result.startswith("Error:"), result + assert result is not None, "an empty stanza beside an inline linkage must not pass silently" + assert result.startswith("Error:"), result def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" From 15ac9a9aa9f1f16044ba95853bb1260961dabf6b Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 17:47:43 +0200 Subject: [PATCH 16/39] fix(mappings): name the stanza as authoritative instead of blaming a conflict `_conflicting_mapping` compared the union of front-port names, so any name that appeared in only one of the two formats was reported as a conflict against an empty shape. Two consequences. A front port the stanza never mentions was reported as "conflicting mapping definitions (inline: [('RP1', 1, 1)], port-mappings stanza: [])", which names a stanza that has no opinion on that port. The rejection is still correct: `port-mappings: []` clears every front port, so a stanza speaks for the whole file and an inline linkage it omits cannot be honoured. Only the diagnostic was wrong, so it now says the stanza is authoritative and how to resolve it. A front port the stanza alone named was also rejected, which blocked the half-finished migration the two formats exist to allow: FP1 kept inline and FP2 moved to the stanza. Such a port had no inline linkage, so nothing disagrees. The loop now walks only the inline names. Iteration was over a set, so with several offending ports the reported name varied between runs. It is sorted now. Selecting `effective = stanza_mappings` stays safe: once the check passes, every inline name is present in the stanza with an equal shape, so no inline mapping can be dropped. --- core/repo.py | 16 ++++++++--- tests/test_repo.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/core/repo.py b/core/repo.py index ec7cf0c3..074d19ae 100644 --- a/core/repo.py +++ b/core/repo.py @@ -295,10 +295,12 @@ def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): def _conflicting_mapping(inline_mappings, stanza_mappings): - """Return an error when both formats describe the same front port differently. + """Return an error when the two formats cannot both be honoured. Carrying both is allowed only while they agree, which is what a half-finished migration looks like; disagreeing is the case where guessing a winner would silently pick one. + A stanza speaks for the whole file, so an inline linkage it omits cannot be honoured + either. A port the stanza alone names is not in question: it had no inline linkage. """ if not (inline_mappings and stanza_mappings): return None @@ -306,9 +308,15 @@ def _conflicting_mapping(inline_mappings, stanza_mappings): def _shape(mappings): return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) - for name in set(inline_mappings) | set(stanza_mappings): - inline = _shape(inline_mappings.get(name, [])) - stanza = _shape(stanza_mappings.get(name, [])) + for name in sorted(inline_mappings): + if name not in stanza_mappings: + return ( + f"Error: front port '{name}' declares an inline rear_port but the port-mappings " + f"stanza does not list it; the stanza is authoritative, so add '{name}' to it " + f"or remove the inline rear_port keys" + ) + inline = _shape(inline_mappings[name]) + stanza = _shape(stanza_mappings[name]) if inline != stanza: return ( f"Error: front port '{name}' has conflicting mapping definitions " diff --git a/tests/test_repo.py b/tests/test_repo.py index b661dc96..45c819f2 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1838,3 +1838,75 @@ def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): assert normalize_port_mappings(data) is None assert "_mappings" not in data["front-ports"][0] + + +class TestAStanzaThatDoesNotListAFrontPort: + """A stanza speaks for the whole file, so a port it omits has no mapping.""" + + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): + """The old wording blamed a conflict against a stanza that never mentioned the port.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP2", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an inline linkage the stanza omits must not pass silently" + assert "conflicting mapping definitions" not in result, result + assert "FP1" in result, result + assert "does not list it" in result, result + + def test_a_disagreement_on_a_shared_front_port_still_reads_as_a_conflict(self): + """Both formats naming one port differently is a real conflict, not an omission.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP1", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None + assert "conflicting mapping definitions" in result, result + + def test_a_stanza_may_add_a_port_the_inline_format_never_linked(self): + """The half-finished migration the two formats exist to allow: both are kept.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [ + {"front_port": "FP1", "rear_port": "RP1"}, + {"front_port": "FP2", "rear_port": "RP2"}, + ], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] From e0fb671423bb31af65259bb536565a1cfb6f07dc Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 12:41:11 +0200 Subject: [PATCH 17/39] refactor: remove two ways for the same fact to be stated twice Both changes are mechanical. No behavior changes. `DeviceTypes.module_bay_types` returned the catalog while `NetBox.module_bay_types` is the boolean support flag, and `_type_relation_changes` reads both in one function. The code was correct, but a mistaken receiver silently disabled the relation instead of failing, because a falsy catalog and a False flag read the same at the call site. The catalog is now `module_bay_type_catalog`. No alias is left behind. `get_module_types` hand-rolled the GraphQL selection for `module_bay_types`, which `ComponentType.graphql_relation_fields` already builds for component templates. Two spellings of one relation shape drift as soon as the relation gains a field. Both now come from `relation_selection` in the component registry, and the module type query derives its relation list from `MODULE_TYPE_RELATIONS` rather than a literal. GraphQL ignores the whitespace the extracted form changes. `test_relation_selections_use_the_shared_helper` asserted on call names, so it still passed with `relation_selection` returning an empty string. It now compares the generated selection text and the query actually sent. --- core/change_detector.py | 2 +- core/component_registry.py | 8 ++++-- core/graphql_client.py | 5 ++-- core/netbox_api.py | 21 +++++++------- tests/test_graphql_client.py | 46 ++++++++++++++++++++++++++++++ tests/test_module_bay_type_sync.py | 5 ++-- tests/test_relation_scope.py | 2 +- 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index afc914f1..1a7de71d 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -474,7 +474,7 @@ def _relation_catalog(self): """ from core.module_bay_types import ModuleBayTypeCatalog - catalog = getattr(self.device_types, "module_bay_types", None) + catalog = getattr(self.device_types, "module_bay_type_catalog", None) return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None def _compare_component_properties( diff --git a/core/component_registry.py b/core/component_registry.py index fe414278..8fb4e474 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -24,6 +24,11 @@ MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) +def relation_selection(name): + """Return the GraphQL fields that identify a related object.""" + return f"{name} {{ id name slug manufacturer {{ slug }} }}" + + @dataclass(frozen=True) class ComponentType: """One kind of component template, described once.""" @@ -50,8 +55,7 @@ def graphql_fields(self): @property def graphql_relation_fields(self): """GraphQL selections for this row's relations, one nested block per relation.""" - # slug plus owning manufacturer is the identity; the name alone is ambiguous. - return [f"{name} {{ id name slug manufacturer {{ slug }} }}" for name in self.relations] + return [relation_selection(name) for name in self.relations] @property def compare_properties(self): diff --git a/core/graphql_client.py b/core/graphql_client.py index f6e9e8f5..a188ea6e 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -13,7 +13,7 @@ import requests from core.compat import supports_module_bay_types -from core.component_registry import BY_ENDPOINT +from core.component_registry import BY_ENDPOINT, MODULE_TYPE_RELATIONS, relation_selection # Module-level dedup: tracks (url, requested_page_size) pairs that have already # emitted the page-size clamping warning so the message appears at most once @@ -523,8 +523,7 @@ def get_module_types(self, manufacturer_slugs=None): """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) module_bay_type_selection = ( - "module_bay_types {\n id\n name\n slug\n" - " manufacturer {\n slug\n }\n }\n " + "".join(f"{relation_selection(name)}\n " for name in MODULE_TYPE_RELATIONS) if self.supports_module_bay_types else "" ) diff --git a/core/netbox_api.py b/core/netbox_api.py index e4d9b253..544ce538 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1632,7 +1632,7 @@ def _type_relation_changes(self, module_type, existing_module): wanted = sorted(set(declared)) current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) if _relation_identities_differ( - self.device_types.module_bay_types, + self.device_types.module_bay_type_catalog, self.device_types._manufacturer_slug(module_type.get("manufacturer")), declared, related, @@ -1655,7 +1655,8 @@ def _resolve_type_relations(self, payload): return {k: v for k, v in payload.items() if k not in names} manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) resolved = { - field: self.device_types.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + field: self.device_types.module_bay_type_catalog.ids_for(manufacturer, value) + for field, value in names.items() } return {**payload, **resolved} @@ -1680,7 +1681,7 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): try: - updates[field] = self.device_types.module_bay_types.ids_for( + updates[field] = self.device_types.module_bay_type_catalog.ids_for( self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted ) except ModuleBayTypeError as exc: @@ -2354,7 +2355,7 @@ def __init__( wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None - self._module_bay_types = None + self._module_bay_type_catalog = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2638,7 +2639,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, from core.module_bay_types import ModuleBayTypeError try: - update_data[pc.property_name] = self.module_bay_types.ids_for( + update_data[pc.property_name] = self.module_bay_type_catalog.ids_for( self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value ) except ModuleBayTypeError as exc: @@ -2989,13 +2990,13 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): ) @property - def module_bay_types(self): + def module_bay_type_catalog(self): """The module-bay-type catalog, built once per run from the library checkout.""" - if self._module_bay_types is None: + if self._module_bay_type_catalog is None: from core.module_bay_types import ModuleBayTypeCatalog - self._module_bay_types = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) - return self._module_bay_types + self._module_bay_type_catalog = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_type_catalog @staticmethod def _manufacturer_slug(manufacturer): @@ -3039,7 +3040,7 @@ def _resolve_relations(self, component, items, manufacturer): continue try: replacements = { - field: self.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + field: self.module_bay_type_catalog.ids_for(manufacturer, value) for field, value in names.items() } except ModuleBayTypeError as exc: self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index e5ec70c7..378a83e1 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -1156,6 +1156,52 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + @pytest.mark.real_http + @pytest.mark.parametrize("caller", ["graphql_relation_fields", "get_module_types"]) + def test_relation_selections_use_the_shared_helper(self, caller): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + expected = "module_bay_types { id name slug manufacturer { slug } }" + if caller == "graphql_relation_fields": + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [expected] + else: + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + assert expected in " ".join(query.split()) + finally: + server.close() + + @pytest.mark.real_http + def test_module_type_query_contains_the_complete_relation_selection(self): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + finally: + server.close() + + selection = "module_bay_types { id name slug manufacturer { slug } }" + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [selection] + assert " ".join(query.split()) == " ".join( + """ +query($pagination: OffsetPaginationInput) { + module_type_list(pagination: $pagination) { + id model part_number airflow description comments weight weight_unit last_updated + module_bay_types { id name slug manufacturer { slug } } + manufacturer { id name slug } + } +} +""".split() + ) + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): """The prefetch runs on clones, not on the client it was cloned from. diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index 6030b764..afe27626 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -569,8 +569,9 @@ class TestCatalogWiring: def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): device_types, _ = make_device_types() - catalog = device_types.module_bay_types + catalog = device_types.module_bay_type_catalog assert isinstance(catalog, ModuleBayTypeCatalog) - assert device_types.module_bay_types is catalog + assert device_types.module_bay_type_catalog is catalog + assert not hasattr(device_types, "module_bay_types") assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py index ca69e25e..caee3dd0 100644 --- a/tests/test_relation_scope.py +++ b/tests/test_relation_scope.py @@ -60,7 +60,7 @@ def two_scope_catalog(tmp_path): def _detector(catalog, handle=None): """Build a detector whose device_types exposes the catalog, as the real one does.""" - device_types = type("DeviceTypes", (), {"module_bay_types": catalog, "module_bay_types_supported": True})() + device_types = type("DeviceTypes", (), {"module_bay_type_catalog": catalog, "module_bay_types_supported": True})() return ChangeDetector(device_types, handle or Handle()) From d704a718a36701dbe4140a322a2d602bd82f3866 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 12:45:00 +0200 Subject: [PATCH 18/39] fix(mappings): only an explicit list may clear a relation A bare `port-mappings:` key parses as None. `normalize_port_mappings` decided the stanza was present from key presence alone, so None read the same as an explicit `port-mappings: []` and every front port was assigned `_mappings: []`. The update path writes that to NetBox as a clear, so a one-character typo deleted every front-to-rear mapping on the type. `core/change_detector.py` already stated the opposite rule for the identical YAML shape, that a bare `module_bay_types:` leaves the relation unmanaged, so the two sides disagreed. The rule is now `is_explicit_list` in `core/normalization.py` and both sides call it. A stanza that is neither null nor a list was silently discarded and reported success, and a list holding a non-mapping raised AttributeError out of a function whose contract is to return an "Error:" string. Both now return an error naming the offending value. This is deliberately stricter than `_relation_change`, which logs and ignores the same shape: that path skips one property, this one rejects the file. A nonempty stanza left a front port it omits unmanaged, so an existing linkage on that port could never be removed. `port-mappings: []` already cleared every declared front port, which means a present stanza speaks for the whole file, and a nonempty one now does too. A half-migrated file still fails the existing conflict check before reaching the clearing path, which is what stops a migration from deleting a mapping. Measured against the library: 348 files carry a stanza and every one already lists every front port it declares, so no current definition changes. Mapping order was not canonical. `_is_subset` compares lists of unnamed dicts positionally, so the same mappings in a different order reported "differs" and re-exported unchanged data. The serializer now emits them sorted, and `_repo_supersedes` sorts both inputs, so a hand-written stanza in another order also compares equal. Sorting keeps duplicates, so multiplicity still differs. Three tests could not fail: two compared a value against a list the same emptiness made equal, and one bypassed export selection by supplying its result. Each now fails against a mutation of the code it covers. --- core/change_detector.py | 4 +- core/export.py | 11 +++- core/nb_serializer.py | 6 +- core/normalization.py | 5 ++ core/repo.py | 25 ++++--- tests/test_exporter.py | 127 ++++++++++++++++++++++++++++++++---- tests/test_nb_serializer.py | 25 ++++++- tests/test_repo.py | 92 ++++++++++++++++++++++++++ 8 files changed, 262 insertions(+), 33 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 1a7de71d..5e2ecf78 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -11,7 +11,7 @@ from enum import Enum from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES -from core.normalization import normalize_values +from core.normalization import is_explicit_list, normalize_values from core.formatting import log_property_diffs from core.schema_reader import load_properties_for_type @@ -50,7 +50,7 @@ def _is_relation_list(value): Blank is checked after stripping, matching the catalog: accepting " " here only defers the rejection to the write path, where it skips the whole component's update. """ - return isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value) + return is_explicit_list(value) and all(isinstance(item, str) and item.strip() for item in value) def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): diff --git a/core/export.py b/core/export.py index 21670232..9dd48507 100644 --- a/core/export.py +++ b/core/export.py @@ -206,8 +206,15 @@ def _default_positions(d: dict) -> dict: filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] return {**d, "front-ports": filled} - nrepo = _normalize_for_compare(_default_positions(_norm_mfr(repo_yaml))) - nnb = _normalize_for_compare(_default_positions(_norm_mfr(nb_serialized))) + def _sort_port_mappings(d: dict) -> dict: + mappings = d.get("port-mappings") + if not isinstance(mappings, list) or not all(isinstance(mapping, dict) for mapping in mappings): + return d + fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") + return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} + + nrepo = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml)))) + nnb = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized)))) return _is_subset(nnb, nrepo) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index a822a130..4977553b 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -169,11 +169,12 @@ def _port_mappings(records: list) -> list: stanza = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): name = getattr(record, "name", None) + mappings = [] for mapping in getattr(record, "mappings", None) or []: rear_port = getattr(mapping, "rear_port", None) if not rear_port: continue - stanza.append( + mappings.append( { "front_port": name, "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, @@ -181,6 +182,9 @@ def _port_mappings(records: list) -> list: "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, } ) + stanza.extend( + sorted(mappings, key=lambda m: (m["front_port_position"], m["rear_port_position"], m["rear_port"] or "")) + ) if getattr(record, "mappings", None): continue legacy = getattr(record, "rear_port", None) diff --git a/core/normalization.py b/core/normalization.py index c8846ec9..d807f38a 100644 --- a/core/normalization.py +++ b/core/normalization.py @@ -1,6 +1,11 @@ """Shared value-normalization helpers for YAML-vs-NetBox comparisons.""" +def is_explicit_list(value): + """Only an explicit YAML list manages a relation; null leaves it unmanaged.""" + return isinstance(value, list) + + def normalize_values(yaml_val, nb_val): """Normalize a YAML/NetBox value pair for comparison. diff --git a/core/repo.py b/core/repo.py index 074d19ae..62472f7a 100644 --- a/core/repo.py +++ b/core/repo.py @@ -13,6 +13,7 @@ from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +from core.normalization import is_explicit_list # Top-level directories that make a checkout a device-type library. LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") @@ -362,9 +363,9 @@ def normalize_port_mappings(data): """ front_ports = data.get("front-ports") or [] port_mappings_stanza = data.get("port-mappings") - - if not front_ports and "port-mappings" not in data: - return None + stanza_authoritative = is_explicit_list(port_mappings_stanza) + if port_mappings_stanza is not None and not stanza_authoritative: + return f"Error: port-mappings must be a list: {port_mappings_stanza!r}" front_by_name = {fp["name"]: fp for fp in front_ports if fp.get("name")} rear_ports_declared = "rear-ports" in data @@ -379,9 +380,10 @@ def normalize_port_mappings(data): # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - stanza_present = "port-mappings" in data - if stanza_present: + if stanza_authoritative: for entry in port_mappings_stanza or []: + if not isinstance(entry, dict): + return f"Error: port-mappings entry must be a mapping: {entry!r}" fp_name = entry.get("front_port") rp_name = entry.get("rear_port") if not fp_name or not rp_name: @@ -397,26 +399,23 @@ def normalize_port_mappings(data): "rear_port_position": entry.get("rear_port_position", 1), } ) - del data["port-mappings"] + data.pop("port-mappings", None) conflict = _conflicting_mapping(inline_mappings, stanza_mappings) if conflict: return conflict - # An explicitly empty stanza states there are no mappings. An absent key states nothing, - # so only the first may clear what NetBox already holds. - if stanza_present and not stanza_mappings: - if inline_mappings: + if stanza_authoritative: + if not stanza_mappings and inline_mappings: return ( "Error: port-mappings is empty but front port(s) " f"{sorted(inline_mappings)} still declare an inline rear_port" ) for fp in front_ports: - fp["_mappings"] = [] + fp["_mappings"] = stanza_mappings.get(fp.get("name"), []) return None - effective = stanza_mappings if stanza_mappings else inline_mappings - for fp_name, mappings in effective.items(): + for fp_name, mappings in inline_mappings.items(): if fp_name in front_by_name: front_by_name[fp_name]["_mappings"] = mappings diff --git a/tests/test_exporter.py b/tests/test_exporter.py index b2e1fa4e..8e063884 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -247,17 +247,36 @@ def test_an_empty_relation_does_not_make_every_definition_differ(self): "module_bay_types: []" would therefore be absent from every library definition and re-export the whole library on a NetBox 4.7 server. """ - from core.nb_serializer import _serialize_relations - - bay = type("Bay", (), {"name": "FPC 0", "module_bay_types": []})() - assert _serialize_relations(bay, ("module_bay_types",)) == {} + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type - repo = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} - as_serialized = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} - with_empty_key = {"model": "MX304", "module-bays": [{"name": "FPC 0", "module_bay_types": []}]} + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +module-bays: + - {name: Slot 0, position: '0'} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + components = {1: {"module_bay_templates": [bay]}} + serialized = serialize_device_type(record, components) - assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" - assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + assert serialized["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert _repo_supersedes(repo, serialized), "an unchanged definition must not re-export" + bay.module_bay_types = [DotDict(name="Control")] + populated = serialize_device_type(record, components) + assert populated["module-bays"][0]["module_bay_types"] == ["Control"] + assert not _repo_supersedes(repo, populated), "a new restriction must trigger export" def test_a_default_positions_does_not_make_every_front_port_differ(self): """The export writes the schema-required positions; it must not re-export the library. @@ -285,6 +304,54 @@ def test_a_non_default_positions_still_differs(self): assert _repo_supersedes(repo, nb) is False + @pytest.mark.parametrize("reverse_repo", [False, True]) + @pytest.mark.parametrize("reverse_netbox", [False, True]) + def test_reordered_netbox_mappings_do_not_trigger_export(self, reverse_repo, reverse_netbox): + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Panel +slug: acme-panel +u_height: 1 +is_full_depth: false +front-ports: + - {name: FP1, type: lc-upc, positions: 2} +port-mappings: + - {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} + - {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 3} +""") + record = DotDict( + id=1, manufacturer=DotDict(name="Acme"), model="Panel", slug="acme-panel", u_height=1, is_full_depth=False + ) + port = DotDict( + name="FP1", + type="lc-upc", + positions=2, + mappings=[ + DotDict(rear_port=DotDict(name="RP2"), front_port_position=2, rear_port_position=3), + DotDict(rear_port=DotDict(name="RP1"), front_port_position=1, rear_port_position=1), + ], + ) + if reverse_repo: + repo["port-mappings"].reverse() + if reverse_netbox: + port.mappings.reverse() + serialized = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + + assert _repo_supersedes(repo, serialized), "unchanged mappings must not trigger export" + repo["port-mappings"].append(dict(repo["port-mappings"][0])) + assert not _repo_supersedes(repo, serialized), "duplicate mappings must keep their multiplicity" + repo["port-mappings"].pop() + port.mappings.append(port.mappings[0]) + duplicated = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, duplicated), "NetBox duplicates must keep their multiplicity" + port.mappings.pop() + port.mappings[0].rear_port_position = 4 + changed = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, changed) + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -1695,10 +1762,42 @@ def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): - """On 4.7 an absent relation means NetBox cleared it, so it must not be resurrected.""" - repo = {"model": "MX304", "module-bays": [{"name": "RE0", "module_bay_types": ["MX304-RE"]}]} - serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0"}]} - - written = self._write(tmp_path, self._item(repo, serialized), supported=True) + """An export selected for a changed description must retain the supported server's answer.""" + from core.graphql_client import DotDict + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +description: old +module-bays: + - {name: Slot 0, position: '0', module_bay_types: [Control]} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme", slug="acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + description="new", + front_image=None, + rear_image=None, + last_updated="2026-01-01T00:00:00Z", + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = True + items = exporter._determine_export_set_for_device_types( + [record], {("acme", "acme-chassis"): repo}, {1: {"module_bay_templates": [bay]}} + ) + assert len(items) == 1 + assert items[0].reason == "differs" + exporter._write_export_items(items, {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Acme" / "Chassis.yaml").read_text()) + assert written["description"] == "new" + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index fdbbf119..95e9452d 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -488,7 +488,7 @@ def test_components_sorted_by_name(self): components = {1: {"interface_templates": [iface_z, iface_a]}} result = serialize_device_type(record, components) names = [i["name"] for i in result["interfaces"]] - assert names == sorted(names) + assert names == ["eth0", "eth9"] class TestRelationSerialization: @@ -635,6 +635,29 @@ def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): ("FP3", 3), ] + def test_mappings_sort_by_numeric_positions_then_rear_port_name(self): + import yaml + + fp = self._front_port( + "FP1", + [ + self._mapping("RP2", "10.0", "2.0"), + self._mapping("RP2", 2, 2), + self._mapping("RP1", "2.0", 2), + self._mapping("RP1", None, None), + ], + positions=2, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == yaml.safe_load(""" +- {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} +- {front_port: FP1, front_port_position: 2, rear_port: RP1, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 10} +""") + def test_a_pre_45_server_still_exports_its_mappings(self): """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" from types import SimpleNamespace diff --git a/tests/test_repo.py b/tests/test_repo.py index 45c819f2..09714d48 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,6 +1,7 @@ import os import pytest +import yaml from unittest.mock import MagicMock, call, mock_open, patch from git import Actor, Repo as GitRepo, exc as git_exc from core.repo import ( @@ -1104,6 +1105,44 @@ def test_null_stanza_is_deleted(self): assert err is None assert "port-mappings" not in data + @pytest.mark.parametrize("stanza", ["", "port-mappings:", "port-mappings: []"]) + def test_only_an_explicit_list_manages_front_port_mappings(self, stanza): + data = yaml.safe_load(f""" +front-ports: + - name: FP1 + type: 8p8c + - name: FP2 + type: 8p8c +{stanza} +""") + + assert normalize_port_mappings(data) is None + assert "port-mappings" not in data + for port in data["front-ports"]: + if stanza == "port-mappings: []": + assert port["_mappings"] == [] + else: + assert "_mappings" not in port + + @pytest.mark.parametrize( + ("stanza", "expected"), + [ + ("RP1", "Error: port-mappings must be a list: 'RP1'"), + ("{FP1: RP1}", "Error: port-mappings must be a list: {'FP1': 'RP1'}"), + ("[RP1]", "Error: port-mappings entry must be a mapping: 'RP1'"), + ], + ) + def test_malformed_stanza_returns_an_error(self, stanza, expected): + data = yaml.safe_load(f""" +front-ports: + - {{name: FP1, type: 8p8c}} +rear-ports: + - {{name: RP1, type: 8p8c}} +port-mappings: {stanza} +""") + + assert normalize_port_mappings(data) == expected + def test_empty_stanza_no_front_ports_still_deleted(self): """Empty port-mappings stanza with no front-ports is cleaned up (not silently skipped).""" data = { @@ -1843,6 +1882,59 @@ def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): class TestAStanzaThatDoesNotListAFrontPort: """A stanza speaks for the whole file, so a port it omits has no mapping.""" + def test_a_nonempty_stanza_clears_an_omitted_front_port_mapping(self): + from types import SimpleNamespace + from core.change_detector import ChangeDetector + + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP1, rear_port: RP1} +""") + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [] + + existing = SimpleNamespace( + name="FP2", + _mappings_canonical=[{"rear_port_name": "RP2", "front_port_position": 1, "rear_port_position": 1}], + ) + detector = ChangeDetector(SimpleNamespace(), LogHandler(False)) + changes = detector._compare_component_properties( + data["front-ports"][1], existing, ["_mappings"], comp_type="front-ports" + ) + assert len(changes) == 1 + assert changes[0].property_name == "_mappings" + assert changes[0].old_value == {("RP2", 1, 1)} + assert changes[0].new_value == set() + + def test_a_half_migrated_file_errors_before_assigning_mappings(self): + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c, rear_port: RP1} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP2, rear_port: RP2} +""") + + assert normalize_port_mappings(data) == ( + "Error: front port 'FP1' declares an inline rear_port but the port-mappings " + "stanza does not list it; the stanza is authoritative, so add 'FP1' to it " + "or remove the inline rear_port keys" + ) + assert all("_mappings" not in port for port in data["front-ports"]) + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): """The old wording blamed a conflict against a stanza that never mentioned the port.""" from core.repo import normalize_port_mappings From c2c16cf813bb4c93fdac29c144e193a5834f6fee Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 20:36:38 +0200 Subject: [PATCH 19/39] fix(mappings): clearing a mapping now obeys --remove-components With the flag off the importer tells the user "will not remove components from existing models". Clearing a front-to-rear linkage removes data, but it reaches NetBox as a `_mappings` property change on an existing component, not as a COMPONENT_REMOVED, so it never passed the gate and applied under a plain --update. `_without_gated_mapping_clears` drops a `_mappings` change whose new value is a strict subset of the old one, which is the case where mappings only go away. Repointing a front port to another rear port still applies, and a component that also carries other property changes keeps them. The filter runs before `_count_actionable_component_changes` at both call sites, so the reported count and the applied changes cannot disagree. Affected ports are named once per type, with the same hint the startup notice uses. This closes a gap that predates the omission change: `port-mappings: []` could already clear every front port without the flag. `update_components` still clears when handed such a change, which is what the two existing write-boundary tests cover. The gate sits above it, with the run configuration, rather than at the write. --- core/netbox_api.py | 47 ++++++++++++++++++++-- tests/test_netbox_api.py | 84 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 544ce538..3321d45d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1,6 +1,7 @@ """NetBox REST and GraphQL API client for importing device and module type libraries.""" from collections import Counter +from dataclasses import replace from contextlib import contextmanager from functools import lru_cache import hashlib @@ -434,6 +435,42 @@ def _image_dir_for_yaml(src_file: str, src_segment: str, dst_segment: str) -> "P # from pynetbox import RequestError as APIRequestError +def _is_mapping_removal(prop_change): + """Return True when a ``_mappings`` change only takes mappings away.""" + return ( + prop_change.property_name == "_mappings" + and isinstance(prop_change.old_value, (set, frozenset)) + and isinstance(prop_change.new_value, (set, frozenset)) + and prop_change.new_value < prop_change.old_value + ) + + +def _without_gated_mapping_clears(changes, remove_components, handle=None): + """Drop mapping clears unless removal is enabled. + + Clearing a front-to-rear linkage deletes data, but it reaches NetBox as a property + change rather than a COMPONENT_REMOVED, so it would otherwise ignore the flag and + contradict the "will not remove components" notice. Filtering here rather than at the + write boundary keeps the actionable count and the applied changes in agreement. + """ + if remove_components: + return changes + kept, gated = [], [] + for change in changes: + remaining = [pc for pc in change.property_changes if not _is_mapping_removal(pc)] + if len(remaining) == len(change.property_changes): + kept.append(change) + continue + gated.append(change.component_name) + if remaining: + kept.append(replace(change, property_changes=remaining)) + if gated and handle is not None: + handle.log( + f"Kept existing port mappings on {sorted(gated)}; use --remove-components with --update to clear them." + ) + return kept + + def _count_actionable_component_changes(changes, remove_components): """Return the count of changes in *changes* that will issue an API call. @@ -1071,7 +1108,10 @@ def _handle_existing_device_type( # Apply component changes component_errors = [] if dt_change.component_changes: - actionable_count = _count_actionable_component_changes(dt_change.component_changes, remove_components) + component_changes = _without_gated_mapping_clears( + dt_change.component_changes, remove_components, self.handle + ) + actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1081,11 +1121,11 @@ def _handle_existing_device_type( self.device_types.update_components( device_type, dt.id, - dt_change.component_changes, + component_changes, parent_type="device", ) if remove_components: - self.device_types.remove_components(dt.id, dt_change.component_changes, parent_type="device") + self.device_types.remove_components(dt.id, component_changes, parent_type="device") after_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1754,6 +1794,7 @@ def _apply_module_type_component_updates( self.device_types.ensure_components_ready(manufacturer_slug=curr_mt["manufacturer"]["slug"]) identity = f"{module_type_res.manufacturer.name}/{module_type_res.model}" component_changes = self.change_detector._compare_components(curr_mt, module_type_res.id, parent_type="module") + component_changes = _without_gated_mapping_clears(component_changes, remove_components, self.handle) if component_changes: actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_updated = self.counter["components_updated"] diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index b2cdbabc..2423e106 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -1,5 +1,6 @@ import os import threading +from types import SimpleNamespace from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -7570,3 +7571,86 @@ def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_setti message = str(exc.value) assert "4.3" in message, f"{version}: the message must name the minimum" assert version in message, f"{version}: the message must name what was found" + + +class TestAMappingClearNeedsTheRemovalFlag: + """Clearing a front-port mapping removes data, so it obeys --remove-components. + + The tool tells the user "will not remove components from existing models" when the flag + is off. A mapping clear reaches NetBox as a COMPONENT_CHANGED property change, so it + bypassed that promise and cleared the linkage under a plain --update. + """ + + def _module_type_losing_a_mapping(self, nb): + """Record an existing FP1 mapped to RP1, and return YAML whose stanza omits it.""" + existing_module = MagicMock() + existing_module.id = 55 + existing_module.manufacturer.name = "Cisco" + existing_module.model = "CM-Map" + + existing_fp = SimpleNamespace( + name="FP1", + _mappings_canonical=[{"rear_port_name": "RP1", "front_port_position": 1, "rear_port_position": 1}], + _mappings_m2m=True, + ) + nb.device_types.components.record("front_port_templates", "module", 55, {"FP1": existing_fp}) + _mark_cache_ready(nb.device_types) + + curr_mt = { + "manufacturer": {"slug": "cisco"}, + "model": "CM-Map", + "slug": "cm-map", + # normalize_port_mappings assigns [] to a front port the stanza omits + "front-ports": [{"name": "FP1", "type": "8p8c", "_mappings": []}], + } + return {"cisco": {"CM-Map": existing_module}}, curr_mt + + def _mapping_clears_sent(self, nb): + """Return the _mappings property changes that reached update_components.""" + from core.change_detector import ChangeType + + sent = [] + for call_args in nb.device_types.update_components.call_args_list: + for change in call_args.args[2]: + if change.change_type is not ChangeType.COMPONENT_CHANGED: + continue + sent += [pc for pc in change.property_changes if pc.property_name == "_mappings"] + return sent + + @pytest.mark.parametrize("remove_components, expected", [(False, 0), (True, 1)]) + def test_the_flag_decides_whether_a_clear_reaches_netbox( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle, remove_components, expected + ): + """Without the flag the clear must never be sent; with it, it must.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=remove_components + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset()] * expected + + def test_a_changed_mapping_still_applies_without_the_flag( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle + ): + """Only removal is gated. Repointing FP1 to another rear port is an ordinary update.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + curr_mt["front-ports"][0]["_mappings"] = [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=False + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset({("RP2", 1, 1)})] From 40e08c0a1fb33b5c5ce29b64d3d6109adacb46b4 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:41:35 +0200 Subject: [PATCH 20/39] chore(lint): widen the ruff rule set and clear what it found (#137) * style: apply the ruff autofix sweep Machine-applied only, no behaviour change: PEP 585/604 builtin generics (List -> list, Optional[X] -> X | None), import ordering, redundant intermediate assignments, composite test assertions split into one assert each, and unused tuple elements renamed to _. The rules that produce these land in the follow-up commit, so this one stays a pure diff to skim and the config arrives with the changes that need judgment. Excluded UP042 (str, Enum -> StrEnum) from the sweep: StrEnum.__str__ returns the bare value where (str, Enum) returns ClassName.MEMBER, which would change logged and serialized output. * chore(lint): widen the ruff rule set and clear what it found Adds the rule families that catch bug classes rather than style: DTZ (timezone-aware datetimes), BLE (blind except), B (bugbear), S (bandit), ERA (commented-out code), PT (pytest style), plus SIM/RET/PERF/C4/RUF/ISC and ten families that were already clean and now stay that way (PIE, FLY, RSE, PGH, TID, INP, A, LOG, G). The changes it required: - Timestamps are timezone-aware. Duration arithmetic uses UTC, which no DST change can skew. Displayed timestamps use .astimezone(), so the operator still reads local wall-clock time; a test pins that, because reaching for UTC here would silently shift every logged line. - SystemExit and the DotDict AttributeError carry their cause. - pytest.raises patterns that were meant literally are escaped. They were matching as regexes, so "bad.url" and a trailing "." accepted more than the test intended. - The _real_library_root fixture is renamed: it returns a value the tests use, so the underscore was telling readers and linters otherwise. - One dead commented import removed, and a commented-out allowlist in repo.py replaced with a line saying why any HTTPS host is accepted. Deliberately not adopted, with the reason recorded in the config: UP042 (StrEnum changes __str__ output), ISC001 and COM812 (the formatter owns that), SIM117 and A002 in tests (style only). The twelve blind excepts are deliberate defensive boundaries around futures, pynetbox calls and YAML parsing. Narrowing them changes runtime behaviour and needs its own analysis, so each is marked with the reason it exists. The rule still applies to new code. ANN, PTH, FBT, TRY003/EM and the PLR refactor rules stay off: ~1000 findings between them, all churn or wide-radius refactors, none of them a bug class this code has hit. * fix(import): normalize a naive started_at at construction ImportRun stored whatever started_at it was given. RunSummary.capture then subtracts it from datetime.now(UTC), so a naive value raised TypeError at the end of the run rather than at the boundary that accepted it. Attach the local timezone when the value is naive, which is what Python means by a naive datetime, and leave an aware value untouched. The field only feeds elapsed-time reporting, so rejecting a plain datetime.now() would cost callers more than the conversion does. Also replaces the endpoint sampling in the log timestamp test with a frozen clock. It read the clock three times and asserted the middle read matched one of the outer two, which a scheduler delay between reads could break. The stub models a real clock (naive now() is local, now(tz) converts the same instant) rather than using replace(), which reinterprets the wall clock and made the test pass against a UTC implementation too. Review findings: https://github.com/marcinpsk/Device-Type-Library-Import/pull/137#discussion_r3969536115 https://github.com/marcinpsk/Device-Type-Library-Import/pull/137#discussion_r3969536124 * chore(lint): satisfy the widened rules for code added on develop The wider rule set lands on this branch, so tests added on develop after it was cut are the first to break it. Import blocks inside three test functions are sorted, and the module-type query assertion binds its expected text to a name. SIM905 wanted that literal replaced with a list of tokens, which would leave the expected GraphQL unreadable. Binding it first keeps the query legible and the rule satisfied. --- core/change_detector.py | 36 +++---- core/compat.py | 2 +- core/component_cache.py | 7 +- core/component_registry.py | 3 +- core/export.py | 38 ++++---- core/graphql_client.py | 6 +- core/import_run.py | 19 ++-- core/log_handler.py | 6 +- core/nb_serializer.py | 7 +- core/netbox_api.py | 65 ++++++------- core/outcomes.py | 29 +++--- core/repo.py | 29 +++--- core/update_failure_resolver.py | 23 ++--- nb-dt-import.py | 38 ++++---- pyproject.toml | 30 +++++- tests/conftest.py | 3 +- tests/integration/test_import.py | 22 ++--- tests/test_change_detector.py | 16 ++-- tests/test_component_cache.py | 11 ++- tests/test_config.py | 5 +- tests/test_docker_workflow.py | 1 - tests/test_errors.py | 6 +- tests/test_export_manifest.py | 3 +- tests/test_exporter.py | 8 +- tests/test_graphql_client.py | 58 ++++++------ tests/test_import_run.py | 88 +++++++++++++---- tests/test_log_handler.py | 35 ++++++- tests/test_module_bay_type_sync.py | 5 +- tests/test_module_bay_types.py | 17 ++-- tests/test_nb_dt_import.py | 94 ++++++++++--------- tests/test_netbox_api.py | 130 ++++++++++++++++---------- tests/test_normalization.py | 16 ++-- tests/test_repo.py | 75 +++++++-------- tests/test_suite_hygiene.py | 9 +- tests/test_update_failure_resolver.py | 3 +- 35 files changed, 547 insertions(+), 396 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 5e2ecf78..57ca4d3f 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -6,13 +6,13 @@ import os from dataclasses import dataclass, field -from functools import lru_cache -from typing import Any, List, Optional from enum import Enum +from functools import lru_cache +from typing import Any from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES -from core.normalization import is_explicit_list, normalize_values from core.formatting import log_property_diffs +from core.normalization import is_explicit_list, normalize_values from core.schema_reader import load_properties_for_type @@ -150,7 +150,7 @@ class ComponentChange: component_type: str # e.g., "interfaces", "power-ports" component_name: str change_type: ChangeType - property_changes: List[PropertyChange] = field(default_factory=list) + property_changes: list[PropertyChange] = field(default_factory=list) @dataclass @@ -161,9 +161,9 @@ class DeviceTypeChange: model: str slug: str is_new: bool = False - property_changes: List[PropertyChange] = field(default_factory=list) - component_changes: List[ComponentChange] = field(default_factory=list) - netbox_id: Optional[int] = None + property_changes: list[PropertyChange] = field(default_factory=list) + component_changes: list[ComponentChange] = field(default_factory=list) + netbox_id: int | None = None @property def has_changes(self) -> bool: @@ -180,8 +180,8 @@ def has_updates(self) -> bool: class ChangeReport: """Aggregated change report for all device types.""" - new_device_types: List[DeviceTypeChange] = field(default_factory=list) - modified_device_types: List[DeviceTypeChange] = field(default_factory=list) + new_device_types: list[DeviceTypeChange] = field(default_factory=list) + modified_device_types: list[DeviceTypeChange] = field(default_factory=list) unchanged_count: int = 0 @@ -257,7 +257,7 @@ def __init__(self, device_types_instance, handle, remove_unmanaged_types: bool = self.verbose = verbose self.remove_unmanaged_types = remove_unmanaged_types - def detect_changes(self, device_types: List[dict], progress=None) -> ChangeReport: + def detect_changes(self, device_types: list[dict], progress=None) -> ChangeReport: """Analyze all device types and generate a change report. Args: @@ -308,7 +308,7 @@ def detect_changes(self, device_types: List[dict], progress=None) -> ChangeRepor return report - def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> List[PropertyChange]: + def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> list[PropertyChange]: """Compare YAML device type properties against NetBox device type. Args: @@ -349,7 +349,7 @@ def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> List[Pr return changes @staticmethod - def _compare_image_properties(yaml_data: dict, netbox_dt) -> List[PropertyChange]: + def _compare_image_properties(yaml_data: dict, netbox_dt) -> list[PropertyChange]: """Compare image properties between YAML and NetBox device type. YAML uses boolean flags (front_image: true) meaning "an image should exist", @@ -388,7 +388,7 @@ def _compare_components( yaml_data: dict, device_type_id: int, parent_type: str = "device", - ) -> List[ComponentChange]: + ) -> list[ComponentChange]: """Compare all components between YAML and cached NetBox data. Args: @@ -418,9 +418,9 @@ def _compare_components( # same as an empty list so chassis YAMLs that omit (e.g.) interfaces can # still drive cleanup of stale templates in NetBox. if yaml_key in yaml_data or self.remove_unmanaged_types: - for existing_name in existing_components.keys(): + for existing_name in existing_components: if existing_name not in yaml_component_names: - changes.append( + changes.append( # noqa: PERF401 ComponentChange( component_type=yaml_key, component_name=existing_name, @@ -481,10 +481,10 @@ def _compare_component_properties( self, yaml_comp: dict, netbox_comp, - properties: List[str], + properties: list[str], comp_type: str = "", manufacturer: str = "", - ) -> List[PropertyChange]: + ) -> list[PropertyChange]: """Compare properties between YAML and NetBox component. *manufacturer* is the owning manufacturer's slug, used to resolve a relation @@ -631,7 +631,7 @@ def _log_modified_summary(self, report: ChangeReport) -> None: if parts: self.handle.log(f" Breakdown: {', '.join(parts)}") - def _log_property_diffs(self, prop_changes: List[PropertyChange], indent: str) -> None: + def _log_property_diffs(self, prop_changes: list[PropertyChange], indent: str) -> None: """Emit diff-u style lines for *prop_changes* at the given *indent*.""" log_property_diffs( [(pc.property_name, pc.old_value, pc.new_value) for pc in prop_changes], diff --git a/core/compat.py b/core/compat.py index 8ebcbf03..2b35eb0a 100644 --- a/core/compat.py +++ b/core/compat.py @@ -19,7 +19,7 @@ def parse_netbox_version(version) -> tuple[int, int]: suffixes NetBox ships ("4.7.0-beta2"). """ raw = [int(re.sub(r"\D.*", "", part.strip()) or "0") for part in str(version).split(".")] - return tuple((raw + [0, 0])[:2]) # type: ignore[return-value] + return tuple(([*raw, 0, 0])[:2]) # type: ignore[return-value] def supports_module_bay_types(version) -> bool: diff --git a/core/component_cache.py b/core/component_cache.py index f314ecf7..61ddbe83 100644 --- a/core/component_cache.py +++ b/core/component_cache.py @@ -304,10 +304,7 @@ def get(self, endpoint_name, parent_type, parent_id, endpoint): if key in cached: return cached[key] - if parent_type == "device": - filter_kwargs = {"device_type_id": parent_id} - else: - filter_kwargs = {"module_type_id": parent_id} + filter_kwargs = {"device_type_id": parent_id} if parent_type == "device" else {"module_type_id": parent_id} result = {item.name: item for item in endpoint.filter(**filter_kwargs)} self.record(endpoint_name, parent_type, parent_id, result) return result @@ -378,7 +375,7 @@ def _finish_endpoint(self, endpoint_name, future): """Mark *endpoint_name* complete on the display, sizing the bar to the result.""" try: total = max(len(future.result()), 1) - except Exception: + except Exception: # a worker error must not break the progress display # noqa: BLE001 total = 1 self._job["display"].finish(endpoint_name, total) self._job["done"].add(endpoint_name) diff --git a/core/component_registry.py b/core/component_registry.py index 8fb4e474..10609fb6 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -10,7 +10,6 @@ """ from dataclasses import dataclass, field -from typing import Optional # What a create call must resolve from a name to a NetBox id before it can POST. LINK_BRIDGE = "bridge" @@ -41,7 +40,7 @@ class ComponentType: relations: tuple[str, ...] = field(default_factory=tuple) graphql_extra: tuple[str, ...] = field(default_factory=tuple) compare_extra: tuple[str, ...] = field(default_factory=tuple) - link: Optional[str] = None + link: str | None = None @property def graphql_fields(self): diff --git a/core/export.py b/core/export.py index 9dd48507..d08b2d4c 100644 --- a/core/export.py +++ b/core/export.py @@ -3,13 +3,15 @@ Entry point: ``Exporter(config, handle, export_dir, force_overwrite, vendor_slugs).run()`` """ +import contextlib import hashlib import os import re import threading +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import Any, List, Optional, Sequence +from typing import Any import requests import yaml @@ -108,7 +110,7 @@ class ExportItem: kind: str # "device-type" | "module-type" | "rack-type" nb_record: Any - repo_yaml: Optional[dict] # None when absent from repo + repo_yaml: dict | None # None when absent from repo serialized: dict # What we will write reason: str # "absent" | "differs" | "images-missing" mfr_name: str @@ -248,7 +250,7 @@ def _is_subset(sub: Any, sup: Any) -> bool: class Exporter: """Exports NetBox device/module/rack types to a local directory in DTL format.""" - def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendor_slugs: Optional[Sequence[str]]): + def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendor_slugs: Sequence[str] | None): """Initialize the Exporter from the resolved run configuration.""" self.config = config self.handle = handle @@ -270,7 +272,7 @@ def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendo handle=handle, page_size=config.graphql_page_size, ) - self._module_image_details: Optional[dict] = None + self._module_image_details: dict | None = None def _get_module_image_details(self) -> dict: """Return module type image details, fetching from NetBox at most once per run.""" @@ -297,7 +299,7 @@ def run(self, progress=None) -> None: self.graphql.detect_module_bay_type_support() # ── Fetch all types from NetBox ────────────────────────────────────── - by_model, by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) + by_model, _by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) all_mt = self.graphql.get_module_types(manufacturer_slugs=self.vendor_slugs) all_rt = self.graphql.get_rack_types(manufacturer_slugs=self.vendor_slugs) @@ -371,9 +373,9 @@ def _compare_vendors_to_items( repo_dt_by_slug, repo_mt_by_key, progress, - ) -> tuple[List[ExportItem], int]: + ) -> tuple[list[ExportItem], int]: """Compare stale device/module types per vendor and return export items.""" - items: List[ExportItem] = [] + items: list[ExportItem] = [] skipped_fresh = 0 compare_task = ( progress.add_task("Comparing vendors", total=len(all_vendor_slugs)) @@ -441,9 +443,9 @@ def _compare_vendors_to_items( return items, skipped_fresh - def _compare_racks_to_items(self, all_rt, manifest, repo_rt_by_key, progress) -> tuple[List[ExportItem], int]: + def _compare_racks_to_items(self, all_rt, manifest, repo_rt_by_key, progress) -> tuple[list[ExportItem], int]: """Compare stale rack types and return export items.""" - items: List[ExportItem] = [] + items: list[ExportItem] = [] skipped_fresh = 0 rack_records = [record for models in all_rt.values() for record in models.values()] rack_task = ( @@ -698,10 +700,8 @@ def _fetch_one(endpoint_name): results = list(pool.map(_fetch_one, COMPONENT_ENDPOINT_NAMES)) finally: for client in _clients: - try: + with contextlib.suppress(Exception): client.close() - except Exception: - pass for endpoint_name, records in results: for rec in records: @@ -717,7 +717,7 @@ def _fetch_one(endpoint_name): def _determine_export_set_for_device_types( self, nb_records: list, repo_dt_by_slug: dict, components_by_dt_id: dict - ) -> List[ExportItem]: + ) -> list[ExportItem]: """Build the list of device types that need exporting to the repo. Includes records absent from the repo, those whose serialized form @@ -733,7 +733,7 @@ def _determine_export_set_for_device_types( manifest_key = f"{mfr_name}/{rec.slug}" repo_yaml = repo_dt_by_slug.get((mfr_slug, rec.slug)) - reason: Optional[str] + reason: str | None if repo_yaml is None: reason = "absent" elif _repo_supersedes(repo_yaml, serialized): @@ -759,7 +759,7 @@ def _determine_export_set_for_device_types( def _determine_export_set_for_module_types( self, nb_records: list, repo_mt_by_key: dict, components_by_mt_id: dict - ) -> List[ExportItem]: + ) -> list[ExportItem]: """Build the list of module types that need exporting to the repo. Includes records absent from the repo and those whose serialized form @@ -796,7 +796,7 @@ def _determine_export_set_for_module_types( ) return items - def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: dict) -> List[ExportItem]: + def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: dict) -> list[ExportItem]: """Build the list of rack types that need exporting to the repo. Includes records absent from the repo and those whose serialized form @@ -833,7 +833,7 @@ def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: ) return items - def _check_missing_images(self, front_url, rear_url, mfr_name: str, slug: str) -> Optional[str]: + def _check_missing_images(self, front_url, rear_url, mfr_name: str, slug: str) -> str | None: """Return ``'images-missing'`` if any expected local image is absent; else None. DTL stores images under ``elevation-images//.{front,rear}.{png,jpg,jpeg,gif}`` @@ -874,7 +874,7 @@ def _download_type_images(self, item: ExportItem) -> bool: """Download images for *item*. Returns True if all downloads succeeded.""" if item.kind == "device-type": return self._download_device_type_images(item) - elif item.kind == "module-type": + if item.kind == "module-type": return self._download_module_type_images(item) return True # rack types have no images @@ -975,7 +975,7 @@ def _download_module_type_images(self, item: ExportItem) -> bool: return ok def _download_image( - self, url_path: str, dest: Path, content_type_out: "Optional[list]" = None + self, url_path: str, dest: Path, content_type_out: "list | None" = None ) -> "str | _SkipSentinel | None": """Download an image from NetBox and write to *dest*. diff --git a/core/graphql_client.py b/core/graphql_client.py index a188ea6e..d66c376a 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -56,7 +56,8 @@ def __getattr__(self, key): try: value = self[key] except KeyError: - raise AttributeError(f"'DotDict' has no attribute '{key}'") + # The dict miss is an implementation detail of attribute lookup, not the error. + raise AttributeError(f"'DotDict' has no attribute '{key}'") from None if isinstance(value, dict) and not isinstance(value, DotDict): value = DotDict(value) self[key] = value @@ -114,7 +115,7 @@ def _response_body_detail(response): return "" try: body = response.text.strip() - except Exception: # pragma: no cover - a body that cannot be decoded is not worth failing on + except Exception: # pragma: no cover - a body that cannot be decoded is not worth failing on # noqa: BLE001 return "" if not body: return "" @@ -299,6 +300,7 @@ def query(self, graphql_query, variables=None, _retries=3): raise GraphQLSchemaError(messages) return body.get("data", {}) + return None def query_all(self, graphql_query, list_key, page_size=None, variables=None, on_page=None): """Auto-paginate a GraphQL list query using offset/limit. diff --git a/core/import_run.py b/core/import_run.py index cdac353b..c99a6ef0 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -1,22 +1,26 @@ """Import pipeline planning and execution.""" +import os from collections import Counter from contextlib import contextmanager from dataclasses import dataclass -from datetime import datetime, timedelta -import os +from datetime import UTC, datetime, timedelta from typing import Any -from core.change_detector import ChangeDetector, ChangeType, IMAGE_PROPERTIES +from core.change_detector import IMAGE_PROPERTIES, ChangeDetector, ChangeType from core.component_cache import NullTaskDisplay, RichTaskDisplay from core.config import RunConfig from core.errors import VendorSelectionError from core.outcomes import EntityKind, Outcome - _PROGRESS_DESC_WIDTH = 28 +def _as_aware(moment): + """Return *moment* with a timezone attached; a naive value means local time.""" + return moment if moment.tzinfo is not None else moment.astimezone() + + @dataclass(frozen=True) class RunSelection: """Selected vendors and source paths for one import run.""" @@ -67,7 +71,7 @@ def capture(cls, netbox, repo, started_at): outcome_counts=netbox.outcomes.summary_by_kind(), failure_lines=tuple(netbox.outcomes.render_failure_report()), duplicate_definitions=tuple(repo.duplicate_definitions), - elapsed=datetime.now() - started_at, + elapsed=datetime.now(UTC) - started_at, ) def outcome_count(self, kind, outcome): @@ -580,7 +584,7 @@ def __init__(self, config, repo, netbox, reporter, progress_factory, *, started_ netbox (NetBox): Connected NetBox interface. reporter (LogHandler): Run message sink. progress_factory: Context manager factory for the Rich progress display. - started_at (datetime | None): Start time used for elapsed-time reporting. + started_at (datetime | None): Start time for elapsed-time reporting; naive means local. """ if not isinstance(config, RunConfig): raise TypeError("config must be a RunConfig") @@ -589,7 +593,8 @@ def __init__(self, config, repo, netbox, reporter, progress_factory, *, started_ self.netbox = netbox self.reporter = reporter self.progress_factory = progress_factory - self.started_at = started_at or datetime.now() + # Normalize here: a naive value would otherwise survive the run and raise in capture(). + self.started_at = _as_aware(started_at) if started_at is not None else datetime.now(UTC) self.progress: Any = None self.task_registry = None self.vendor_task_id = None diff --git a/core/log_handler.py b/core/log_handler.py index 2a53eab9..3842cd3c 100644 --- a/core/log_handler.py +++ b/core/log_handler.py @@ -19,7 +19,7 @@ def __init__(self, verbose: bool): def _timestamp(self): """Return the current time formatted as HH:MM:SS.""" - return datetime.now().strftime("%H:%M:%S") + return datetime.now().astimezone().strftime("%H:%M:%S") def set_console(self, console): """Set the Rich Console instance used for output, or None to fall back to print().""" @@ -74,6 +74,6 @@ def log_ports_created(self, created_ports, parent_type: str, port_type: str = "p for port in created_ports: self.verbose_log( f"{port_type} Template Created: {port.name} - " - + f"{port.type if hasattr(port, 'type') else ''} - {getattr(port, parent_attribute).id} - " - + f"{port.id}" + f"{port.type if hasattr(port, 'type') else ''} - {getattr(port, parent_attribute).id} - " + f"{port.id}" ) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 4977553b..9ca1094a 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -4,7 +4,8 @@ comparison against existing repo YAML files. """ -from typing import Any, Sequence +from collections.abc import Sequence +from typing import Any from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS @@ -107,9 +108,7 @@ def _should_include(field: str, val: Any) -> bool: return False if isinstance(val, str) and val == "": return False - if field in _OMIT_IF_EQUAL and val == _OMIT_IF_EQUAL[field]: - return False - return True + return not (field in _OMIT_IF_EQUAL and val == _OMIT_IF_EQUAL[field]) def _serialize_component(record: Any, fields: Sequence[str]) -> dict: diff --git a/core/netbox_api.py b/core/netbox_api.py index 3321d45d..02f2566c 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1,21 +1,23 @@ """NetBox REST and GraphQL API client for importing device and module type libraries.""" -from collections import Counter -from dataclasses import replace -from contextlib import contextmanager -from functools import lru_cache +import glob import hashlib import json +import os import tempfile import time +from collections import Counter +from contextlib import contextmanager, suppress +from dataclasses import replace +from functools import lru_cache +from pathlib import Path +from typing import Any + import pynetbox import requests -import os -import glob -from pathlib import Path -from typing import Any, Optional from core.change_detector import ChangeDetector, ChangeType +from core.compat import parse_netbox_version, supports_module_bay_types from core.component_cache import ComponentCache from core.component_registry import ( BY_YAML_KEY, @@ -26,9 +28,8 @@ MODULE_TYPE_COMPONENTS, MODULE_TYPE_RELATIONS, ) -from core.compat import parse_netbox_version, supports_module_bay_types -from core.formatting import log_property_diffs from core.errors import FatalError, UnknownError +from core.formatting import log_property_diffs from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.normalization import values_equal from core.outcomes import EntityKind, Outcome, OutcomeRegistry @@ -247,7 +248,7 @@ def _is_image_hash_changed(local_path: str, hash_cache: dict, log_fn=None) -> bo return current != cached -def _load_image_hash_cache(path: Optional[str], log_fn=None) -> dict: +def _load_image_hash_cache(path: str | None, log_fn=None) -> dict: """Load the image-hash cache from *path* (JSON), returning an empty dict when it cannot be read. An absent file is the normal first run and stays quiet. Anything else means the @@ -298,10 +299,8 @@ def _save_image_hash_cache(path: str, cache: dict) -> bool: return True except OSError: if tmp_path is not None: - try: + with suppress(OSError): os.unlink(tmp_path) - except OSError: - pass return False @@ -360,6 +359,7 @@ def _retry_on_connection_error(func, *args, **kwargs): raise wait = _RETRY_BACKOFF[attempt] if attempt < len(_RETRY_BACKOFF) else _RETRY_BACKOFF[-1] time.sleep(wait) + return None # Module type scalar properties that can be compared and updated. @@ -432,9 +432,6 @@ def _image_dir_for_yaml(src_file: str, src_segment: str, dst_segment: str) -> "P return Path(*parts) -# from pynetbox import RequestError as APIRequestError - - def _is_mapping_removal(prop_change): """Return True when a ``_mappings`` change only takes mappings away.""" return ( @@ -542,7 +539,7 @@ def __init__(self, config, handle): _cache_dir = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "nb-dt-import" try: _cache_dir.mkdir(parents=True, exist_ok=True) - self._image_hash_cache_path: Optional[str] = str(_cache_dir / "image-hashes.json") + self._image_hash_cache_path: str | None = str(_cache_dir / "image-hashes.json") except OSError as exc: self.handle.log( "[yellow]Warning: could not create image hash cache directory " @@ -800,7 +797,7 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er device_type_id=dt.id, device_type_yaml=device_type, ) - except Exception as exc: # defensive: classifier must never break the run + except Exception as exc: # defensive: classifier must never break the run # noqa: BLE001 self.handle.verbose_log(f"Failure classifier raised {type(exc).__name__}: {exc}") return False, None @@ -833,7 +830,7 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er try: for step in resolution.remediation_steps: step() - except Exception as exc: + except Exception as exc: # a failed remediation is reported, not raised # noqa: BLE001 self.handle.log(f"Auto-resolve failed for {dt.model}: {exc}") return False, resolution @@ -1476,11 +1473,11 @@ def filter_new_module_types(module_types, all_module_types): Returns: list[dict]: Module types not found in *all_module_types*. """ - new_module_types = [] - for module_type in module_types: - if NetBox._find_existing_module_type(module_type, all_module_types) is None: - new_module_types.append(module_type) - return new_module_types + return [ + module_type + for module_type in module_types + if NetBox._find_existing_module_type(module_type, all_module_types) is None + ] def _log_module_property_diffs(self, mfr_slug, model, fields_info, component_changes=None): """Emit diff-u style lines for changed module type properties and component changes. @@ -1900,7 +1897,7 @@ def _process_single_module_type( if module_type_res is not None: self.handle.verbose_log( f"Module Type Cached: {module_type_res.manufacturer.name} - " - + f"{module_type_res.model} - {module_type_res.id}" + f"{module_type_res.model} - {module_type_res.id}" ) # Upload images before the scalar PATCH so attachments are created # even if the property update later fails (module already exists in @@ -1940,7 +1937,7 @@ def _process_single_module_type( all_module_types.setdefault(manufacturer_slug, {})[curr_mt["model"]] = module_type_res self.handle.verbose_log( f"Module Type Created: {module_type_res.manufacturer.name} - " - + f"{module_type_res.model} - {module_type_res.id}" + f"{module_type_res.model} - {module_type_res.id}" ) except pynetbox.RequestError as excep: self.handle.log(f"Error creating Module Type: {excep.error} (Context: {src_file})") @@ -2283,7 +2280,7 @@ class _FrontPortRecordWithMappings: All other attribute accesses are forwarded to the underlying record. """ - __slots__ = ("_record", "_mappings_canonical", "_mappings_m2m") + __slots__ = ("_mappings_canonical", "_mappings_m2m", "_record") def __init__(self, record): """Wrap *record* and pre-compute a canonical mappings list for ChangeDetector compatibility. @@ -2294,7 +2291,7 @@ def __init__(self, record): """ object.__setattr__(self, "_record", record) mappings_raw = getattr(record, "mappings", None) - canonical: Optional[list] + canonical: list | None if mappings_raw is not None: # NetBox >= 4.5: mappings is a list of PortTemplateMapping objects canonical = [] @@ -2511,7 +2508,7 @@ def _create_generic( payload = extract_error_payload(excep.error) per_item = payload if isinstance(payload, list) and len(payload) == len(to_create) else [] reported = 0 - for item, error in zip(to_create, per_item): + for item, error in zip(to_create, per_item, strict=False): if error: reported += 1 self._log_component_error( @@ -3154,7 +3151,7 @@ def upload_images(self, baseurl, token, images, device_type): file_handles = {} try: for field, path in images.items(): - file_handles[field] = (os.path.basename(path), open(path, "rb")) + file_handles[field] = (os.path.basename(path), open(path, "rb")) # noqa: SIM115 response = requests.patch( url, headers=headers, @@ -3172,11 +3169,9 @@ def upload_images(self, baseurl, token, images, device_type): except OSError as e: self.handle.log(f"Error reading image file for device type {device_type}: {e}") finally: - for _, (_, fh) in file_handles.items(): - try: + for _, fh in file_handles.values(): + with suppress(Exception): fh.close() - except Exception: - pass def upload_image_attachment(self, baseurl, token, image_path, object_type, object_id): """Upload an image as an Image Attachment to a NetBox object. diff --git a/core/outcomes.py b/core/outcomes.py index cf0b75e8..a9a9c798 100644 --- a/core/outcomes.py +++ b/core/outcomes.py @@ -16,7 +16,6 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List, Optional class EntityKind(str, Enum): @@ -48,9 +47,9 @@ class OutcomeRecord: kind: EntityKind identity: str # human-readable identifier (e.g. "Supermicro/SuperServer-6028TR-HTR") outcome: Outcome - reason: Optional[str] = None - blocking_objects: List[str] = field(default_factory=list) - hint: Optional[str] = None + reason: str | None = None + blocking_objects: list[str] = field(default_factory=list) + hint: str | None = None class OutcomeRegistry: @@ -62,7 +61,7 @@ class OutcomeRegistry: def __init__(self) -> None: """Initialise an empty registry.""" - self._records: List[OutcomeRecord] = [] + self._records: list[OutcomeRecord] = [] def record( self, @@ -70,9 +69,9 @@ def record( identity: str, outcome: Outcome, *, - reason: Optional[str] = None, - blocking_objects: Optional[List[str]] = None, - hint: Optional[str] = None, + reason: str | None = None, + blocking_objects: list[str] | None = None, + hint: str | None = None, ) -> None: """Append a new outcome record.""" self._records.append( @@ -87,27 +86,27 @@ def record( ) @property - def records(self) -> List[OutcomeRecord]: + def records(self) -> list[OutcomeRecord]: """All recorded outcomes (read-only view).""" return list(self._records) - def failures(self) -> List[OutcomeRecord]: + def failures(self) -> list[OutcomeRecord]: """Return only the FAILED records.""" return [r for r in self._records if r.outcome == Outcome.FAILED] - def partials(self) -> List[OutcomeRecord]: + def partials(self) -> list[OutcomeRecord]: """Return only the PARTIAL records (some but not all changes applied).""" return [r for r in self._records if r.outcome == Outcome.PARTIAL] - def summary_by_kind(self) -> Dict[EntityKind, Dict[Outcome, int]]: + def summary_by_kind(self) -> dict[EntityKind, dict[Outcome, int]]: """Aggregate counts grouped by ``(kind, outcome)``.""" - agg: Dict[EntityKind, Dict[Outcome, int]] = {} + agg: dict[EntityKind, dict[Outcome, int]] = {} for r in self._records: agg.setdefault(r.kind, {}).setdefault(r.outcome, 0) agg[r.kind][r.outcome] += 1 return agg - def render_failure_report(self) -> List[str]: + def render_failure_report(self) -> list[str]: """Render a multi-line operator-facing failure report. Returns an empty list when no failures or partials were recorded. @@ -118,7 +117,7 @@ def render_failure_report(self) -> List[str]: if not failures and not partials: return [] - lines: List[str] = [] + lines: list[str] = [] lines.append("=" * 60) lines.append("FAILED / PARTIAL UPDATE REPORT") lines.append("=" * 60) diff --git a/core/repo.py b/core/repo.py index 62472f7a..4c368911 100644 --- a/core/repo.py +++ b/core/repo.py @@ -4,12 +4,13 @@ import json import os import pickle +from collections.abc import Sequence from glob import glob from re import sub as re_sub -from typing import Optional, Sequence from urllib.parse import urlparse -from git import Repo, exc + import yaml +from git import Repo, exc from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError @@ -79,7 +80,7 @@ def find_class(self, module, name): _INDEX_MAX_BYTES = 10 * 1024 * 1024 # 10 MiB — DTL index files are typically <1 MiB -def _resolve_index_path(base_dir: str, stem: str) -> "Optional[str]": +def _resolve_index_path(base_dir: str, stem: str) -> "str | None": """Return the path to a DTL index file, preferring the JSON form over the legacy pickle. Upstream DTL replaced ``tests/known-*.pickle`` with ``tests/known-*.json`` (GHSA-492p-5wp7-2w7c). @@ -96,8 +97,8 @@ def _resolve_index_path(base_dir: str, stem: str) -> "Optional[str]": def _vendor_slugs_from_index( - index_path: "Optional[str]", slugs_lower: list, slug_format, subdir_filter: "Optional[str]" = None -) -> "Optional[set]": + index_path: "str | None", slugs_lower: list, slug_format, subdir_filter: "str | None" = None +) -> "set | None": """Load a (model_name, vendor_dir) index and return the set of vendor slugs matching *slugs_lower*. *index_path* may point at a ``.json`` (current) or ``.pickle`` (legacy) file; the loader is @@ -109,7 +110,7 @@ def _vendor_slugs_from_index( return None try: entries = _safe_index_load(index_path) - except Exception: + except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return None result = set() for model_name, vendor_dir in entries: @@ -122,7 +123,7 @@ def _vendor_slugs_from_index( return result -def _safe_abs_path(repo_root: str, relpath: str) -> "Optional[str]": +def _safe_abs_path(repo_root: str, relpath: str) -> "str | None": """Return the absolute path for *relpath* inside *repo_root*, or None if it escapes the root.""" abs_path = os.path.normpath(os.path.join(repo_root, *relpath.replace("\\", "/").split("/"))) return abs_path if abs_path.startswith(os.path.normpath(repo_root) + os.sep) else None @@ -215,9 +216,7 @@ def validate_git_url(url): if url.startswith("https://"): parsed = urlparse(url) if parsed.scheme == "https" and parsed.hostname: - # Optional: enforce an allowlist if desired - # if parsed.hostname not in ("github.com", "gitlab.com"): - # return False, f"Host not allowed: {parsed.hostname}" + # Any HTTPS host is accepted: the library is forked and self-hosted widely. return True, None return False, "Invalid HTTPS URL" @@ -434,7 +433,7 @@ def parse_single_file(file): `src` set to the file path. str: Error string beginning with "Error:" describing YAML parsing or other failure. """ - with open(file, "r") as stream: + with open(file) as stream: try: data = yaml.safe_load(stream) manufacturer = data["manufacturer"] @@ -450,7 +449,7 @@ def parse_single_file(file): return data except yaml.YAMLError as excep: return f"Error: {excep}" - except Exception as e: + except Exception as e: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return f"Error: {e}" @@ -625,7 +624,7 @@ def clone_repo(self): except Exception as git_error: raise UnknownError("Git Repository Error", cause=git_error) from git_error - def get_devices(self, base_path, vendors: Optional[Sequence[str]] = None): + def get_devices(self, base_path, vendors: Sequence[str] | None = None): """Discover device YAML files and vendor directories under a base path. Args: @@ -702,7 +701,7 @@ def resolve_slug_files(self, slugs): device_files: dict = {} # vendor_slug -> [abs_path] try: known_slugs = _safe_index_load(device_index) - except Exception: + except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return None for entry_slug, relpath in known_slugs: @@ -769,7 +768,7 @@ def discover_vendors(self, devices_path, modules_path, racks_path): # Return sorted list by slug return sorted(vendors_dict.values(), key=lambda v: v["slug"]) - def parse_files(self, files: list, slugs: Optional[Sequence[str]] = None, progress=None): + def parse_files(self, files: list, slugs: Sequence[str] | None = None, progress=None): """Parse YAML device files into device type dicts, optionally filtering and tracking progress. Args: diff --git a/core/update_failure_resolver.py b/core/update_failure_resolver.py index 82543a5d..205610e4 100644 --- a/core/update_failure_resolver.py +++ b/core/update_failure_resolver.py @@ -20,9 +20,10 @@ from __future__ import annotations import json +from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from typing import Any, Callable, List, Optional +from typing import Any class FailureKind(str, Enum): @@ -57,10 +58,10 @@ class FailureResolution: kind: FailureKind description: str = "" - blocking_objects: List[str] = field(default_factory=list) - dependent_devices_count: Optional[int] = None - dependent_devices_sample: List[str] = field(default_factory=list) - remediation_steps: List[Callable[[], None]] = field(default_factory=list) + blocking_objects: list[str] = field(default_factory=list) + dependent_devices_count: int | None = None + dependent_devices_sample: list[str] = field(default_factory=list) + remediation_steps: list[Callable[[], None]] = field(default_factory=list) hint: str = "" @property @@ -91,7 +92,7 @@ def extract_error_payload(error: Any) -> Any: if isinstance(error, (bytes, bytearray)): try: error = error.decode("utf-8", errors="replace") - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return error if isinstance(error, str): try: @@ -120,7 +121,7 @@ def _matches_subdevice_role_constraint(payload: Any) -> bool: return False -def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, List[str]]: +def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, list[str]]: """Query NetBox for devices using *device_type_id*. Returns ``(count, sample_names)`` where ``sample_names`` is up to 5 names @@ -135,7 +136,7 @@ def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, Lis filter_kwargs = {"device_type_id": device_type_id} try: devices = list(netbox.dcim.devices.filter(**filter_kwargs, limit=5)) - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return -1, [] sample = [getattr(d, "name", None) or str(getattr(d, "id", "?")) for d in devices[:5]] if len(devices) < 5: @@ -143,12 +144,12 @@ def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, Lis # We capped at limit=5; query the real total separately. try: total = netbox.dcim.devices.count(**filter_kwargs) - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 total = len(devices) return total, sample -def _list_device_bay_templates(netbox: Any, device_type_id: int) -> Optional[List[Any]]: +def _list_device_bay_templates(netbox: Any, device_type_id: int) -> list[Any] | None: """Return all ``DeviceBayTemplate`` records attached to *device_type_id*. Returns ``None`` when the NetBox query itself fails (network error, 5xx, etc.) @@ -160,7 +161,7 @@ def _list_device_bay_templates(netbox: Any, device_type_id: int) -> Optional[Lis """ try: return list(netbox.dcim.device_bay_templates.filter(device_type_id=device_type_id)) - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return None diff --git a/nb-dt-import.py b/nb-dt-import.py index 6d53f2da..cbd5cb68 100644 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Import NetBox device and module types from the community library.""" -from contextlib import contextmanager -from datetime import datetime import sys +from contextlib import contextmanager +from datetime import UTC, datetime -from pynetbox.core.query import RequestError as NetBoxRequestError import requests +from pynetbox.core.query import RequestError as NetBoxRequestError from rich.panel import Panel from rich.progress import ( BarColumn, @@ -80,10 +80,7 @@ def _effective_speed(task, primary_attr): def render(self, task): """Render the current or finished speed.""" - if task.finished: - speed = self._effective_speed(task, "finished_speed") - else: - speed = self._effective_speed(task, "speed") + speed = self._effective_speed(task, "finished_speed") if task.finished else self._effective_speed(task, "speed") if speed is None: return Text("- it/s") return Text(f"{speed:.1f} it/s") @@ -135,7 +132,7 @@ def _run_export_diff(config: RunConfig, handle): def _run(config: RunConfig): """Build and execute the selected run pipeline.""" - started_at = datetime.now() + started_at = datetime.now(UTC) handle = LogHandler(config.verbose) for notice in config.notices: handle.log(notice) @@ -161,40 +158,41 @@ def main(): try: config = resolve_run_config() except ConfigError as exc: - raise SystemExit(str(exc)) + raise SystemExit(str(exc)) from exc try: return _run(config) except FatalError as exc: if config.verbose and exc.formatted_traceback: print(exc.formatted_traceback, end="") - raise SystemExit(str(exc)) + raise SystemExit(str(exc)) from exc except requests.exceptions.ConnectionError as exc: + detail = _fmt_connection_error(config.netbox_url, exc) print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: {_fmt_connection_error(config.netbox_url, exc)}", + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: {detail}", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc except GraphQLError as exc: print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: NetBox GraphQL request failed — {exc}\n" - f"[{datetime.now().strftime('%H:%M:%S')}] This may be a temporary connectivity issue. " + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: NetBox GraphQL request failed — {exc}\n" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] This may be a temporary connectivity issue. " "Check that NetBox is reachable and try again.", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc except NetBoxRequestError as exc: print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: NetBox REST API request failed — {exc}\n" - f"[{datetime.now().strftime('%H:%M:%S')}] Check that NetBox is reachable and" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: NetBox REST API request failed — {exc}\n" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Check that NetBox is reachable and" " the API token has the required permissions.", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc if __name__ == "__main__": try: main() except KeyboardInterrupt: - print(f"[{datetime.now().strftime('%H:%M:%S')}] Interrupted by user (Ctrl-C). Exiting.") - raise SystemExit(130) + print(f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Interrupted by user (Ctrl-C). Exiting.") + raise SystemExit(130) from None diff --git a/pyproject.toml b/pyproject.toml index 99ff4081..c1cad3cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,8 +31,25 @@ dev = [ line-length = 120 [tool.ruff.lint] -select = ["E", "F", "W", "D", "C901"] +select = [ + "E", "F", "W", "D", "C901", + "I", # isort + "UP", # pyupgrade; safe at the 3.12 floor + "B", # bugbear + "DTZ", # datetimes must carry a timezone + "BLE", # no bare `except Exception` + "S", # bandit + "ERA", # no commented-out code + "SIM", "RET", "PERF", "C4", "RUF", "ISC", + "PIE", "FLY", "RSE", "PGH", "TID", "INP", "A", "LOG", "G", + "PT", # pytest style +] ignore = [ + # Conflicts with the formatter, which owns implicit string concatenation. + "ISC001", + # StrEnum.__str__ returns the bare value where (str, Enum) returns ClassName.MEMBER, + # so the rewrite would change logged and serialized output. + "UP042", # Docstring style choices (pick D211+D212 convention) "D203", # no-blank-line-before-class (incompatible with D211) "D213", # multi-line-summary-second-line (incompatible with D212) @@ -52,7 +69,16 @@ ignore = [ max-complexity = 15 [tool.ruff.lint.per-file-ignores] -"tests/**" = ["D102", "D103"] # test functions don't need docstrings +"tests/**" = [ + "D102", "D103", # test functions don't need docstrings + "S101", # asserting is what a test does + "S105", "S106", "S107", # fixture credentials are not secrets + "S108", "S603", "S607", # tmp paths and subprocess calls are the test's subject + "INP001", # tests/ is not an importable package + "ERA001", # commented sample payloads document the case under test + "SIM117", # nested `with` keeps one patch per line and reads better here + "A002", # `id` mirrors the NetBox field name the stub stands in for +] [tool.mypy] python_version = "3.12" diff --git a/tests/conftest.py b/tests/conftest.py index a2d55c4e..32544ae5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import os -import pytest from unittest.mock import MagicMock, patch +import pytest + from core.config import RunConfig diff --git a/tests/integration/test_import.py b/tests/integration/test_import.py index acb110f7..961697e5 100644 --- a/tests/integration/test_import.py +++ b/tests/integration/test_import.py @@ -19,29 +19,29 @@ Test scenarios -------------- A. All component types created with correct field values - – interfaces (including mgmt_only), power-ports (draw values), + - interfaces (including mgmt_only), power-ports (draw values), console-ports, console-server-ports, power-outlets (power_port link), rear-ports (positions), front-ports (M2M rear_ports), device-bays, module-bays (position). B. Device-type properties stored correctly - – u_height (decimal), is_full_depth (bool), weight, weight_unit, + - u_height (decimal), is_full_depth (bool), weight, weight_unit, airflow, part_number, comments. C. Image linkage - – front_image and rear_image URLs are set on the device type (not just + - front_image and rear_image URLs are set on the device type (not just "uploaded" as orphan files) and the URLs return HTTP 200. D. GraphQL schema consistency - – Query every device-type schema field and every + - Query every device-type schema field and every registry field directly through the GraphQL client so a removed/renamed schema field raises an explicit error rather than a silent false-positive. E. Front-port multi-position linkage - – FP1 → RP1 position 1; FP2 → RP1 position 2 (same rear port). + - FP1 → RP1 position 1; FP2 → RP1 position 2 (same rear port). F. Module-type component creation - – All module component types created; front-port rear_port mapping set. + - All module component types created; front-port rear_port mapping set. G. Idempotency - – Second run: 0 new, 0 modified device types and module types. + - Second run: 0 new, 0 modified device types and module types. H. Update mode - – Delete one interface via REST API; re-run with --update; verify it is + - Delete one interface via REST API; re-run with --update; verify it is recreated with the original type value. Usage:: @@ -67,8 +67,8 @@ import urllib3 from core.change_detector import get_device_type_properties -from core.config import resolve_run_config from core.component_registry import COMPONENT_TYPES +from core.config import resolve_run_config from core.graphql_client import NetBoxGraphQLClient urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -399,7 +399,7 @@ def test_graphql_schema() -> None: ok("get_manufacturers() returned TestVendor") # ── Device types: every schema property present ── - dt_by_model, dt_by_slug = client.get_device_types() + _dt_by_model, dt_by_slug = client.get_device_types() fd = dt_by_slug.get(("testvendor", "testvendor-full-device")) if fd is None: fail("get_device_types() did not return full-device") @@ -426,7 +426,7 @@ def test_graphql_schema() -> None: expected_fields = component.graphql_fields try: records = client.get_component_templates(endpoint_name) - except Exception as exc: + except Exception as exc: # the test reports whatever the client raised # noqa: BLE001 fail( f"get_component_templates({endpoint_name!r}) raised {type(exc).__name__}: {exc} — " f"likely a GraphQL schema error (field removed or renamed)." diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index f56450d3..bb79c675 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -1,8 +1,6 @@ from types import SimpleNamespace from unittest.mock import MagicMock -from core.component_cache import ComponentCache -from core.graphql_client import DotDict from core.change_detector import ( ChangeDetector, ChangeReport, @@ -11,6 +9,8 @@ DeviceTypeChange, PropertyChange, ) +from core.component_cache import ComponentCache +from core.graphql_client import DotDict def _cache(**records): @@ -651,12 +651,14 @@ def test_a_reader_failure_propagates(self): from core.change_detector import _load_device_type_properties - with patch( - "core.change_detector.load_properties_for_type", - side_effect=RuntimeError("schema unavailable"), + with ( + patch( + "core.change_detector.load_properties_for_type", + side_effect=RuntimeError("schema unavailable"), + ), + pytest.raises(RuntimeError, match="schema unavailable"), ): - with pytest.raises(RuntimeError, match="schema unavailable"): - _load_device_type_properties("/nonexistent") + _load_device_type_properties("/nonexistent") # --------------------------------------------------------------------------- diff --git a/tests/test_component_cache.py b/tests/test_component_cache.py index a915936f..49387cb3 100644 --- a/tests/test_component_cache.py +++ b/tests/test_component_cache.py @@ -7,6 +7,7 @@ import threading import time +from typing import ClassVar import pytest @@ -19,7 +20,6 @@ from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError - # ── Fakes ───────────────────────────────────────────────────────────────────── @@ -267,7 +267,8 @@ def test_a_miss_filters_by_device_type_and_caches_the_answer(self): first = cache.get("interface_templates", "device", 1, endpoint) second = cache.get("interface_templates", "device", 1, endpoint) - assert set(first) == {"eth0"} and first == second + assert set(first) == {"eth0"} + assert first == second assert endpoint.filter_calls == [{"device_type_id": 1}] def test_a_miss_filters_by_module_type_for_a_module_parent(self): @@ -349,7 +350,7 @@ def test_each_prefetch_worker_owns_and_closes_its_graphql_client(self): """Concurrent endpoint requests must not share a requests session.""" class WorkerClient: - instances = [] + instances: ClassVar[list] = [] def __init__(self, *args, **kwargs): self.thread_id = None @@ -377,7 +378,7 @@ def test_close_waits_for_workers_and_closes_their_graphql_clients(self): """Cancelling a prefetch must not leak the sessions of workers already running.""" class WorkerClient: - instances = [] + instances: ClassVar[list] = [] started = threading.Event() def __init__(self, *args, **kwargs): @@ -417,7 +418,7 @@ def test_a_prefetch_cannot_be_collected_for_another_vendor(self): cache = make_cache() cache.begin_prefetch(manufacturer_slug="cisco") - with pytest.raises(ValueError, match="cisco.*juniper"): + with pytest.raises(ValueError, match=r"cisco.*juniper"): cache.ensure_ready(manufacturer_slug="juniper") cache.close() diff --git a/tests/test_config.py b/tests/test_config.py index e289960c..0c77c249 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,6 +2,7 @@ import os import pathlib +import re import subprocess import sys @@ -92,7 +93,9 @@ class TestRequiredVariables: """Required environment values fail with a typed configuration error.""" def test_one_missing_variable_uses_the_catalogue_error(self): - with pytest.raises(EnvironmentVariableError, match='Environment variable "NETBOX_TOKEN" is not set.'): + with pytest.raises( + EnvironmentVariableError, match=re.escape('Environment variable "NETBOX_TOKEN" is not set.') + ): resolve_run_config(argv=[], env={"NETBOX_URL": "http://netbox.local"}) def test_all_missing_variables_use_one_typed_error(self): diff --git a/tests/test_docker_workflow.py b/tests/test_docker_workflow.py index 8b8ce602..19fb5d4f 100644 --- a/tests/test_docker_workflow.py +++ b/tests/test_docker_workflow.py @@ -4,7 +4,6 @@ import yaml - _WORKFLOW = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "docker.yml" diff --git a/tests/test_errors.py b/tests/test_errors.py index 16a91d64..cbd7e0d6 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -23,8 +23,10 @@ ), ( SSLVerificationError(False), - "SSL verification failed. IGNORE_SSL_ERRORS is False. " - "Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING.", + ( + "SSL verification failed. IGNORE_SSL_ERRORS is False. " + "Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING." + ), ), (GitCommandError("my-repo"), 'Git error for repo "my-repo".'), (GitInvalidRepositoryError("my-repo"), 'The repo "my-repo" is not a valid git repo.'), diff --git a/tests/test_export_manifest.py b/tests/test_export_manifest.py index c646a241..0cfddd01 100644 --- a/tests/test_export_manifest.py +++ b/tests/test_export_manifest.py @@ -1,7 +1,8 @@ """Tests for core/export_manifest.py.""" import json -from core.export_manifest import load_manifest, save_manifest, is_entry_fresh, update_entry + +from core.export_manifest import is_entry_fresh, load_manifest, save_manifest, update_entry class TestLoadManifest: diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 8e063884..7bfa5bd1 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1,15 +1,16 @@ """Tests for core/export.py — Exporter class.""" from dataclasses import replace +from unittest.mock import MagicMock, patch import pytest -from unittest.mock import MagicMock, patch import yaml from core.config import RunConfig from core.export import ( - ExportItem, + _SKIP, Exporter, + ExportItem, _canon_mfr_slug, _is_subset, _make_filename, @@ -17,11 +18,9 @@ _repo_supersedes, _sanitize_attachment_filename, _yaml_equal, - _SKIP, ) from core.log_handler import LogHandler - # ── Fixtures ────────────────────────────────────────────────────────────────── @@ -286,6 +285,7 @@ def test_a_default_positions_does_not_make_every_front_port_differ(self): against an entry that omits it would report every such definition as differing. """ from types import SimpleNamespace + from core.nb_serializer import _serialize_front_port legacy = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 378a83e1..7f538d09 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -3,9 +3,9 @@ import ast import pathlib import textwrap +from unittest.mock import MagicMock, patch import pytest -from unittest.mock import MagicMock, patch import requests from core.graphql_client import DotDict, NetBoxGraphQLClient @@ -161,9 +161,10 @@ def test_a_v1_token_clone_keeps_the_legacy_auth_scheme(self): @pytest.mark.real_http def test_an_unreachable_server_fails_the_probe_as_a_graphql_error(self): """It is the export's first request, so a raw transport error escapes as a traceback.""" - from core.graphql_client import GraphQLError from helpers import FakeNetBox + from core.graphql_client import GraphQLError + server = FakeNetBox() url = server.url server.close() # nothing is listening on that port any more @@ -606,8 +607,7 @@ class TestGetManufacturers: def _make_client(self): from core.graphql_client import NetBoxGraphQLClient - client = NetBoxGraphQLClient("http://netbox.local", "tok") - return client + return NetBoxGraphQLClient("http://netbox.local", "tok") def test_returns_dict_keyed_by_name(self, mock_post): data = { @@ -1159,9 +1159,10 @@ def _make_client(self): @pytest.mark.real_http @pytest.mark.parametrize("caller", ["graphql_relation_fields", "get_module_types"]) def test_relation_selections_use_the_shared_helper(self, caller): - from core.component_registry import BY_ENDPOINT from helpers import FakeNetBox + from core.component_registry import BY_ENDPOINT + expected = "module_bay_types { id name slug manufacturer { slug } }" if caller == "graphql_relation_fields": assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [expected] @@ -1177,9 +1178,10 @@ def test_relation_selections_use_the_shared_helper(self, caller): @pytest.mark.real_http def test_module_type_query_contains_the_complete_relation_selection(self): - from core.component_registry import BY_ENDPOINT from helpers import FakeNetBox + from core.component_registry import BY_ENDPOINT + server = FakeNetBox() try: client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) @@ -1190,8 +1192,7 @@ def test_module_type_query_contains_the_complete_relation_selection(self): selection = "module_bay_types { id name slug manufacturer { slug } }" assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [selection] - assert " ".join(query.split()) == " ".join( - """ + expected_query = """ query($pagination: OffsetPaginationInput) { module_type_list(pagination: $pagination) { id model part_number airflow description comments weight weight_unit last_updated @@ -1199,8 +1200,8 @@ def test_module_type_query_contains_the_complete_relation_selection(self): manufacturer { id name slug } } } -""".split() - ) +""" + assert " ".join(query.split()) == " ".join(expected_query.split()) def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): """The prefetch runs on clones, not on the client it was cloned from. @@ -1216,7 +1217,8 @@ def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): client.clone().get_component_templates("module_bay_templates") queries = [call.kwargs["json"]["query"] for call in mock_post.call_args_list] - assert queries and all("module_bay_types" in q for q in queries) + assert queries + assert all("module_bay_types" in q for q in queries) def test_the_module_type_query_selects_the_relation_only_when_supported(self, mock_post): """The selection has to track the server, in both directions. @@ -1772,11 +1774,11 @@ def test_primary_fails_no_mappings_field_reraises(self, mock_post): selection — meaning there is no older tier left to fall back to. """ from dataclasses import replace + from unittest.mock import patch + import core.graphql_client as gc_module from core.component_registry import BY_ENDPOINT from core.graphql_client import GraphQLError - from unittest.mock import patch - import core.graphql_client as gc_module # A NetBox whose schema dropped the mappings block entirely. stripped = {"front_port_templates": replace(BY_ENDPOINT["front_port_templates"], graphql_extra=())} @@ -1889,9 +1891,8 @@ def test_request_exception_retries_then_raises_graphql_error(self, mock_post): mock_post.side_effect = requests.exceptions.ConnectionError("connection refused") client = self._make_client() - with patch("core.graphql_client.time.sleep"): - with pytest.raises(GraphQLError, match="connection refused"): - client.query("{ test }", _retries=1) + with patch("core.graphql_client.time.sleep"), pytest.raises(GraphQLError, match="connection refused"): + client.query("{ test }", _retries=1) def test_request_exception_retries_before_exhausting(self, mock_post): """A RequestException on the first attempt stores last_exc and retries.""" @@ -1903,9 +1904,8 @@ def test_request_exception_retries_before_exhausting(self, mock_post): mock_post.side_effect = requests.exceptions.Timeout("timed out") client = self._make_client() - with patch("core.graphql_client.time.sleep"): - with pytest.raises(GraphQLError, match="timed out"): - client.query("{ test }", _retries=1) + with patch("core.graphql_client.time.sleep"), pytest.raises(GraphQLError, match="timed out"): + client.query("{ test }", _retries=1) # ── Vendor-scoped filtering tests ───────────────────────────────────────── @@ -2012,7 +2012,7 @@ def test_single_vendor_filter(self, mock_post, slugs): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=slugs) + by_model, _by_slug = client.get_device_types(manufacturer_slugs=slugs) assert ("cisco", "Catalyst 3850") in by_model assert by_model[("cisco", "Catalyst 3850")].model == "Catalyst 3850" @@ -2068,7 +2068,7 @@ def test_multiple_vendor_filter(self, mock_post, slugs): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=slugs) + by_model, _by_slug = client.get_device_types(manufacturer_slugs=slugs) assert ("cisco", "Catalyst 3850") in by_model assert ("juniper", "EX4300") in by_model @@ -2106,7 +2106,7 @@ def test_none_manufacturer_slugs_unfiltered(self, mock_post): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=None) + _by_model, _by_slug = client.get_device_types(manufacturer_slugs=None) # Verify no filter in the query and no manufacturer variable sent call_payload = mock_post.call_args_list[0][1]["json"] @@ -2477,12 +2477,12 @@ def test_vendor_scoped_two_queries(self, mock_post): # Verify both filters were applied via GraphQL variables (not string interpolation) calls = mock_post.call_args_list - # calls[0]: device_type filter query – data page + # calls[0]: device_type filter query - data page device_payload = calls[0][1]["json"] device_query = device_payload["query"] device_vars = device_payload["variables"] - # calls[1]: device_type filter query – empty terminator (pagination ends) - # calls[2]: module_type filter query – data page + # calls[1]: device_type filter query - empty terminator (pagination ends) + # calls[2]: module_type filter query - data page module_payload = calls[2][1]["json"] module_query = module_payload["query"] module_vars = module_payload["variables"] @@ -2650,7 +2650,7 @@ def test_get_device_types_includes_last_updated(self, mocker): } ] client = self._mock_graphql(mocker, items) - by_model, by_slug = client.get_device_types() + by_model, _by_slug = client.get_device_types() record = by_model[("acme", "M")] assert record.last_updated == "2024-01-15T10:00:00Z" @@ -2922,7 +2922,8 @@ def test_schema_error_still_falls_back_to_the_older_field_tier(self): server.shutdown() assert result == [] - assert "T1" in attempts and "T2" in attempts + assert "T1" in attempts + assert "T2" in attempts @pytest.mark.real_http @@ -2992,7 +2993,8 @@ def test_schema_error_still_triggers_the_unfiltered_fallback(self, method): finally: server.shutdown() - assert "filtered" in attempts and "unfiltered" in attempts + assert "filtered" in attempts + assert "unfiltered" in attempts class TestErrorHandlingStandards: diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 9c9d5694..26b09352 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -2,12 +2,12 @@ from collections import Counter from contextlib import contextmanager +from types import SimpleNamespace import pytest +from helpers import recording_handle as _recording_handle from core.errors import VendorSelectionError -from types import SimpleNamespace - from core.import_run import ( ImportRun, RunSummary, @@ -20,7 +20,6 @@ from core.log_handler import LogHandler from core.outcomes import EntityKind, Outcome from core.repo import DTLRepo -from helpers import recording_handle as _recording_handle class _ComponentCache: @@ -118,7 +117,7 @@ def discover_vendors(self, _devices_path, _modules_path, _racks_path): @staticmethod def resolve_slug_files(_slugs): """Force the pipeline to use its full file scan.""" - return None + return @staticmethod def get_devices(path, vendors): @@ -435,31 +434,31 @@ def __init__(self, *, counter=None): from core.outcomes import OutcomeRegistry self.outcomes = OutcomeRegistry() - base = dict( - added=0, - properties_updated=0, - components_updated=0, - components_added=0, - components_removed=0, - images=0, - manufacturer=0, - module_added=0, - module_updated=0, - rack_type_added=0, - rack_type_updated=0, - ) + base = { + "added": 0, + "properties_updated": 0, + "components_updated": 0, + "components_added": 0, + "components_removed": 0, + "images": 0, + "manufacturer": 0, + "module_added": 0, + "module_updated": 0, + "rack_type_added": 0, + "rack_type_updated": 0, + } base.update(counter or {}) self.counter = Counter(base) def _summary_lines(netbox): """Render a run summary through the real LogHandler and return its lines.""" - from datetime import datetime + from datetime import UTC, datetime handle, console = _recording_handle() handle.verbose = True repo = SimpleNamespace(duplicate_definitions=[]) - summary = RunSummary.capture(netbox, repo, datetime.now()) + summary = RunSummary.capture(netbox, repo, datetime.now(UTC)) _log_run_summary(handle, summary) return console.lines @@ -534,3 +533,54 @@ def test_headline_partials_match_the_itemised_rows(): assert "2 device types partially updated" in text assert "1 modules partially updated" in text assert "Partial updates: 3" in text + + +class TestNaiveStartedAtIsNormalized: + """A naive started_at used to survive construction and fail much later, in the summary.""" + + def test_a_naive_started_at_is_stored_timezone_aware(self, make_config, tmp_path): + from datetime import datetime + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=datetime(2020, 1, 1, 12, 0, 0), # noqa: DTZ001 - a naive value is the subject of this test + ) + + assert run.started_at.tzinfo is not None, "a naive value cannot be subtracted from an aware now()" + + def test_a_run_started_naive_still_produces_a_summary(self, make_config, tmp_path): + """The real failure: RunSummary.capture subtracts started_at from an aware now().""" + from datetime import datetime + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=datetime(2020, 1, 1, 12, 0, 0), # noqa: DTZ001 - a naive value is the subject of this test + ) + + summary = RunSummary.capture(_NetBoxBoundary(), SimpleNamespace(duplicate_definitions=[]), run.started_at) + + assert summary.elapsed.total_seconds() > 0 + + def test_an_aware_started_at_is_left_alone(self, make_config, tmp_path): + from datetime import UTC, datetime + + given = datetime(2020, 1, 1, 12, 0, 0, tzinfo=UTC) + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=given, + ) + + assert run.started_at == given diff --git a/tests/test_log_handler.py b/tests/test_log_handler.py index 3eaf98a5..5b5d9d7f 100644 --- a/tests/test_log_handler.py +++ b/tests/test_log_handler.py @@ -1,9 +1,9 @@ -from types import SimpleNamespace from inspect import signature +from types import SimpleNamespace from unittest.mock import MagicMock, patch -from core.log_handler import LogHandler from core.graphql_client import NetBoxGraphQLClient +from core.log_handler import LogHandler from core.netbox_api import DeviceTypes, NetBox from core.repo import DTLRepo @@ -166,3 +166,34 @@ def test_progress_group_supports_nested_blocks(): handle.end_progress_group() print_mock.assert_called_once_with("[12:00:00] Nested message") + + +class TestTimestampStaysOnLocalWallClock: + """The timestamp is timezone-aware now, which must not move what the operator reads to UTC.""" + + def test_the_timestamp_reads_local_time_not_utc(self, monkeypatch): + """A UTC-based fix for DTZ005 would silently shift every logged line by the local offset.""" + import time + from datetime import UTC, datetime + + monkeypatch.setenv("TZ", "Asia/Tokyo") + time.tzset() + try: + # One absolute instant: 18:04:05 UTC is 03:04:05 the next day in Tokyo. + instant = datetime(2026, 1, 1, 18, 4, 5, tzinfo=UTC) + + class _FrozenClock: + """Models a real clock: naive now() is local, now(tz) converts the same instant.""" + + @staticmethod + def now(tz=None): + if tz is not None: + return instant.astimezone(tz) + return instant.astimezone().replace(tzinfo=None) + + monkeypatch.setattr("core.log_handler.datetime", _FrozenClock) + + assert LogHandler(False)._timestamp() == "03:04:05", "local wall-clock, not the 18:04:05 UTC reading" + finally: + monkeypatch.undo() + time.tzset() diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index afe27626..b2084a1b 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -12,6 +12,7 @@ import pynetbox import pytest +from helpers import FakeNetBox, recording_handle, write_module_bay_type from core.change_detector import ChangeType, ComponentChange, PropertyChange from core.component_registry import BY_YAML_KEY @@ -19,7 +20,6 @@ from core.module_bay_types import ModuleBayTypeCatalog from core.netbox_api import DeviceTypes, NetBox from core.outcomes import EntityKind, Outcome -from helpers import FakeNetBox, recording_handle, write_module_bay_type # The suite patches requests.Session by default, which would stop every request below. pytestmark = pytest.mark.real_http @@ -484,7 +484,8 @@ def test_an_older_server_creates_the_bay_without_the_field(self, make_device_typ ) posted = server.sent("POST", "module_bay_templates") - assert posted and "module_bay_types" not in posted[0] + assert posted + assert "module_bay_types" not in posted[0] def test_an_unresolvable_bay_is_never_posted(self, make_device_types, server): device_types, _ = make_device_types() diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index 82a2e4d1..e629b393 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -7,9 +7,9 @@ """ import pytest +from helpers import FakeNetBox, write_module_bay_type from core.module_bay_types import ModuleBayCatalogError, ModuleBayTypeCatalog, ModuleBayTypeError -from helpers import FakeNetBox, write_module_bay_type class Handle: @@ -142,7 +142,8 @@ def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): ) with pytest.raises(ModuleBayTypeError) as exc: cat.ids_for("juniper", ["MX304-RE"]) - assert "mx304_re" in str(exc.value) and "mx304-re" in str(exc.value) + assert "mx304_re" in str(exc.value) + assert "mx304-re" in str(exc.value) assert not server.sent("POST", "module_bay_types") @@ -185,7 +186,8 @@ def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, ca with pytest.raises(ModuleBayCatalogError) as exc: cat.identities_for("generic", ["SFP"]) - assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) + assert "slug" in str(exc.value) + assert "sfp.yaml" in str(exc.value) def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog): """A parser error escaping the boundary ends the run before it reports anything.""" @@ -222,7 +224,8 @@ def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): cat, _ = catalog(root=tmp_path) with pytest.raises(ModuleBayCatalogError) as exc: cat.ids_for("generic", ["SFP"]) - assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) + assert "Duplicate" in str(exc.value) + assert "SFP" in str(exc.value) @pytest.mark.real_http @@ -235,7 +238,8 @@ def test_a_bare_key_is_refused_rather_than_clearing(self, catalog): with pytest.raises(ModuleBayTypeError) as exc: cat.ids_for("juniper", None) assert "list of names" in str(exc.value) - assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + assert not server.sent("POST", "module_bay_types") + assert not server.sent("POST", "manufacturers") def test_a_non_string_entry_is_refused(self, catalog): cat, _ = catalog() @@ -250,7 +254,8 @@ def test_an_empty_list_is_allowed_and_holds_no_classes(self, catalog): """`module_bay_types: []` is an explicit instruction, not a malformed value.""" cat, server = catalog() assert cat.ids_for("juniper", []) == [] - assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + assert not server.sent("POST", "module_bay_types") + assert not server.sent("POST", "manufacturers") def test_duplicate_names_collapse(self, catalog): """The relationship is a set, so a repeated name must not produce a repeated id.""" diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index fea59e41..bb3af27d 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -312,14 +312,14 @@ def test_items_per_second_column_uses_elapsed_fallback_when_finished_speed_missi @pytest.fixture(scope="session", autouse=True) -def _real_library_root(tmp_path_factory): +def real_library_root(tmp_path_factory): """Point the repo mocks at a real library tree: the pipeline stats these paths before reading them.""" global _LIBRARY_ROOT root = tmp_path_factory.mktemp("library") for name in ("device-types", "module-types", "rack-types"): (root / name).mkdir() _LIBRARY_ROOT = root - yield root + return root def _make_mock_repo(device_types=None): @@ -479,7 +479,7 @@ def test_explicit_total_skips_update_in_finally(self, nb_dt_import): # --------------------------------------------------------------------------- -# filter_device_types_by_change_keys – empty-keys branch +# filter_device_types_by_change_keys - empty-keys branch # --------------------------------------------------------------------------- @@ -494,7 +494,7 @@ def test_empty_change_keys_returns_empty_list(self, nb_dt_import): # --------------------------------------------------------------------------- -# select_device_types_* – None report branches +# select_device_types_* - None report branches # --------------------------------------------------------------------------- @@ -558,15 +558,17 @@ def test_cleanup_on_exception(self, nb_dt_import): mock_progress.add_task.return_value = 1 mock_dt = MagicMock() - with pytest.raises(ValueError): - with import_run_module._image_progress_scope(mock_progress, mock_dt, total=3): - raise ValueError("boom") + with ( + pytest.raises(ValueError, match="boom"), + import_run_module._image_progress_scope(mock_progress, mock_dt, total=3), + ): + raise ValueError("boom") assert mock_dt._image_progress is None # --------------------------------------------------------------------------- -# main() – comprehensive branch coverage +# main() - comprehensive branch coverage # --------------------------------------------------------------------------- @@ -826,9 +828,9 @@ def test_missing_env_var_triggers_system_exit(self, nb_dt_import): patch("nb_dt_import.NetBox"), patch.dict(os.environ, {}, clear=True), patch("core.config.load_dotenv"), + pytest.raises(SystemExit), ): - with pytest.raises(SystemExit): - nb_dt_import.main() + nb_dt_import.main() def test_vendors_and_slugs_flags_log_lines(self, nb_dt_import, capsys): """--vendors and --slugs args cause their respective log lines to execute.""" @@ -1088,9 +1090,8 @@ def test_entry_point_connection_error_message_references_netbox(self, capsys): with patch("core.repo.DTLRepo") as MockDTLRepo, patch("core.netbox_api.NetBox"): MockDTLRepo.side_effect = _requests.exceptions.ConnectionError("Remote end closed") - with patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]): - with pytest.raises(SystemExit): - runpy.run_path(_NB_DT_IMPORT_PATH, run_name="__main__") + with patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), pytest.raises(SystemExit): + runpy.run_path(_NB_DT_IMPORT_PATH, run_name="__main__") captured = capsys.readouterr() assert "connection" in captured.err.lower() or "netbox" in captured.err.lower() @@ -1352,7 +1353,7 @@ class TestLogRunSummary: def test_rack_types_counters_are_logged(self, nb_dt_import): """When netbox.rack_types is True, rack_type_added/updated counters are logged.""" - from datetime import datetime + from datetime import UTC, datetime handle = MagicMock() mock_nb = MagicMock() @@ -1373,7 +1374,7 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): ) mock_nb.outcomes.render_failure_report.return_value = [] mock_repo = SimpleNamespace(duplicate_definitions=[]) - summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now()) + summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now(UTC)) import_run_module._log_run_summary(handle, summary) @@ -1383,7 +1384,7 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): def test_duplicate_definitions_are_logged(self, nb_dt_import): """When dtl_repo has duplicate_definitions, each entry is logged with kept/ignored.""" - from datetime import datetime + from datetime import UTC, datetime handle = MagicMock() mock_nb = MagicMock() @@ -1411,7 +1412,7 @@ def test_duplicate_definitions_are_logged(self, nb_dt_import): } ] mock_nb.outcomes.render_failure_report.return_value = [] - summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now()) + summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now(UTC)) import_run_module._log_run_summary(handle, summary) @@ -1481,9 +1482,9 @@ def test_main_turns_a_fatal_error_into_system_exit(self, nb_dt_import, make_conf with ( patch.object(nb_dt_import, "resolve_run_config", return_value=make_config(verbose=False)), patch.object(nb_dt_import, "_run", side_effect=error), + pytest.raises(SystemExit) as exc_info, ): - with pytest.raises(SystemExit) as exc_info: - nb_dt_import.main() + nb_dt_import.main() assert str(exc_info.value) == 'An unknown error occurred: "Git Repository Error"' assert capsys.readouterr().out == "" @@ -1498,9 +1499,9 @@ def test_main_prints_the_cause_traceback_only_in_verbose_mode(self, nb_dt_import with ( patch.object(nb_dt_import, "resolve_run_config", return_value=make_config(verbose=True)), patch.object(nb_dt_import, "_run", side_effect=error), + pytest.raises(SystemExit), ): - with pytest.raises(SystemExit): - nb_dt_import.main() + nb_dt_import.main() output = capsys.readouterr().out assert "Traceback (most recent call last)" in output @@ -1895,9 +1896,9 @@ def __exit__(self, exc_type, exc, tb): patch("nb_dt_import.NetBox", return_value=mock_nb), patch("core.import_run._process_device_types", side_effect=RuntimeError("boom")), patch("nb_dt_import.get_progress_panel", return_value=_Ctx()), + pytest.raises(RuntimeError, match="boom"), ): - with pytest.raises(RuntimeError, match="boom"): - nb_dt_import.main() + nb_dt_import.main() mock_nb.device_types.components.close.assert_called() @@ -2032,19 +2033,19 @@ def test_import_run_stops_preload_in_finally_on_error(self, make_config): ) netbox = _make_mock_netbox() - with ( + with ( # noqa: PT012 patch("core.import_run._process_device_types", side_effect=RuntimeError("boom")), patch("core.import_run._finalize_task_registry") as mock_finalize, + pytest.raises(RuntimeError, match="boom"), ): - with pytest.raises(RuntimeError, match="boom"): - run = import_run_module.ImportRun( - config, - dtl_repo, - netbox, - handle, - lambda _show_remaining_time: nullcontext(progress), - ) - run.execute() + run = import_run_module.ImportRun( + config, + dtl_repo, + netbox, + handle, + lambda _show_remaining_time: nullcontext(progress), + ) + run.execute() netbox.device_types.components.close.assert_called() mock_finalize.assert_called_once_with(progress, {}) @@ -2125,9 +2126,9 @@ class TestExportDiffVendorFilterEndToEnd: """ def test_single_vendor_reaches_the_graphql_request( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco") payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2138,10 +2139,10 @@ def test_single_vendor_reaches_the_graphql_request( assert "Nothing to export" in capsys.readouterr().out def test_multiple_vendors_are_sent_as_a_json_list( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): """A tuple would not compare equal here, and NetBox would not accept it as a list variable.""" - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco,Juniper") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco,Juniper") payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2152,10 +2153,10 @@ def test_multiple_vendors_are_sent_as_a_json_list( assert "Nothing to export" in capsys.readouterr().out def test_no_vendors_queries_every_manufacturer( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): """Without --vendors, config.vendors is (), which the GraphQL layer rejects if it reaches it.""" - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root) + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root) payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2203,11 +2204,11 @@ def log_message(self, *args): threading.Thread(target=server.serve_forever, daemon=True).start() return f"http://127.0.0.1:{server.server_port}", server, bodies - def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypatch, tmp_path, real_library_root): url, server, bodies = self._serve() monkeypatch.setenv("NETBOX_URL", url) try: - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco,Juniper") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco,Juniper") finally: server.shutdown() server.server_close() @@ -2229,20 +2230,21 @@ def _queries_for_version(self, nb_dt_import, monkeypatch, tmp_path, library_root return [payload["query"] for payload in bodies] def test_a_47_server_is_asked_for_the_module_bay_type_relation( - self, nb_dt_import, monkeypatch, tmp_path, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, real_library_root ): """Not selecting it exports every bay without its restriction, silently. Asserted on the module-type query, which this run always issues; the component queries only run once there is something to export. """ - queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.7.0") + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, real_library_root, "4.7.0") module_types = [q for q in queries if "module_type_list(" in q] - assert module_types and all("module_bay_types" in q for q in module_types) + assert module_types + assert all("module_bay_types" in q for q in module_types) - def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, real_library_root): """Selecting a field the schema lacks fails the whole query.""" - queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.6.9") + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, real_library_root, "4.6.9") assert not [q for q in queries if "module_bay_types" in q] diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 2423e106..a2f61af4 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -1,24 +1,24 @@ import os import threading -from types import SimpleNamespace from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest +from helpers import paginate_dispatch, recording_handle from requests.exceptions import ConnectionError as RequestsConnectionError -from unittest.mock import MagicMock, patch + from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES -from core.outcomes import EntityKind, Outcome from core.netbox_api import ( + DeviceTypes, NetBox, NetBoxError, - DeviceTypes, SSLVerificationError, _delete_image_attachment, _FrontPortRecordWithMappings, ) -from helpers import paginate_dispatch, recording_handle - +from core.outcomes import EntityKind, Outcome # All component list keys used by the GraphQL client for empty-response fallback. _ALL_COMPONENT_KEYS = [component.list_key for component in COMPONENT_TYPES] @@ -806,8 +806,10 @@ def test_per_item_error_reports_every_failing_item( messages = [call.args[0] for call in mock_handle.log.call_args_list] assert len(messages) == 2 - assert "eth0" in messages[0] and "This field must be unique." in messages[0] - assert "eth2" in messages[1] and "Invalid choice." in messages[1] + assert "eth0" in messages[0] + assert "This field must be unique." in messages[0] + assert "eth2" in messages[1] + assert "Invalid choice." in messages[1] def test_non_list_error_logs_failed_items( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle @@ -1053,7 +1055,7 @@ def test_empty_src_returns_none(self): assert _image_dir_for_yaml("", "device-types", "elevation-images") is None def test_unknown_src_returns_none(self): - from core.netbox_api import _image_dir_for_yaml, _UNKNOWN_SRC + from core.netbox_api import _UNKNOWN_SRC, _image_dir_for_yaml assert _image_dir_for_yaml(_UNKNOWN_SRC, "device-types", "elevation-images") is None @@ -1921,6 +1923,7 @@ def test_update_property_change_request_error_logged( "Device Type Updated" log MUST NOT be emitted. """ import pynetbox as real_pynb2 + from core.change_detector import ChangeReport, DeviceTypeChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -1991,6 +1994,7 @@ def _build_subdevice_role_flip_setup( report zero dependent devices and one blocking device-bay template. """ import pynetbox as real_pynb2 + from core.change_detector import ChangeReport, DeviceTypeChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -2024,8 +2028,10 @@ def _build_subdevice_role_flip_setup( # Force the .error property to return the parsed dict (pynetbox usually does this). err.error = { "subdevice_role": [ - "Must delete all device bay templates associated with this device " - "before declassifying it as a parent device." + ( + "Must delete all device bay templates associated with this device " + "before declassifying it as a parent device." + ) ] } @@ -2049,7 +2055,7 @@ def test_constraint_failure_logs_hint_when_flag_off( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """Without --force-resolve-conflicts, classifier hint is logged but no auto-resolve runs.""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=False ) mock_nb_api.dcim.device_types.update.side_effect = err @@ -2076,13 +2082,14 @@ def test_constraint_failure_logs_hint_when_flag_off( assert failures[0].outcome == Outcome.FAILED assert "SuperServer" in failures[0].identity assert "module-bay-1" in failures[0].blocking_objects - assert failures[0].hint and "--force-resolve-conflicts" in failures[0].hint + assert failures[0].hint + assert "--force-resolve-conflicts" in failures[0].hint def test_constraint_failure_auto_resolves_when_flag_on_and_safe( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """With flag on + zero dependents, blocking templates are deleted and PATCH retried.""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) # First update call fails; second (after auto-resolve) succeeds. @@ -2103,7 +2110,7 @@ def test_constraint_failure_auto_resolve_retry_still_fails( """If the retried PATCH after auto-resolve still fails, count it as a failure exactly once.""" import pynetbox as real_pynb2 - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) err2 = real_pynb2.RequestError(MagicMock(status_code=500, content=b'{"detail":"still bad"}')) @@ -2124,7 +2131,7 @@ def test_constraint_failure_blocked_when_devices_in_use( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """With flag on but live devices reference the type, no remediation runs (safety gate).""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) live_device = MagicMock() @@ -2694,14 +2701,14 @@ def test_existing_module_with_new_image_is_actionable( } # existing images are empty → the image is "new" → actionable - with patch.object(nb, "_fetch_module_type_existing_images", return_value={42: set()}): - with patch( + with ( + patch.object(nb, "_fetch_module_type_existing_images", return_value={42: set()}), + patch( "core.netbox_api.NetBox._discover_module_image_files", return_value=[str(img)], - ): - result, existing_images_map, _ = nb.filter_actionable_module_types( - [module_type], all_mts, only_new=False - ) + ), + ): + result, existing_images_map, _ = nb.filter_actionable_module_types([module_type], all_mts, only_new=False) assert result == [module_type] # Verify the upload worklist is propagated so create_module_types can upload the image. assert existing_images_map == {42: set()} @@ -3534,6 +3541,7 @@ def test_property_update_success_counter_incremented( ): """Successful property update increments components_updated counter.""" from collections import Counter as _Counter + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_nb_api = mock_pynetbox.api.return_value @@ -3562,6 +3570,7 @@ def test_property_update_request_error_logged( ): """RequestError during property update is caught and logged.""" import pynetbox as real_pynb2 + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -3695,6 +3704,7 @@ def test_request_error_on_delete_is_logged( ): """RequestError during component deletion is caught and logged.""" import pynetbox as real_pynb2 + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -4100,12 +4110,11 @@ def test_file_close_exception_swallowed( fake_fh = MagicMock() fake_fh.close.side_effect = OSError("cannot close") - with patch("builtins.open", return_value=fake_fh): - with patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req2.RequestException - mock_req.patch.side_effect = _req2.RequestException("server error") - # Should NOT raise despite close() raising - dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) + with patch("builtins.open", return_value=fake_fh), patch("core.netbox_api.requests") as mock_req: + mock_req.RequestException = _req2.RequestException + mock_req.patch.side_effect = _req2.RequestException("server error") + # Should NOT raise despite close() raising + dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) # The RequestException log should still have been called assert mock_handle.log.called @@ -4556,6 +4565,7 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynetbox, mock_handle): """RequestError during update is logged; processing continues.""" import pynetbox + from core.graphql_client import DotDict mock_pynetbox.api.return_value.version = "4.3" @@ -4618,7 +4628,7 @@ class TestVerifyCompatibility: """Tests for NetBox.verify_compatibility() version thresholds.""" @pytest.mark.parametrize( - "version_str, expected_m2m, expected_module_bay_types", + ("version_str", "expected_m2m", "expected_module_bay_types"), [ ("4.3", False, False), ("4.4", False, False), @@ -5023,6 +5033,7 @@ def test_load_for_vendor_populates_existing_device_types( ): """load_for_vendor populates existing_device_types and existing_device_types_by_slug.""" from unittest.mock import patch as _patch + from core.graphql_client import DotDict mock_nb_api = mock_pynetbox.api.return_value @@ -5055,6 +5066,7 @@ def test_load_for_vendor_replaces_prior_data( ): """A second call to load_for_vendor replaces data from the first call.""" from unittest.mock import patch as _patch + from core.graphql_client import DotDict mock_nb_api = mock_pynetbox.api.return_value @@ -5086,6 +5098,7 @@ def test_load_for_vendor_resets_state_before_fetch_on_failure( ): """State is reset before the fetch so a raised exception leaves a clean slate.""" from unittest.mock import patch as _patch + import pytest mock_nb_api = mock_pynetbox.api.return_value @@ -5307,6 +5320,7 @@ def test_returns_false_when_no_cache_entry(self, tmp_path): def test_returns_false_when_hash_matches(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed data = b"unchanged_content" @@ -5317,6 +5331,7 @@ def test_returns_false_when_hash_matches(self, tmp_path): def test_returns_true_when_hash_differs(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed img = tmp_path / "front.png" @@ -5326,6 +5341,7 @@ def test_returns_true_when_hash_differs(self, tmp_path): def test_returns_false_on_file_read_error(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed path = str(tmp_path / "missing.png") @@ -5381,9 +5397,8 @@ def test_verify_images_reuploads_missing_image( mock_resp = MagicMock() mock_resp.ok = False # image missing on server → "missing" - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + nb.create_device_types([device_type]) nb.device_types.upload_images.assert_called_once() @@ -5466,9 +5481,8 @@ def test_verify_images_skips_ok_image( mock_resp = MagicMock() mock_resp.ok = True # image accessible on server mock_resp.headers = {"Content-Type": "image/png"} - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + nb.create_device_types([device_type]) nb.device_types.upload_images.assert_not_called() @@ -5503,9 +5517,8 @@ def test_default_mode_still_skips_without_http( "src": str(dev_types_dir / "ap.yaml"), } - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get") as mock_get: - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get") as mock_get: + nb.create_device_types([device_type]) mock_get.assert_not_called() nb.device_types.upload_images.assert_not_called() @@ -5549,10 +5562,9 @@ def test_verify_ok_seeds_hash_cache( mock_resp = MagicMock() mock_resp.ok = True mock_resp.headers = {"Content-Type": "image/png"} - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - with patch("core.netbox_api._save_image_hash_cache") as mock_save: - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + with patch("core.netbox_api._save_image_hash_cache") as mock_save: + nb.create_device_types([device_type]) # Hash cache must have been seeded with the local file's hash assert str(img) in nb._image_hash_cache @@ -5598,6 +5610,7 @@ def test_delete_image_attachment_success(self, mock_settings, mock_handle): def test_delete_image_attachment_logs_request_errors(self, mock_settings, mock_handle): import requests + from core.netbox_api import _delete_image_attachment with patch("core.netbox_api.requests.delete", side_effect=requests.RequestException("boom")): @@ -5616,6 +5629,7 @@ def test_load_module_type_properties_falls_back_without_a_schema(self, tmp_path) def test_fmt_connection_error_contains_url_and_hint(self): """_fmt_connection_error returns a message with the URL and a reachability hint.""" import requests as _requests + from core.netbox_api import _fmt_connection_error url = "http://netbox.example.com" @@ -5627,9 +5641,10 @@ def test_fmt_connection_error_contains_url_and_hint(self): def test_fmt_connection_error_verify_compatibility_uses_it(self, mock_settings, mock_pynetbox, mock_handle): """verify_compatibility uses _fmt_connection_error for ConnectionError.""" - import requests as _requests from unittest.mock import PropertyMock + import requests as _requests + type(mock_pynetbox.api.return_value).version = PropertyMock( side_effect=_requests.exceptions.ConnectionError("drop") ) @@ -5638,7 +5653,8 @@ def test_fmt_connection_error_verify_compatibility_uses_it(self, mock_settings, NetBox(mock_settings, mock_handle) exc_msg = str(exc_info.value.args[0]) if exc_info.value.args else "" - assert mock_settings.netbox_url in exc_msg and "Connection" in exc_msg + assert mock_settings.netbox_url in exc_msg + assert "Connection" in exc_msg def test_check_image_url_reports_the_transport_error_when_log_fn_provided(self): """The wire detail goes to log_fn; the verdict goes to the return value.""" @@ -5702,7 +5718,8 @@ def test_a_corrupt_cache_file_is_reported_by_path(self, mock_settings, mock_pyne assert nb._image_hash_cache == {} logged = " ".join(str(call) for call in mock_handle.log.call_args_list) - assert str(path) in logged and "unreadable image hash cache" in logged + assert str(path) in logged + assert "unreadable image hash cache" in logged def test_a_cache_file_holding_the_wrong_shape_is_reported( self, mock_settings, mock_pynetbox, mock_handle, tmp_path @@ -6050,6 +6067,7 @@ def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pynetbox, mock_handle): from types import SimpleNamespace + from core.update_failure_resolver import FailureKind mock_pynetbox.api.return_value.version = "4.3" @@ -6073,6 +6091,7 @@ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pyn def test_try_resolve_update_logs_auto_resolve_failure(self, mock_settings, mock_pynetbox, mock_handle): from types import SimpleNamespace + from core.update_failure_resolver import FailureKind mock_pynetbox.api.return_value.version = "4.3" @@ -6102,9 +6121,11 @@ def boom(): def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( self, mock_settings, mock_pynetbox, mock_handle ): + from types import SimpleNamespace + import pynetbox as real_pynb import requests - from types import SimpleNamespace + from core.update_failure_resolver import FailureKind mock_pynetbox.RequestError = real_pynb.RequestError @@ -6194,6 +6215,7 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( ): import pynetbox as real_pynb import requests + from core.change_detector import PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -6434,6 +6456,7 @@ def test_apply_updates_for_type_logs_retryable_exception( ): import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -6460,6 +6483,7 @@ def test_remove_components_logs_retryable_exception( ): import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -6510,6 +6534,7 @@ class TestRemainingCoverageBranches: def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb import requests + from core.graphql_client import DotDict mock_pynetbox.RequestError = real_pynb.RequestError @@ -7174,13 +7199,13 @@ class TestSummaryWordingMatchesTheFailedOperation: def _summary_text(self, nb): """Render the real end-of-run summary for *nb* and return it as one string.""" - from datetime import datetime + from datetime import UTC, datetime from types import SimpleNamespace from core.import_run import RunSummary, _log_run_summary handle, console = recording_handle() - summary = RunSummary.capture(nb, SimpleNamespace(duplicate_definitions=[]), datetime.now()) + summary = RunSummary.capture(nb, SimpleNamespace(duplicate_definitions=[]), datetime.now(UTC)) _log_run_summary(handle, summary) return "\n".join(console.lines) @@ -7249,14 +7274,14 @@ class TestSkippedComponentReasonReachesTheReport: def _dt(self, mock_pynetbox, make_device_types, parent_id=1): mock_nb_api = mock_pynetbox.api.return_value mock_nb_api.version = "4.3" - dt = make_device_types(nb_api=mock_nb_api) - return dt + return make_device_types(nb_api=mock_nb_api) def test_unresolvable_power_port_reaches_the_report( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The Powerman case: an outlet dropped for a bad power_port must name the reason.""" import pynetbox as real_pynb + from core.change_detector import ChangeReport, ChangeType, ComponentChange, DeviceTypeChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -7342,6 +7367,7 @@ def test_component_update_transport_failure_reports_the_transport_error( """A retry-exhausted update must name the transport error, not the generic label.""" import pynetbox as real_pynb import requests + from core.change_detector import ( ChangeReport, ChangeType, @@ -7407,6 +7433,7 @@ def test_component_removal_transport_failure_reports_the_transport_error( """A retry-exhausted removal must name the transport error too.""" import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -7436,6 +7463,7 @@ def test_failed_component_create_reports_the_netbox_message( ): """The report must name the constraint, not the generic failure label.""" import pynetbox as real_pynb + from core.change_detector import ChangeReport, ChangeType, ComponentChange, DeviceTypeChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -7617,7 +7645,7 @@ def _mapping_clears_sent(self, nb): sent += [pc for pc in change.property_changes if pc.property_name == "_mappings"] return sent - @pytest.mark.parametrize("remove_components, expected", [(False, 0), (True, 1)]) + @pytest.mark.parametrize(("remove_components", "expected"), [(False, 0), (True, 1)]) def test_the_flag_decides_whether_a_clear_reaches_netbox( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle, remove_components, expected ): diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 6d759ea1..1aba6fe8 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -8,7 +8,7 @@ class TestNormalizeValues: def test_netbox_choice_object_reads_value(self): choice = SimpleNamespace(value="1000base-t") - y, n = normalize_values("1000base-t", choice) + _y, n = normalize_values("1000base-t", choice) assert n == "1000base-t" def test_empty_string_normalized_to_none(self): @@ -32,26 +32,26 @@ def test_trailing_spaces_stripped(self): assert n == "def" def test_numeric_yaml_coerces_string_netbox(self): - y, n = normalize_values(1.0, "1.0") + _y, n = normalize_values(1.0, "1.0") assert n == 1.0 def test_numeric_netbox_coerces_string_yaml(self): - y, n = normalize_values("2.5", 2.5) + y, _n = normalize_values("2.5", 2.5) assert y == 2.5 def test_int_yaml_preserves_int_type(self): """YAML int 166 vs NetBox '166.00' should normalize nb to int 166, not float 166.0.""" - y, n = normalize_values(166, "166.00") + _y, n = normalize_values(166, "166.00") assert n == 166 assert isinstance(n, int) def test_int_netbox_preserves_int_type(self): - y, n = normalize_values("166", 166) + y, _n = normalize_values("166", 166) assert y == 166 assert isinstance(y, int) def test_float_yaml_stays_float(self): - y, n = normalize_values(26.1, "26.10") + _y, n = normalize_values(26.1, "26.10") assert n == 26.1 def test_bool_not_coerced(self): @@ -66,11 +66,11 @@ def test_bool_as_int_trap(self): assert n == "1.0" def test_non_numeric_string_netbox_stays_string(self): - y, n = normalize_values(1.0, "notanumber") + _y, n = normalize_values(1.0, "notanumber") assert n == "notanumber" def test_non_numeric_string_yaml_stays_string(self): - y, n = normalize_values("notanumber", 1.0) + y, _n = normalize_values("notanumber", 1.0) assert y == "notanumber" diff --git a/tests/test_repo.py b/tests/test_repo.py index 09714d48..c33af6b2 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,9 +1,15 @@ import os +import re +from unittest.mock import MagicMock, call, mock_open, patch import pytest import yaml -from unittest.mock import MagicMock, call, mock_open, patch -from git import Actor, Repo as GitRepo, exc as git_exc +from git import Actor +from git import Repo as GitRepo +from git import exc as git_exc + +from core.errors import UnknownError +from core.log_handler import LogHandler from core.repo import ( DTLRepo, GitBranchNotFoundError, @@ -15,11 +21,9 @@ _safe_index_load, _safe_json_load, _safe_pickle_load, - validate_git_url, normalize_port_mappings, + validate_git_url, ) -from core.errors import UnknownError -from core.log_handler import LogHandler def _dtl_repo(config, repo_path, handle): @@ -46,31 +50,31 @@ def test_https_valid(self): assert err is None def test_https_no_hostname_invalid(self): - ok, err = validate_git_url("https://") + ok, _err = validate_git_url("https://") assert ok is False def test_git_at_scp_valid(self): - ok, err = validate_git_url("git@github.com:org/repo.git") + ok, _err = validate_git_url("git@github.com:org/repo.git") assert ok is True def test_git_at_no_colon_invalid(self): - ok, err = validate_git_url("git@github.com/org/repo.git") + ok, _err = validate_git_url("git@github.com/org/repo.git") assert ok is False def test_ssh_valid(self): - ok, err = validate_git_url("ssh://git@github.com/org/repo.git") + ok, _err = validate_git_url("ssh://git@github.com/org/repo.git") assert ok is True def test_ssh_no_hostname_invalid(self): - ok, err = validate_git_url("ssh://") + ok, _err = validate_git_url("ssh://") assert ok is False def test_file_valid(self): - ok, err = validate_git_url("file:///tmp/repo") + ok, _err = validate_git_url("file:///tmp/repo") assert ok is True def test_file_empty_path_invalid(self): - ok, err = validate_git_url("file://") + ok, _err = validate_git_url("file://") assert ok is False def test_empty_url_invalid(self): @@ -79,11 +83,11 @@ def test_empty_url_invalid(self): assert "Empty" in err def test_ftp_invalid(self): - ok, err = validate_git_url("ftp://example.com/repo.git") + ok, _err = validate_git_url("ftp://example.com/repo.git") assert ok is False def test_none_invalid(self): - ok, err = validate_git_url(None) + ok, _err = validate_git_url(None) assert ok is False @@ -143,7 +147,7 @@ def test_invalid_url_raises_before_the_clone_path(self): mock_args.repo_url = "ftp://bad.url" mock_args.repo_branch = "master" with _clone_present(False), patch("core.repo.Repo"): - with pytest.raises(InvalidGitURLError, match="Invalid Git URL: ftp://bad.url"): + with pytest.raises(InvalidGitURLError, match=re.escape("Invalid Git URL: ftp://bad.url")): _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) def test_invalid_path_raises_before_repository_access(self, tmp_path): @@ -255,7 +259,7 @@ class TestDTLRepoRealGit: @pytest.fixture(autouse=True) def mock_git_repo(self): """Override the global autouse git mock so these tests exercise real git.""" - yield None + return @pytest.fixture(autouse=True) def clear_ambient_git_env(self, monkeypatch): @@ -358,8 +362,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_get_relative_path(self): repo = self._make_repo() @@ -473,7 +476,7 @@ def test_repo_open_git_error_uses_the_configured_url_without_reading_self_repo(s failure = git_exc.GitCommandError("status", 1) with _clone_present(), patch("core.repo.Repo", side_effect=failure): - with pytest.raises(GitCommandError, match="https://example.invalid/repo.git") as exc_info: + with pytest.raises(GitCommandError, match=re.escape("https://example.invalid/repo.git")) as exc_info: _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) assert "cmdline: status" in exc_info.value.formatted_traceback @@ -499,7 +502,9 @@ def test_invalid_git_repository_raises_its_catalogue_error(self): invalid = git_exc.InvalidGitRepositoryError("/tmp/repo") with _clone_present(), patch("core.repo.Repo", side_effect=invalid): - with pytest.raises(GitInvalidRepositoryError, match='The repo "/tmp/repo" is not a valid git repo.'): + with pytest.raises( + GitInvalidRepositoryError, match=re.escape('The repo "/tmp/repo" is not a valid git repo.') + ): _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) @@ -547,8 +552,7 @@ def _make_repo(self, tmp_path): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) - return repo + return _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) def test_get_devices_all_vendors(self, tmp_path): repo = self._make_repo(tmp_path) @@ -556,7 +560,7 @@ def test_get_devices_all_vendors(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "Juniper").mkdir() - files, vendors = repo.get_devices(str(devices)) + _files, vendors = repo.get_devices(str(devices)) assert len(vendors) == 2 assert any(v["name"] == "Cisco" for v in vendors) @@ -566,7 +570,7 @@ def test_get_devices_filters_vendors(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "Juniper").mkdir() - files, vendors = repo.get_devices(str(devices), vendors=["cisco"]) + _files, vendors = repo.get_devices(str(devices), vendors=["cisco"]) assert len(vendors) == 1 assert vendors[0]["name"] == "Cisco" @@ -576,7 +580,7 @@ def test_get_devices_skips_testing_folder(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "testing").mkdir() - files, vendors = repo.get_devices(str(devices)) + _files, vendors = repo.get_devices(str(devices)) assert not any(v["name"] == "testing" for v in vendors) @@ -596,8 +600,7 @@ def _make_repo(self, tmp_path): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) - return repo + return _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) def test_discovers_vendors_from_single_path(self, tmp_path): """Test discovery from a single existing path.""" @@ -677,7 +680,7 @@ def test_handles_os_errors_gracefully(self, tmp_path): def mock_listdir(path): if "devices" in path: raise OSError("Permission denied") - elif "modules" in path: + if "modules" in path: return ["Cisco"] return [] @@ -1253,7 +1256,7 @@ def test_valid_existing_dir_returns_true(self, tmp_path): """Existing writable directory returns True.""" from core.repo import validate_repo_path - ok, msg = validate_repo_path(str(tmp_path)) + ok, _msg = validate_repo_path(str(tmp_path)) assert ok is True @@ -1268,6 +1271,7 @@ def test_parse_device_type_returns_error_when_normalize_fails(tmp_path): Covers repo.py lines 225-226: 'if err: return err'. """ from unittest.mock import patch + from core.repo import parse_single_file yaml_file = tmp_path / "test.yaml" @@ -1345,8 +1349,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_get_racks_path_ends_with_rack_types(self): repo = self._make_repo() @@ -1398,8 +1401,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_keyboard_interrupt_is_reraised(self): import pytest @@ -1434,8 +1436,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_item_without_manufacturer_is_included_without_dedup(self): """Item missing 'manufacturer' key skips dedup and is appended as-is.""" @@ -1536,8 +1537,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_returns_none_when_pickle_missing(self, tmp_path): """Returns None gracefully when the device pickle doesn't exist.""" @@ -1884,6 +1884,7 @@ class TestAStanzaThatDoesNotListAFrontPort: def test_a_nonempty_stanza_clears_an_omitted_front_port_mapping(self): from types import SimpleNamespace + from core.change_detector import ChangeDetector data = yaml.safe_load(""" diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index e85d411e..c456b12b 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -39,9 +39,12 @@ def _orphaned_docstrings(tree): if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue for statement in node.body[1:]: - if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): - if isinstance(statement.value.value, str): - yield node.name, statement.lineno + if ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, str) + ): + yield node.name, statement.lineno def test_the_scan_follows_pytest_discovery(tmp_path): diff --git a/tests/test_update_failure_resolver.py b/tests/test_update_failure_resolver.py index 93a4bf2f..1185c099 100644 --- a/tests/test_update_failure_resolver.py +++ b/tests/test_update_failure_resolver.py @@ -9,11 +9,10 @@ from core.update_failure_resolver import ( FailureKind, - extract_error_payload, classify_device_type_update_failure, + extract_error_payload, ) - SUBDEVICE_ROLE_ERROR_DICT = { "subdevice_role": [ "Must delete all device bay templates associated with this device before declassifying it as a parent device." From 7f7d0ba3c40786c9a8d63a488dcb9d62c03c92a9 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Sat, 17 Oct 2026 07:01:37 +0200 Subject: [PATCH 21/39] fix(module-types): update the profile and attribute data of existing module types (#148) --- core/graphql_client.py | 5 + core/netbox_api.py | 62 +++- core/repo.py | 21 +- .../module-types/TestVendor/power-supply.yaml | 13 + tests/helpers.py | 44 ++- tests/integration/test_import.py | 66 +++++ tests/test_graphql_client.py | 18 ++ tests/test_module_type_profile.py | 265 ++++++++++++++++++ 8 files changed, 485 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/module-types/TestVendor/power-supply.yaml create mode 100644 tests/test_module_type_profile.py diff --git a/core/graphql_client.py b/core/graphql_client.py index d66c376a..c6bf066f 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -542,6 +542,8 @@ def get_module_types(self, manufacturer_slugs=None): weight weight_unit last_updated + profile {{ id name }} + attribute_data {module_bay_type_selection}manufacturer {{ id name @@ -555,6 +557,9 @@ def get_module_types(self, manufacturer_slugs=None): result: dict = {} for item in items: record = _to_dotdict(item) + # Attribute values are user data, so the id coercion must not touch them. + if "attribute_data" in item: + record["attribute_data"] = item["attribute_data"] mfr_slug = record.manufacturer.slug result.setdefault(mfr_slug, {})[record.model] = record diff --git a/core/netbox_api.py b/core/netbox_api.py index 02f2566c..85468c6d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -365,8 +365,8 @@ def _retry_on_connection_error(func, *args, **kwargs): # Module type scalar properties that can be compared and updated. # Loaded from the cloned devicetype-library schema at runtime; the list below # serves as a fallback when the schema is not yet available (e.g. before the -# first repo clone). Identity fields (manufacturer, model) and complex objects -# (attribute_data) are excluded by the schema reader. +# first repo clone). Identity fields (manufacturer, model) are excluded, and +# profile and attribute_data are compared by NetBox._profile_changes. _MODULE_TYPE_PROPERTIES_FALLBACK = [ "part_number", "description", @@ -378,6 +378,29 @@ def _retry_on_connection_error(func, *args, **kwargs): _MODULE_TYPE_SCHEMA_EXCLUDE = {"manufacturer", "model", "attribute_data", "profile"} +# The REST API names the attribute_data model field "attributes" and ignores the model name. +_REST_ATTRIBUTES_FIELD = "attributes" + + +def _with_rest_attributes(module_type): + """Return *module_type* with attribute_data under the name the REST API reads.""" + if "attribute_data" not in module_type: + return module_type + payload = {k: v for k, v in module_type.items() if k != "attribute_data"} + payload[_REST_ATTRIBUTES_FIELD] = module_type["attribute_data"] + return payload + + +def _json_equal(a, b) -> bool: + """Compare JSON values: numbers by value (400 equals 400.0), but a boolean never equals a number.""" + if isinstance(a, bool) or isinstance(b, bool): + return type(a) is type(b) and a == b + if isinstance(a, dict) and isinstance(b, dict): + return a.keys() == b.keys() and all(_json_equal(a[key], b[key]) for key in a) + if isinstance(a, list) and isinstance(b, list): + return len(a) == len(b) and all(_json_equal(x, y) for x, y in zip(a, b, strict=True)) + return a == b + @lru_cache(maxsize=1) def _load_module_type_properties(repo_path): @@ -1591,6 +1614,7 @@ def filter_actionable_module_types(self, module_types, all_module_types, only_ne if not values_equal(module_type[f], nb_val): changed_fields_info.append((f, nb_val, module_type[f])) + changed_fields_info += self._profile_changes(module_type, existing_module) changed_fields_info += self._type_relation_changes(module_type, existing_module) component_changes = detector._compare_components(module_type, existing_module.id, parent_type="module") @@ -1645,6 +1669,33 @@ def _fetch_module_type_existing_images(self): ) return module_type_existing_images + @staticmethod + def _profile_changes(module_type, existing_module): + """Return (field, current, wanted) for the profile and for each declared attribute that differs. + + NetBox merges PATCHed attributes into the stored ones, so an attribute the YAML omits is + left alone, the same as an omitted property. + """ + changes: list[tuple[str, Any, Any]] = [] + if "profile" in module_type and hasattr(existing_module, "profile"): + wanted = module_type["profile"]["name"] if module_type["profile"] else None + current = existing_module.profile.name if existing_module.profile else None + if wanted != current: + changes.append(("profile", current, wanted)) + if "attribute_data" in module_type and hasattr(existing_module, "attribute_data"): + declared = module_type["attribute_data"] or {} + stored = existing_module.attribute_data + if not declared: + # NetBox replaces, not merges, the stored attributes with an empty value. + if stored: + changes.append(("attribute_data", stored, {})) + return changes + stored = stored if isinstance(stored, dict) else {} + for key, wanted in declared.items(): + if not _json_equal(stored.get(key, _MISSING), wanted): + changes.append((f"attribute_data.{key}", stored.get(key), wanted)) + return changes + def _type_relation_changes(self, module_type, existing_module): """Return (field, current, wanted) for each of the type's own relations that differs. @@ -1713,6 +1764,11 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + for field, _current, _wanted in self._profile_changes(curr_mt, module_type_res): + if field == "profile": + updates["profile"] = curr_mt["profile"] + else: + updates[_REST_ATTRIBUTES_FIELD] = curr_mt["attribute_data"] or {} if self.module_bay_types: from core.module_bay_types import ModuleBayTypeError @@ -1920,7 +1976,7 @@ def _process_single_module_type( from core.module_bay_types import ModuleBayTypeError try: - payload = self._resolve_type_relations(curr_mt) + payload = self._resolve_type_relations(_with_rest_attributes(curr_mt)) except ModuleBayTypeError as exc: self.handle.log(f"Error creating Module Type: {exc} (Context: {src_file})") self._record_failure( diff --git a/core/repo.py b/core/repo.py index 4c368911..2c261f1d 100644 --- a/core/repo.py +++ b/core/repo.py @@ -421,6 +421,14 @@ def normalize_port_mappings(data): return None +def _is_strict_json(value) -> bool: + """Return True when *value* survives a strict JSON round trip unchanged, as NetBox will store it.""" + try: + return json.loads(json.dumps(value, allow_nan=False)) == value + except (TypeError, ValueError): + return False + + def parse_single_file(file): """Load a YAML device mapping, convert its `manufacturer` to a slug dictionary, and record the source path. @@ -440,8 +448,17 @@ def parse_single_file(file): # Use only slug for manufacturer lookup - more resilient to case mismatches # (e.g., RuggedCOM vs RuggedCom in upstream data) data["manufacturer"] = {"slug": re_sub(r"\W+", "-", manufacturer.lower())} - if "profile" in data and isinstance(data["profile"], str): - data["profile"] = {"name": data["profile"]} + profile = data.get("profile") + if isinstance(profile, str): + data["profile"] = {"name": profile} + elif profile is not None and not (isinstance(profile, dict) and isinstance(profile.get("name"), str)): + return f"Error: profile must be a name or a mapping with a name, not {profile!r}" + attribute_data = data.get("attribute_data") + if attribute_data is not None and not isinstance(attribute_data, dict): + return f"Error: attribute_data must be a mapping, not {attribute_data!r}" + for field in ("profile", "attribute_data"): + if not _is_strict_json(data.get(field)): + return f"Error: {field} must hold JSON values only (string keys, finite numbers): {data[field]!r}" data["src"] = file err = normalize_port_mappings(data) if err: diff --git a/tests/fixtures/module-types/TestVendor/power-supply.yaml b/tests/fixtures/module-types/TestVendor/power-supply.yaml new file mode 100644 index 00000000..a9391f1a --- /dev/null +++ b/tests/fixtures/module-types/TestVendor/power-supply.yaml @@ -0,0 +1,13 @@ +--- +manufacturer: TestVendor +model: Test Power Supply +part_number: TPS-650 +profile: Power supply +attribute_data: + input_current: AC + input_voltage: 230 + wattage: 650 + hot_swappable: true +power-ports: + - name: '{module}:IN' + type: iec-60320-c14 diff --git a/tests/helpers.py b/tests/helpers.py index c6f0f381..732c0e12 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,5 +1,6 @@ """Shared test utilities for the NetBox device-type importer test suite.""" +import copy import json import re import threading @@ -58,6 +59,15 @@ def recording_handle(): return handle, console +class _Refused(Exception): + """A request NetBox answers with 400 and *body*.""" + + def __init__(self, body): + """Keep the body to answer with.""" + super().__init__(body) + self.body = body + + class FakeNetBox: """A local HTTP server answering the slice of the NetBox REST API the importer uses. @@ -78,6 +88,9 @@ class FakeNetBox: # GraphQL collections are named "_list"; the client asks for one per query. _LIST_KEY = re.compile(r"\b(\w+_list)\b") + # A nested reference NetBox looks up by its attributes, keyed by field name. + _NESTED_REFERENCES = (("manufacturer", "manufacturers"), ("profile", "module_type_profiles")) + def __init__(self, errors=None, netbox_version="4.7.0", **collections): """Start the server with *collections* seeded, keyed by collection name. @@ -152,10 +165,25 @@ def _create(self, name, payload): return [self._create_one(name, item) for item in payload] return self._create_one(name, payload) + def _resolve_references(self, payload): + """Replace each nested reference with the object it names, as NetBox does. + + Raises _Refused with NetBox's own 400 body when no object matches. + """ + for field, collection in self._NESTED_REFERENCES: + wanted = payload.get(field) + if not isinstance(wanted, dict): + continue + match = next((r for r in self.collection(collection) if wanted.items() <= r.items()), None) + if match is None: + raise _Refused({field: [f"Related object not found using the provided attributes: {wanted}"]}) + payload[field] = {"id": match["id"], "name": match.get("name")} + return payload + def _create_one(self, name, payload): """Store one new record and return it as NetBox would echo it back.""" collection = self.collection(name) - record = {"id": 1000 + len(collection), **payload} + record = {"id": 1000 + len(collection), **self._resolve_references(payload)} # NetBox echoes a foreign key back as a nested object, not the id it was given. if isinstance(record.get("manufacturer"), int): owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) @@ -163,6 +191,14 @@ def _create_one(self, name, payload): collection.append(record) return record + @staticmethod + def _answer(status, produce): + """Return (status, body) for what *produce* returns, or the 400 it raises.""" + try: + return status, produce() + except _Refused as refusal: + return 400, refusal.body + def _patch(self, name, payload): """Merge a bulk update into the stored records and return the updated ones.""" updated = [] @@ -170,7 +206,7 @@ def _patch(self, name, payload): record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) if record is None: continue - record.update(entry) + record.update(self._resolve_references(entry)) updated.append(record) return updated @@ -224,7 +260,7 @@ def do_POST(self): state.requests.append(("POST", name, payload)) if self._refused(name): return - self._reply(201, state._create(name, payload)) + self._reply(*state._answer(201, lambda: state._create(name, copy.deepcopy(payload)))) def do_PATCH(self): name = self._collection_name() @@ -232,7 +268,7 @@ def do_PATCH(self): state.requests.append(("PATCH", name, payload)) if self._refused(name): return - self._reply(200, state._patch(name, payload)) + self._reply(*state._answer(200, lambda: state._patch(name, copy.deepcopy(payload)))) def log_message(self, *args): """Silence the default stderr access log.""" diff --git a/tests/integration/test_import.py b/tests/integration/test_import.py index 961697e5..0237ea97 100644 --- a/tests/integration/test_import.py +++ b/tests/integration/test_import.py @@ -38,11 +38,15 @@ - FP1 → RP1 position 1; FP2 → RP1 position 2 (same rear port). F. Module-type component creation - All module component types created; front-port rear_port mapping set. + - Profile and attribute data stored on the module type. G. Idempotency - Second run: 0 new, 0 modified device types and module types. H. Update mode - Delete one interface via REST API; re-run with --update; verify it is recreated with the original type value. +I. Module-type profile update + - Change the profile and one attribute via REST API; re-run with --update; + verify both are restored from the YAML. Usage:: @@ -126,6 +130,12 @@ def api(path: str, **params) -> dict: return r.json() +def api_patch(path: str, payload: dict) -> None: + r = session.patch(f"{NETBOX_URL}/api{path}", json=payload, timeout=30) + if r.status_code != 200: + fail(f"PATCH {path} returned {r.status_code}: {r.text[:200]}") + + def api_delete(path: str) -> None: r = session.delete(f"{NETBOX_URL}/api{path}", timeout=30) if r.status_code not in (200, 204): @@ -373,6 +383,24 @@ def test_module_types() -> None: fail(f"full-module: {len(broken)} front ports have empty rear-port linkage: {broken}") ok("full-module: front port rear-port mapping set correctly") + assert_module_profile(get_one("/dcim/module-types/", model="Test Power Supply")) + + +# The profile and attribute data in tests/fixtures/module-types/TestVendor/power-supply.yaml. +PSU_PROFILE = "Power supply" +PSU_ATTRIBUTES = {"input_current": "AC", "input_voltage": 230, "wattage": 650, "hot_swappable": True} + + +def assert_module_profile(module_type: dict) -> None: + """Fail unless *module_type* (a REST record) carries the fixture's profile and attributes.""" + profile = (module_type.get("profile") or {}).get("name") + if profile != PSU_PROFILE: + fail(f"power-supply: profile = {profile!r}, expected {PSU_PROFILE!r}") + ok(f"power-supply: profile = {profile!r}") + if module_type.get("attributes") != PSU_ATTRIBUTES: + fail(f"power-supply: attributes = {module_type.get('attributes')!r}, expected {PSU_ATTRIBUTES!r}") + ok("power-supply: attributes match the YAML") + # ────────────────────────────────────────────────────────────────────────────── # Scenario D: GraphQL schema consistency @@ -471,6 +499,15 @@ def test_graphql_schema() -> None: ok("get_module_types() completed without schema error") +def test_graphql_module_profile() -> None: + """The change detection reads profile and attribute_data from this query.""" + client = NetBoxGraphQLClient(NETBOX_URL, NETBOX_TOKEN, ignore_ssl=IGNORE_SSL) + psu = client.get_module_types().get("testvendor", {}).get("Test Power Supply") + if psu is None or psu.profile is None or psu.profile.name != PSU_PROFILE or psu.attribute_data != PSU_ATTRIBUTES: + fail(f"get_module_types() did not return the power supply profile and attribute data: {psu!r}") + ok("get_module_types() returned profile and attribute_data") + + # ────────────────────────────────────────────────────────────────────────────── # Scenario G: Idempotency # ────────────────────────────────────────────────────────────────────────────── @@ -570,3 +607,32 @@ def test_update_mode() -> None: if not re.search(r"(? None: + print("\n=== Scenario I: Update mode — module type profile and attributes ===") + psu = get_one("/dcim/module-types/", model="Test Power Supply") + fan = get_one("/dcim/module-type-profiles/", name="Fan") + api_patch(f"/dcim/module-types/{psu['id']}/", {"profile": fan["id"], "attributes": {"wattage": 100}}) + changed = get_one("/dcim/module-types/", model="Test Power Supply") + if changed["profile"]["name"] != "Fan" or changed["attributes"].get("wattage") != 100: + fail(f"REST PATCH did not change the power supply: {changed['profile']!r} {changed['attributes']!r}") + ok("Changed profile to Fan and wattage to 100") + + result = run_importer("--update") + if result.returncode != 0: + fail(f"Importer --update exited with code {result.returncode}") + assert_module_profile(get_one("/dcim/module-types/", model="Test Power Supply")) + + # A no-change run prints no change banner, so first prove the run reached its summary. + result2 = run_importer() + if result2.returncode != 0 or not re.search(r"(? Date: Sat, 17 Oct 2026 14:35:15 +0200 Subject: [PATCH 22/39] fix(module-types): drop the old profile's attributes when the profile changes NetBox's AttributesField merges a non-empty PATCHed value into the stored attributes. A profile change from Fan {rpm} to Power supply kept rpm, and a profile schema that rejects rpm failed the update on every run. When the profile changes and NetBox holds attributes the YAML does not declare, the importer first PATCHes profile and attributes to null (ModuleType.clean() drops the attributes of a type without a profile). Then it writes the new profile and the declared attributes. Verified on NetBox 4.7 with the REST serializer in a rolled-back transaction. The test fake now merges attributes the way NetBox does, so the profile tests run against the real semantics. Scenario I of the integration test adds a Fan attribute and expects it to be gone after --update. --- core/netbox_api.py | 14 +++++++++++++- tests/helpers.py | 8 ++++++++ tests/integration/test_import.py | 4 ++-- tests/test_module_type_profile.py | 19 +++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 85468c6d..20c9e8a1 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1674,7 +1674,7 @@ def _profile_changes(module_type, existing_module): """Return (field, current, wanted) for the profile and for each declared attribute that differs. NetBox merges PATCHed attributes into the stored ones, so an attribute the YAML omits is - left alone, the same as an omitted property. + left alone, the same as an omitted property. A profile change drops the old attributes. """ changes: list[tuple[str, Any, Any]] = [] if "profile" in module_type and hasattr(existing_module, "profile"): @@ -1764,9 +1764,15 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + stale_attributes = False for field, _current, _wanted in self._profile_changes(curr_mt, module_type_res): if field == "profile": updates["profile"] = curr_mt["profile"] + stored = getattr(module_type_res, "attribute_data", None) + declared = curr_mt.get("attribute_data") or {} + stale_attributes = bool(stored) and (not isinstance(stored, dict) or bool(stored.keys() - declared)) + if stale_attributes and declared: + updates[_REST_ATTRIBUTES_FIELD] = declared else: updates[_REST_ATTRIBUTES_FIELD] = curr_mt["attribute_data"] or {} if self.module_bay_types: @@ -1784,6 +1790,12 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): if not updates: return True, False try: + if stale_attributes: + # NetBox merges PATCHed attributes into the stored ones; dropping the profile clears them. + _retry_on_connection_error( + self.netbox.dcim.module_types.update, + [{"id": module_type_res.id, "profile": None, _REST_ATTRIBUTES_FIELD: None}], + ) _retry_on_connection_error(self.netbox.dcim.module_types.update, [{"id": module_type_res.id, **updates}]) self.handle.verbose_log( f"Module Type Updated: {module_type_res.manufacturer.name} - " diff --git a/tests/helpers.py b/tests/helpers.py index 732c0e12..3265dd94 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -206,6 +206,14 @@ def _patch(self, name, payload): record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) if record is None: continue + if "attributes" in entry: + # NetBox's AttributesField merges a non-empty value into the stored attributes. + attributes = entry.pop("attributes") + stored = record.get("attribute_data") + merge = attributes and isinstance(stored, dict) + record["attribute_data"] = {**stored, **attributes} if merge else attributes + if "profile" in entry and entry["profile"] is None: + record["attribute_data"] = None # ModuleType.clean() drops the attributes of a type without a profile record.update(self._resolve_references(entry)) updated.append(record) return updated diff --git a/tests/integration/test_import.py b/tests/integration/test_import.py index 0237ea97..49dcbbf6 100644 --- a/tests/integration/test_import.py +++ b/tests/integration/test_import.py @@ -618,11 +618,11 @@ def test_update_mode_module_profile() -> None: print("\n=== Scenario I: Update mode — module type profile and attributes ===") psu = get_one("/dcim/module-types/", model="Test Power Supply") fan = get_one("/dcim/module-type-profiles/", name="Fan") - api_patch(f"/dcim/module-types/{psu['id']}/", {"profile": fan["id"], "attributes": {"wattage": 100}}) + api_patch(f"/dcim/module-types/{psu['id']}/", {"profile": fan["id"], "attributes": {"wattage": 100, "rpm": 1000}}) changed = get_one("/dcim/module-types/", model="Test Power Supply") if changed["profile"]["name"] != "Fan" or changed["attributes"].get("wattage") != 100: fail(f"REST PATCH did not change the power supply: {changed['profile']!r} {changed['attributes']!r}") - ok("Changed profile to Fan and wattage to 100") + ok("Changed profile to Fan, wattage to 100, and added the Fan attribute rpm") result = run_importer("--update") if result.returncode != 0: diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 4886b860..8bf7afa5 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -189,6 +189,25 @@ def test_a_changed_profile_and_attribute_are_patched(self, netbox, server, defin {"id": 7, "profile": {"name": "Transceiver"}, "attributes": SFP_ATTRIBUTES} ] + @pytest.mark.parametrize("stored", [{"rpm": 3000}, {**SFP_ATTRIBUTES, "rpm": 3000}]) + def test_a_changed_profile_drops_the_old_profile_attributes(self, netbox, server, definition, stored): + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, stored) + + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (True, True) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} + assert stored_type["attribute_data"] == SFP_ATTRIBUTES + + def test_an_unchanged_profile_keeps_attributes_the_yaml_omits(self, netbox, server, definition): + nb, _ = netbox + record = existing(server, {"id": 3, "name": "Transceiver"}, {**SFP_ATTRIBUTES, "reach": 300, "rpm": 3000}) + + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (True, True) + + assert server.collection("module_types")[0]["attribute_data"] == {**SFP_ATTRIBUTES, "rpm": 3000} + def test_an_unknown_profile_fails_the_module_type(self, netbox, server, definition): nb, console = netbox record = existing(server, {"id": 4, "name": "Fan"}) From 43211c97d840dc5a8b2a162c1ceb7b4f5f0f1e5a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 17 Oct 2026 14:46:54 +0200 Subject: [PATCH 23/39] fix(module-types): restore the old profile when a profile change fails The profile change clears the stored profile and attributes first, then writes the new ones. NetBox refuses a bulk PATCH that names one object twice, so the two steps cannot share a transaction. When the second PATCH failed (for example, an unknown profile), the module type kept neither the old nor the new profile. On that failure, PATCH the old profile (by name) and the old attributes back, then report the original error. A failed restore is logged. The test fake now resolves references before it writes anything, as NetBox validates the whole object before it saves. --- core/netbox_api.py | 29 +++++++++++++++++++++++++---- tests/helpers.py | 3 ++- tests/test_module_type_profile.py | 11 +++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 20c9e8a1..bc79c720 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1791,12 +1791,11 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): return True, False try: if stale_attributes: - # NetBox merges PATCHed attributes into the stored ones; dropping the profile clears them. + self._patch_profile_change(module_type_res, updates) + else: _retry_on_connection_error( - self.netbox.dcim.module_types.update, - [{"id": module_type_res.id, "profile": None, _REST_ATTRIBUTES_FIELD: None}], + self.netbox.dcim.module_types.update, [{"id": module_type_res.id, **updates}] ) - _retry_on_connection_error(self.netbox.dcim.module_types.update, [{"id": module_type_res.id, **updates}]) self.handle.verbose_log( f"Module Type Updated: {module_type_res.manufacturer.name} - " f"{module_type_res.model} - {module_type_res.id} " @@ -1812,6 +1811,28 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): return False, False return True, True + def _patch_profile_change(self, module_type_res, updates): + """PATCH *updates* on a cleared module type, and put the old profile and attributes back if it fails. + + NetBox merges PATCHed attributes into the stored ones, so the old profile's attributes must go + first. A module type without a profile has no attributes, so dropping the profile clears them. + """ + update = self.netbox.dcim.module_types.update + type_id = module_type_res.id + _retry_on_connection_error(update, [{"id": type_id, "profile": None, _REST_ATTRIBUTES_FIELD: None}]) + try: + _retry_on_connection_error(update, [{"id": type_id, **updates}]) + except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS): + old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None + restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} + try: + _retry_on_connection_error(update, [restore]) + except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS) as restore_error: + self.handle.log( + f"Could not restore the profile and attributes of Module Type {type_id}: {restore_error}" + ) + raise + def _create_module_type_components(self, curr_mt, module_type_id, src_file): """Create all component templates for a newly created module type. diff --git a/tests/helpers.py b/tests/helpers.py index 3265dd94..85229a09 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -206,6 +206,7 @@ def _patch(self, name, payload): record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) if record is None: continue + entry = self._resolve_references(entry) # a refused entry changes nothing if "attributes" in entry: # NetBox's AttributesField merges a non-empty value into the stored attributes. attributes = entry.pop("attributes") @@ -214,7 +215,7 @@ def _patch(self, name, payload): record["attribute_data"] = {**stored, **attributes} if merge else attributes if "profile" in entry and entry["profile"] is None: record["attribute_data"] = None # ModuleType.clean() drops the attributes of a type without a profile - record.update(self._resolve_references(entry)) + record.update(entry) updated.append(record) return updated diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 8bf7afa5..6acf578e 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -200,6 +200,17 @@ def test_a_changed_profile_drops_the_old_profile_attributes(self, netbox, server assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} assert stored_type["attribute_data"] == SFP_ATTRIBUTES + def test_a_rejected_profile_change_keeps_the_old_profile_and_attributes(self, netbox, server, definition): + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) + curr_mt = definition(SFP_YAML.replace("profile: Transceiver", "profile: No Such Profile")) + + assert nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") == (False, False) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 4, "name": "Fan"} + assert stored_type["attribute_data"] == {"rpm": 3000} + def test_an_unchanged_profile_keeps_attributes_the_yaml_omits(self, netbox, server, definition): nb, _ = netbox record = existing(server, {"id": 3, "name": "Transceiver"}, {**SFP_ATTRIBUTES, "reach": 300, "rpm": 3000}) From f12ce58d0fdef64d4117009c0a6d2f0c5924cffc Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 17 Oct 2026 15:05:38 +0200 Subject: [PATCH 24/39] fix(module-types): make the profile-change restore survive lost replies A PATCH can commit while its reply is lost. Three gaps followed: - A clearing PATCH that committed but failed every retry skipped the restore, so the module type kept no profile and no attributes. - A restore after a lost write merged the old attributes into the new ones, because NetBox merges a non-empty attributes PATCH. - Stored attributes of {} skipped the clear. A profile-only change to a profile with required keys then failed, while no attributes (null) is valid, as on create. The clear now runs inside the guarded block, the restore clears again before it writes the old values, and any stored attributes value that the new state does not keep exactly triggers the clear. The test fake can now lose the reply of a committed PATCH. --- core/netbox_api.py | 10 ++++++++-- tests/helpers.py | 11 ++++++++++- tests/test_module_type_profile.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index bc79c720..05261df8 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1770,7 +1770,10 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): updates["profile"] = curr_mt["profile"] stored = getattr(module_type_res, "attribute_data", None) declared = curr_mt.get("attribute_data") or {} - stale_attributes = bool(stored) and (not isinstance(stored, dict) or bool(stored.keys() - declared)) + # A merge keeps every stored key, and an undeclared set must end as no attributes, as on create. + stale_attributes = stored is not None and ( + not declared or not isinstance(stored, dict) or bool(stored.keys() - declared) + ) if stale_attributes and declared: updates[_REST_ATTRIBUTES_FIELD] = declared else: @@ -1816,16 +1819,19 @@ def _patch_profile_change(self, module_type_res, updates): NetBox merges PATCHed attributes into the stored ones, so the old profile's attributes must go first. A module type without a profile has no attributes, so dropping the profile clears them. + A lost reply can hide a committed write, so the restore clears again before it writes. """ update = self.netbox.dcim.module_types.update type_id = module_type_res.id - _retry_on_connection_error(update, [{"id": type_id, "profile": None, _REST_ATTRIBUTES_FIELD: None}]) + cleared = {"id": type_id, "profile": None, _REST_ATTRIBUTES_FIELD: None} try: + _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [{"id": type_id, **updates}]) except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS): old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} try: + _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [restore]) except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS) as restore_error: self.handle.log( diff --git a/tests/helpers.py b/tests/helpers.py index 85229a09..23b582b2 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -102,6 +102,8 @@ def __init__(self, errors=None, netbox_version="4.7.0", **collections): self.errors = dict(errors or {}) self.netbox_version = netbox_version self.requests = [] + # 1-based PATCH numbers that commit, then close the connection with no reply. + self.lost_patch_responses = set() self._server = HTTPServer(("127.0.0.1", 0), self._handler()) threading.Thread(target=self._server.serve_forever, daemon=True).start() @@ -219,6 +221,13 @@ def _patch(self, name, payload): updated.append(record) return updated + def _reply_to_patch(self, handler, answer): + """Send *answer*, or close the connection when the test chose to lose this PATCH's reply.""" + if sum(method == "PATCH" for method, _, _ in self.requests) in self.lost_patch_responses: + handler.close_connection = True + return + handler._reply(*answer) + def _handler(self): """Build the request handler class bound to this server's state.""" state = self @@ -277,7 +286,7 @@ def do_PATCH(self): state.requests.append(("PATCH", name, payload)) if self._refused(name): return - self._reply(*state._answer(200, lambda: state._patch(name, copy.deepcopy(payload)))) + state._reply_to_patch(self, state._answer(200, lambda: state._patch(name, copy.deepcopy(payload)))) def log_message(self, *args): """Silence the default stderr access log.""" diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 6acf578e..63d30acf 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -211,6 +211,36 @@ def test_a_rejected_profile_change_keeps_the_old_profile_and_attributes(self, ne assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} + @pytest.mark.parametrize("lost", [{1, 2, 3, 4}, {2, 3, 4, 5}], ids=["clear", "write"]) + def test_a_profile_change_with_lost_responses_keeps_the_old_profile_and_attributes( + self, netbox, server, definition, monkeypatch, lost + ): + """Each PATCH commits but its reply never arrives, so every retry is exhausted.""" + monkeypatch.setattr("core.netbox_api.time.sleep", lambda _seconds: None) + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) + server.lost_patch_responses = lost + + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (False, False) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 4, "name": "Fan"} + assert stored_type["attribute_data"] == {"rpm": 3000} + + def test_a_profile_only_change_drops_empty_old_attributes(self, netbox, server, definition): + """A new profile can reject {} (required keys) but accepts no attributes, as on create.""" + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {}) + + assert nb._try_update_module_type(definition(SFP_YAML.split("attribute_data")[0]), record, "x.yaml") == ( + True, + True, + ) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} + assert stored_type["attribute_data"] is None + def test_an_unchanged_profile_keeps_attributes_the_yaml_omits(self, netbox, server, definition): nb, _ = netbox record = existing(server, {"id": 3, "name": "Transceiver"}, {**SFP_ATTRIBUTES, "reach": 300, "rpm": 3000}) From 01374ef8ec3c2d5a83f1cad22d8acf7c17f302e1 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 17 Oct 2026 15:14:14 +0200 Subject: [PATCH 25/39] fix(module-types): restore the profile after any transport error A reply cut off after its headers raises ChunkedEncodingError, which is not one of the retried connection errors. It skipped the restore and left the module type with no profile and no attributes. The profile change now restores on any requests transport error, then re-raises it. --- core/netbox_api.py | 4 ++-- tests/helpers.py | 14 ++++++++++++-- tests/test_module_type_profile.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 05261df8..deef7f55 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1827,13 +1827,13 @@ def _patch_profile_change(self, module_type_res, updates): try: _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [{"id": type_id, **updates}]) - except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS): + except (pynetbox.RequestError, requests.exceptions.RequestException): old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} try: _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [restore]) - except (pynetbox.RequestError, *_RETRYABLE_EXCEPTIONS) as restore_error: + except (pynetbox.RequestError, requests.exceptions.RequestException) as restore_error: self.handle.log( f"Could not restore the profile and attributes of Module Type {type_id}: {restore_error}" ) diff --git a/tests/helpers.py b/tests/helpers.py index 23b582b2..8f3f60a1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -102,8 +102,9 @@ def __init__(self, errors=None, netbox_version="4.7.0", **collections): self.errors = dict(errors or {}) self.netbox_version = netbox_version self.requests = [] - # 1-based PATCH numbers that commit, then close the connection with no reply. + # 1-based PATCH numbers that commit, then close the connection with no reply or a cut-off one. self.lost_patch_responses = set() + self.truncated_patch_responses = set() self._server = HTTPServer(("127.0.0.1", 0), self._handler()) threading.Thread(target=self._server.serve_forever, daemon=True).start() @@ -223,7 +224,16 @@ def _patch(self, name, payload): def _reply_to_patch(self, handler, answer): """Send *answer*, or close the connection when the test chose to lose this PATCH's reply.""" - if sum(method == "PATCH" for method, _, _ in self.requests) in self.lost_patch_responses: + number = sum(method == "PATCH" for method, _, _ in self.requests) + if number in self.lost_patch_responses: + handler.close_connection = True + return + if number in self.truncated_patch_responses: + handler.send_response(answer[0]) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", "1000") + handler.end_headers() + handler.wfile.write(b"[{") handler.close_connection = True return handler._reply(*answer) diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 63d30acf..f55fb75e 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -8,6 +8,7 @@ import pynetbox import pytest +import requests from helpers import FakeNetBox, recording_handle from core.graphql_client import _to_dotdict @@ -227,6 +228,18 @@ def test_a_profile_change_with_lost_responses_keeps_the_old_profile_and_attribut assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} + def test_a_truncated_clear_reply_restores_the_old_profile_and_attributes(self, netbox, server, definition): + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) + server.truncated_patch_responses = {1} + + with pytest.raises(requests.exceptions.ChunkedEncodingError): + nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 4, "name": "Fan"} + assert stored_type["attribute_data"] == {"rpm": 3000} + def test_a_profile_only_change_drops_empty_old_attributes(self, netbox, server, definition): """A new profile can reject {} (required keys) but accepts no attributes, as on create.""" nb, _ = netbox From dc423f77aafb1bc520ee31fd76c787868301e668 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 17 Oct 2026 15:22:43 +0200 Subject: [PATCH 26/39] fix(module-types): restore the profile after any failed profile change A 200 reply with invalid JSON raises pynetbox's ContentError, which the restore did not catch either. Listing exception types cannot cover every way a reply can break after the clear has committed. Restore on any exception, then re-raise it, so the caller sees the original error. --- core/netbox_api.py | 4 ++-- tests/helpers.py | 7 ++++--- tests/test_module_type_profile.py | 12 +++++++++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index deef7f55..017c8854 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1827,13 +1827,13 @@ def _patch_profile_change(self, module_type_res, updates): try: _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [{"id": type_id, **updates}]) - except (pynetbox.RequestError, requests.exceptions.RequestException): + except Exception: # any failure after the clear leaves an unknown state; restore, then re-raise old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} try: _retry_on_connection_error(update, [cleared]) _retry_on_connection_error(update, [restore]) - except (pynetbox.RequestError, requests.exceptions.RequestException) as restore_error: + except Exception as restore_error: # noqa: BLE001 # logged; the original error is re-raised self.handle.log( f"Could not restore the profile and attributes of Module Type {type_id}: {restore_error}" ) diff --git a/tests/helpers.py b/tests/helpers.py index 8f3f60a1..e69c0733 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -102,9 +102,10 @@ def __init__(self, errors=None, netbox_version="4.7.0", **collections): self.errors = dict(errors or {}) self.netbox_version = netbox_version self.requests = [] - # 1-based PATCH numbers that commit, then close the connection with no reply or a cut-off one. + # 1-based PATCH numbers that commit, then reply with nothing, a cut-off body, or invalid JSON. self.lost_patch_responses = set() self.truncated_patch_responses = set() + self.garbled_patch_responses = set() self._server = HTTPServer(("127.0.0.1", 0), self._handler()) threading.Thread(target=self._server.serve_forever, daemon=True).start() @@ -228,10 +229,10 @@ def _reply_to_patch(self, handler, answer): if number in self.lost_patch_responses: handler.close_connection = True return - if number in self.truncated_patch_responses: + if number in self.truncated_patch_responses | self.garbled_patch_responses: handler.send_response(answer[0]) handler.send_header("Content-Type", "application/json") - handler.send_header("Content-Length", "1000") + handler.send_header("Content-Length", "1000" if number in self.truncated_patch_responses else "2") handler.end_headers() handler.wfile.write(b"[{") handler.close_connection = True diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index f55fb75e..68f20c95 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -228,12 +228,18 @@ def test_a_profile_change_with_lost_responses_keeps_the_old_profile_and_attribut assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} - def test_a_truncated_clear_reply_restores_the_old_profile_and_attributes(self, netbox, server, definition): + @pytest.mark.parametrize( + ("reply", "error"), + [("truncated", requests.exceptions.ChunkedEncodingError), ("garbled", pynetbox.core.query.ContentError)], + ) + def test_a_broken_clear_reply_restores_the_old_profile_and_attributes( + self, netbox, server, definition, reply, error + ): nb, _ = netbox record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) - server.truncated_patch_responses = {1} + setattr(server, f"{reply}_patch_responses", {1}) - with pytest.raises(requests.exceptions.ChunkedEncodingError): + with pytest.raises(error): nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") [stored_type] = server.collection("module_types") From d225dde7b2a20493627d2d40d2898ffb866f1728 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 17 Oct 2026 15:34:16 +0200 Subject: [PATCH 27/39] fix(module-types): keep declared empty attributes and restore after a failed clear - A profile change with an explicit `attribute_data: {}` ended with no attributes (null). A declared value is now always sent. An empty PATCH replaces the stored attributes, so it needs no clear. - When the clear before a restore failed, the restore did not run. The clear and the restore now run as separate steps, and each failure is logged. --- core/netbox_api.py | 23 ++++++++++++----------- tests/test_module_type_profile.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 017c8854..53e1863a 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1770,12 +1770,13 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): updates["profile"] = curr_mt["profile"] stored = getattr(module_type_res, "attribute_data", None) declared = curr_mt.get("attribute_data") or {} - # A merge keeps every stored key, and an undeclared set must end as no attributes, as on create. + if "attribute_data" in curr_mt: + updates[_REST_ATTRIBUTES_FIELD] = declared + # A non-empty PATCH keeps every stored key; an undeclared set must end as no attributes, as on create. stale_attributes = stored is not None and ( - not declared or not isinstance(stored, dict) or bool(stored.keys() - declared) + "attribute_data" not in curr_mt + or (bool(declared) and (not isinstance(stored, dict) or bool(stored.keys() - declared))) ) - if stale_attributes and declared: - updates[_REST_ATTRIBUTES_FIELD] = declared else: updates[_REST_ATTRIBUTES_FIELD] = curr_mt["attribute_data"] or {} if self.module_bay_types: @@ -1830,13 +1831,13 @@ def _patch_profile_change(self, module_type_res, updates): except Exception: # any failure after the clear leaves an unknown state; restore, then re-raise old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} - try: - _retry_on_connection_error(update, [cleared]) - _retry_on_connection_error(update, [restore]) - except Exception as restore_error: # noqa: BLE001 # logged; the original error is re-raised - self.handle.log( - f"Could not restore the profile and attributes of Module Type {type_id}: {restore_error}" - ) + for step, payload in (("clear", cleared), ("restore", restore)): + try: + _retry_on_connection_error(update, [payload]) + except Exception as step_error: # noqa: BLE001 # logged; the original error is re-raised + self.handle.log( + f"Could not {step} the profile and attributes of Module Type {type_id}: {step_error}" + ) raise def _create_module_type_components(self, curr_mt, module_type_id, src_file): diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 68f20c95..d2ac5fc6 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -246,6 +246,30 @@ def test_a_broken_clear_reply_restores_the_old_profile_and_attributes( assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} + def test_a_failed_recovery_clear_still_restores_the_old_profile(self, netbox, server, definition): + nb, console = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) + curr_mt = definition(SFP_YAML.replace("profile: Transceiver", "profile: No Such Profile")) + server.garbled_patch_responses = {3} + + assert nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") == (False, False) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 4, "name": "Fan"} + assert stored_type["attribute_data"] == {"rpm": 3000} + assert any("Could not clear" in line for line in console.lines) + + def test_a_profile_change_keeps_explicit_empty_attributes(self, netbox, server, definition): + nb, _ = netbox + record = existing(server, {"id": 4, "name": "Fan"}, {}) + curr_mt = definition(SFP_YAML.split("attribute_data")[0] + "attribute_data: {}\n") + + assert nb._try_update_module_type(curr_mt, record, "x.yaml") == (True, True) + + [stored_type] = server.collection("module_types") + assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} + assert stored_type["attribute_data"] == {} + def test_a_profile_only_change_drops_empty_old_attributes(self, netbox, server, definition): """A new profile can reject {} (required keys) but accepts no attributes, as on create.""" nb, _ = netbox From e49eb5edaae7bf32298af828430e0c686b6d5e0e Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Sat, 17 Oct 2026 21:37:07 +0200 Subject: [PATCH 28/39] fix(netbox): retry broken write replies without repeating a committed write (#150) * fix(netbox): retry writes after a cut-off or non-JSON reply A write whose reply was cut off (requests ChunkedEncodingError) or was not JSON (pynetbox ContentError) was not retried. Neither error is a ConnectionError, so it escaped every write handler and stopped the import with a traceback. Both are now in _RETRYABLE_EXCEPTIONS: the write is retried, and after the last retry only that object fails. The module bay type catalog now also reports a ContentError as a ModuleBayTypeError, so it fails one definition and not the run. * fix(netbox): look up a created object before retrying its write A write can commit in NetBox while its reply is lost, cut off or not JSON. The importer then retried the same POST. NetBox refused the second POST because the object already existed, so the importer recorded a failure for an object that it had created. A bulk component create failed the whole batch in the same way. A retried bulk DELETE had the same problem: NetBox refuses a delete that names a missing id, so the retry reported an error for a component that was gone. The retry machinery now lives in core/retry.py, and both core/netbox_api.py and core/module_bay_types.py use it. - call_with_retry() takes an optional find_existing hook. Before each retry, the hook checks if the earlier attempt committed. If so, the helper returns that result and does not send the write again. A transient error in the check counts as a failed attempt. - create_with_retry() looks the object up by its natural key and returns the same shape that the create returns: one Record for a dict payload, a list in payload order for a list payload. A bulk lookup that finds only some items does not count. NetBox bulk create is atomic, so a partial match means another writer, and the create is sent again. - delete_with_retry() looks the id up before a retry. If the object is gone, the delete counts as done. A delete that NetBox refuses on the first attempt is still an error. Natural keys: manufacturer slug; manufacturer and model for device, rack and module types; parent and name for component templates; manufacturer and name for module bay types. The module bay type catalog did not retry. Its lookups and creates now go through the shared helpers. After the retries are exhausted it still raises ModuleBayTypeError, so only that definition fails. PATCH writes do not change. They are idempotent. The test fake now handles DELETE, refuses a duplicate natural key and a delete of a missing id with a 400 as NetBox does, and can break GET replies. New tests drive a real pynetbox client against it. * fix(images): retry image uploads and deletes through pynetbox The device-type image PATCH, the image-attachment POST and the image-attachment DELETE called requests directly. They had no retry after a lost, cut-off or non-JSON reply. They now go through pynetbox and core.retry, like the other writes. - The attachment upload uses create_with_retry. Before a retry, it looks up the attachment by object_type, object_id and name, so a lost reply does not create a second attachment. When that lookup finds more than one attachment, the upload logs the problem and returns False. - The attachment delete uses delete_with_retry (NetBoxModelViewSet gives image-attachments a bulk delete). - The device-type image PATCH is idempotent, so it uses call_with_retry. pynetbox sends multipart only for a dict payload, and Endpoint.update always sends a list. The PATCH therefore uses a pynetbox Request with the device-type id as key, the same call Record.save makes, with no GET first. - requests reads each file to its end when it builds the body, so a retry sent an empty file. call_with_retry now rewinds every file in a dict argument before each attempt. - upload_images returns True or False. A device type stores its image hashes only after the upload succeeds. Before, a failed re-upload of a changed image stored the new hash, and --verify-images then skipped that image on the next run. - A rejection logs the HTTP status and NetBox's escaped, truncated body, as before. Exhausted retries log the error and return False. Neither stops the run. - DeviceTypes no longer takes ignore_ssl: the pynetbox session already carries the verify setting. The upload tests run a real pynetbox client against the local fake NetBox. The fake now parses multipart bodies, records the bytes of each file, answers a PATCH on one object's URL, can drop a request before it commits, and can refuse with a raw body. The mock tests of the old requests calls are removed. * fix(netbox): give each REST call a timeout The pynetbox session had no timeout, so a stalled NetBox could hang the run. The image writes lost their own 30 s and 60 s timeouts when they moved onto pynetbox. The session now sends a (30, 300) second (connect, read) timeout with every request. The read limit is long so that slow bulk writes still finish. A timeout is retryable, and the lookup before a retry stops a duplicate create. The fake NetBox now runs one thread per request and has a "stalled" reply mode. It also sends the API-Version header, so a test can build the client through connect_api. * fix(retry): look up a bulk create in chunks of 50 keys The lookup before a retry put every name of a bulk create into one GET query string. A bulk create of a few hundred components then makes a URL that can pass a proxy URL limit. The server answers 414, the RequestError escapes the retry, and the committed components are logged as a failed create. The lookup now sends at most 50 keys per request. --- core/module_bay_types.py | 28 +- core/netbox_api.py | 358 +++++++++++------------ core/retry.py | 99 +++++++ tests/helpers.py | 246 ++++++++++++---- tests/test_image_writes.py | 338 ++++++++++++++++++++++ tests/test_module_bay_type_sync.py | 1 - tests/test_module_bay_types.py | 45 +++ tests/test_module_type_profile.py | 63 +++-- tests/test_netbox_api.py | 441 ++--------------------------- tests/test_write_retries.py | 241 ++++++++++++++++ 10 files changed, 1175 insertions(+), 685 deletions(-) create mode 100644 core/retry.py create mode 100644 tests/test_image_writes.py create mode 100644 tests/test_write_retries.py diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 76697cd6..92457dff 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -19,6 +19,7 @@ import yaml from core.errors import FatalError +from core.retry import call_with_retry, create_with_retry # Directory in the devicetype-library holding the catalog, one directory per manufacturer. CATALOG_DIRNAME = "module-bay-types" @@ -197,28 +198,29 @@ def _unreadable(exc): return entries @staticmethod - def _request(action, description): - """Run one NetBox request, reporting a rejection as a catalog error. + def _request(action, description, *args, **kwargs): + """Call ``action(*args, **kwargs)``, a core.retry helper, and report a rejection as a catalog error. Callers resolve one definition at a time and recover from ModuleBayTypeError. A raw RequestError or a dropped connection escapes that recovery and ends the run, which can leave a parent half created and every later definition unprocessed. """ try: - return action() + return action(*args, **kwargs) except pynetbox.RequestError as exc: raise ModuleBayTypeError(f"NetBox rejected {description}: {exc}") from exc - except requests.exceptions.RequestException as exc: - raise ModuleBayTypeError(f"NetBox could not be reached for {description}: {exc}") from exc + except (requests.exceptions.RequestException, pynetbox.core.query.ContentError) as exc: + raise ModuleBayTypeError(f"NetBox gave no usable reply for {description}: {exc}") from exc def _netbox_id(self, entry): """Return the id of the NetBox object for *entry*, creating it if absent.""" manufacturer_id = self._manufacturer_id(entry["manufacturer"]) existing = self._request( + call_with_retry, + f"the lookup of module bay type {entry['name']!r}", lambda: list( self._netbox.dcim.module_bay_types.filter(manufacturer_id=manufacturer_id, name=entry["name"]) ), - f"the lookup of module bay type {entry['name']!r}", ) for record in existing: if record.slug != entry["slug"]: @@ -234,8 +236,12 @@ def _netbox_id(self, entry): if entry.get("description"): payload["description"] = entry["description"] created = self._request( - lambda: self._netbox.dcim.module_bay_types.create(payload), + create_with_retry, f"creating module bay type {entry['name']!r}", + self._netbox.dcim.module_bay_types, + payload, + "name", + manufacturer_id=manufacturer_id, ) self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") return created.id @@ -249,15 +255,19 @@ def _manufacturer_id(self, name): if name in self._manufacturer_ids: return self._manufacturer_ids[name] found = self._request( - lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), + call_with_retry, f"the lookup of manufacturer {name!r}", + lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), ) if found: self._manufacturer_ids[name] = found[0].id else: created = self._request( - lambda: self._netbox.dcim.manufacturers.create({"name": name, "slug": manufacturer_slug(name)}), + create_with_retry, f"creating manufacturer {name!r}", + self._netbox.dcim.manufacturers, + {"name": name, "slug": manufacturer_slug(name)}, + "slug", ) self._handle.verbose_log(f"Manufacturer Created: {name} - {created.id}") self._manufacturer_ids[name] = created.id diff --git a/core/netbox_api.py b/core/netbox_api.py index 53e1863a..651ce08f 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -5,9 +5,8 @@ import json import os import tempfile -import time from collections import Counter -from contextlib import contextmanager, suppress +from contextlib import ExitStack, contextmanager, suppress from dataclasses import replace from functools import lru_cache from pathlib import Path @@ -15,6 +14,7 @@ import pynetbox import requests +from pynetbox.core.query import Request from core.change_detector import ChangeDetector, ChangeType from core.compat import parse_netbox_version, supports_module_bay_types @@ -33,6 +33,7 @@ from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.normalization import values_equal from core.outcomes import EntityKind, Outcome, OutcomeRegistry +from core.retry import MAX_RETRIES, RETRYABLE_EXCEPTIONS, call_with_retry, create_with_retry, delete_with_retry from core.schema_reader import load_properties_for_type from core.update_failure_resolver import ( FailureKind, @@ -102,23 +103,22 @@ def _build_auth_header(token): _LOG_ESCAPES.update({ord("\n"): "\\n", ord("\r"): "\\r", ord("\t"): "\\t"}) -def _format_request_error(exc) -> str: - """Return *exc* as text, with NetBox's response body appended when there is one. +def _format_request_error(exc: pynetbox.RequestError) -> str: + """Return the status of a request NetBox refused, with its response body appended when there is one. - ``requests.HTTPError`` stringifies to only the status line, discarding the body where - NetBox reports the actual cause (e.g. ``{"front_image": ["Upload a valid image..."]}`` - for an image above its pixel cap). Without the body an operator sees nothing but - ``400 Client Error: Bad Request``. + The body holds the actual cause (e.g. ``{"front_image": ["Upload a valid image..."]}`` + for an image above its pixel cap), and pynetbox drops a body that is not JSON from its message. The body is server-controlled, so control characters are escaped before truncation. """ - body = (getattr(getattr(exc, "response", None), "text", "") or "").strip() + status = f"HTTP {exc.req.status_code} {exc.req.reason}" + body = (exc.error or "").strip() if not body: - return str(exc) + return status body = body.translate(_LOG_ESCAPES) if len(body) > _MAX_ERROR_BODY_CHARS: body = f"{body[:_MAX_ERROR_BODY_CHARS]}… (truncated)" - return f"{exc} — NetBox returned: {body}" + return f"{status} — NetBox returned: {body}" def _fmt_connection_error(url: str, exc: Exception) -> str: @@ -142,16 +142,22 @@ def _fmt_connection_error(url: str, exc: Exception) -> str: ) -# Transient connection errors that warrant a retry -_RETRYABLE_EXCEPTIONS = (requests.exceptions.ConnectionError, requests.exceptions.Timeout) - -_MAX_RETRIES = 3 -_RETRY_BACKOFF = (2, 5, 10) # seconds to wait before each retry attempt - # Sentinel used when a YAML record has no "src" key (e.g. synthesised entries). # _image_dir_for_yaml treats this value as "no path known" and returns None. _UNKNOWN_SRC = "Unknown" +# (connect, read) seconds for each REST call; it stops a stalled server, not a slow bulk write. +_REST_TIMEOUT = (30, 300) + + +class _TimeoutSession(requests.Session): + """A requests session that gives each request ``_REST_TIMEOUT`` unless the caller sets one.""" + + def request(self, *args, **kwargs): + """Send the request with the default timeout when the caller gives none.""" + kwargs.setdefault("timeout", _REST_TIMEOUT) + return super().request(*args, **kwargs) + def _check_image_url( base_url: str, @@ -320,48 +326,6 @@ def _store_image_hashes(cache: dict, images: dict, log_fn=None) -> None: log_fn(f"[yellow]Cannot hash image {path}, so it will be re-uploaded next run: {exc}[/yellow]") -def _delete_image_attachment(base_url: str, token: str, att_id: int, ignore_ssl: bool, handle) -> bool: - """Delete a NetBox image attachment by ID via DELETE /api/extras/image-attachments/{id}/. - - Args: - base_url: NetBox base URL. - token: API token used for the Authorization header. - att_id: Numeric ID of the image attachment to delete. - ignore_ssl: When True, skip SSL certificate verification. - handle: Log handler with a ``log`` method for error reporting. - - Returns: - bool: True on success, False on any HTTP or network error. - """ - url = f"{base_url}/api/extras/image-attachments/{att_id}/" - headers = {"Authorization": _build_auth_header(token)} - try: - response = requests.delete(url, headers=headers, verify=(not ignore_ssl), timeout=30) - response.raise_for_status() - return True - except requests.RequestException as e: - handle.log(f"Error deleting image attachment {att_id}: {_format_request_error(e)}") - return False - - -def _retry_on_connection_error(func, *args, **kwargs): - """Call *func* with retries on transient connection errors. - - Retries up to ``_MAX_RETRIES`` times with exponential-ish backoff - for ``ConnectionError`` and ``Timeout`` from requests/urllib3. - Non-retryable exceptions propagate immediately. - """ - for attempt in range(_MAX_RETRIES + 1): - try: - return func(*args, **kwargs) - except _RETRYABLE_EXCEPTIONS: - if attempt >= _MAX_RETRIES: - raise - wait = _RETRY_BACKOFF[attempt] if attempt < len(_RETRY_BACKOFF) else _RETRY_BACKOFF[-1] - time.sleep(wait) - return None - - # Module type scalar properties that can be compared and updated. # Loaded from the cloned devicetype-library schema at runtime; the list below # serves as a fallback when the schema is not yet available (e.g. before the @@ -591,7 +555,6 @@ def __init__(self, config, handle): self.netbox, self.handle, self.counter, - self.ignore_ssl, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, module_bay_types_supported=self.module_bay_types, @@ -646,6 +609,7 @@ def connect_api(self): """Connect to the NetBox API using the stored URL and token credentials.""" try: self.netbox = pynetbox.api(self.url, token=self.token, threading=True) + self.netbox.http_session = _TimeoutSession() if self.ignore_ssl: self.handle.verbose_log("IGNORE_SSL_ERRORS is True, catching exception and disabling SSL verification.") import urllib3 @@ -743,15 +707,15 @@ def create_manufacturers(self, vendors): if to_create: self.handle.log(f"Creating {len(to_create)} new manufacturers...") try: - created_manufacturers = _retry_on_connection_error(self.netbox.dcim.manufacturers.create, to_create) + created_manufacturers = create_with_retry(self.netbox.dcim.manufacturers, to_create, "slug") for manufacturer in created_manufacturers: self.handle.verbose_log(f"Manufacturer Created: {manufacturer.name} - {manufacturer.id}") self.counter.update({"manufacturer": 1}) except pynetbox.RequestError as request_error: # Log error with detailed API error message self.handle.log(f"Error creating manufacturers: {request_error.error}") - except _RETRYABLE_EXCEPTIONS as e: - self.handle.log(f"Connection error creating manufacturers after {_MAX_RETRIES} retries: {e}") + except RETRYABLE_EXCEPTIONS as e: + self.handle.log(f"Connection error creating manufacturers after {MAX_RETRIES} retries: {e}") else: self.handle.verbose_log("No new manufacturers to create.") @@ -859,7 +823,7 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er # Retry the original PATCH exactly once. try: - _retry_on_connection_error( + call_with_retry( self.netbox.dcim.device_types.update, [{"id": dt.id, **updates}], ) @@ -868,7 +832,7 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er except pynetbox.RequestError as e: self.handle.log(f"Retry after auto-resolve still failed for {dt.model}: {e.error}") return False, resolution - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log(f"Connection error during retry after auto-resolve for {dt.model}: {e}") return False, resolution @@ -1075,8 +1039,7 @@ def _handle_existing_device_type( """ if saved_images: self._filter_images_for_upload(dt, saved_images) - if saved_images: - self.device_types.upload_images(self.url, self.token, saved_images, dt.id) + if saved_images and self.device_types.upload_images(saved_images, dt.id): _store_image_hashes(self._image_hash_cache, saved_images, log_fn=self.handle.log) self._persist_hash_cache() @@ -1104,7 +1067,7 @@ def _handle_existing_device_type( if updates: property_attempted = True try: - _retry_on_connection_error(self.netbox.dcim.device_types.update, [{"id": dt.id, **updates}]) + call_with_retry(self.netbox.dcim.device_types.update, [{"id": dt.id, **updates}]) dt.update(updates) # keep local cache in sync self.counter.update({"properties_updated": 1}) property_succeeded = True @@ -1120,9 +1083,9 @@ def _handle_existing_device_type( self.handle.verbose_log( f"Updated device type {dt.model} properties after auto-resolve: {list(updates.keys())}" ) - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( - f"Connection error updating device type {dt.model} after {_MAX_RETRIES} retries: {e}" + f"Connection error updating device type {dt.model} after {MAX_RETRIES} retries: {e}" ) # Apply component changes @@ -1183,7 +1146,12 @@ def _create_new_device_type(self, device_type, src_file): True when the caller should skip to the next iteration. """ try: - dt = _retry_on_connection_error(self.netbox.dcim.device_types.create, device_type) + dt = create_with_retry( + self.netbox.dcim.device_types, + device_type, + "model", + manufacturer=device_type["manufacturer"]["slug"], + ) self.counter.update({"added": 1}) self.handle.verbose_log(f"Device Type Created: {dt.manufacturer.name} - " + f"{dt.model} - {dt.id}") return dt, False @@ -1200,16 +1168,16 @@ def _create_new_device_type(self, device_type, src_file): src_file, ) return None, True - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( f"Connection error creating device type" f" {device_type.get('manufacturer', {}).get('slug', '')} {device_type.get('model', '')}" - f" after {_MAX_RETRIES} retries: {e} (Context: {src_file})" + f" after {MAX_RETRIES} retries: {e} (Context: {src_file})" ) self._record_failure( EntityKind.DEVICE_TYPE, self._yaml_identity(device_type), - f"Connection error after {_MAX_RETRIES} retries: {e}", + f"Connection error after {MAX_RETRIES} retries: {e}", src_file, ) return None, True @@ -1263,8 +1231,7 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima Outcome.PARTIAL, reason=_summarise_component_errors(component_errors), ) - if saved_images: - self.device_types.upload_images(self.url, self.token, saved_images, dt_id) + if saved_images and self.device_types.upload_images(saved_images, dt_id): _store_image_hashes(self._image_hash_cache, saved_images, log_fn=self.handle.log) self._persist_hash_cache() @@ -1423,7 +1390,7 @@ def create_rack_types(self, rack_types, progress=None, only_new=False, all_rack_ } if updates: try: - _retry_on_connection_error(self.netbox.dcim.rack_types.update, [{"id": existing.id, **updates}]) + call_with_retry(self.netbox.dcim.rack_types.update, [{"id": existing.id, **updates}]) self.counter.update({"rack_type_updated": 1}) self.handle.verbose_log( f"Rack Type Updated: {manufacturer_slug} - {model} - {existing.id} " @@ -1434,22 +1401,24 @@ def create_rack_types(self, rack_types, progress=None, only_new=False, all_rack_ self._record_failure( EntityKind.RACK_TYPE, f"{manufacturer_slug}/{model}", str(e.error), src_file ) - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( - f"Connection error updating Rack Type {model} after {_MAX_RETRIES} retries:" + f"Connection error updating Rack Type {model} after {MAX_RETRIES} retries:" f" {e} (Context: {src_file})" ) self._record_failure( EntityKind.RACK_TYPE, f"{manufacturer_slug}/{model}", - f"Connection error after {_MAX_RETRIES} retries: {e}", + f"Connection error after {MAX_RETRIES} retries: {e}", src_file, ) else: self.handle.verbose_log(f"Rack Type Unchanged: {manufacturer_slug} - {model} - {existing.id}") else: try: - rt = _retry_on_connection_error(self.netbox.dcim.rack_types.create, rack_type) + rt = create_with_retry( + self.netbox.dcim.rack_types, rack_type, "model", manufacturer=manufacturer_slug + ) self.counter.update({"rack_type_added": 1}) all_rack_types.setdefault(manufacturer_slug, {})[model] = rt self.handle.verbose_log(f"Rack Type Created: {manufacturer_slug} - {model} - {rt.id}") @@ -1458,15 +1427,15 @@ def create_rack_types(self, rack_types, progress=None, only_new=False, all_rack_ self._record_failure( EntityKind.RACK_TYPE, f"{manufacturer_slug}/{model}", str(excep.error), src_file ) - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( - f"Connection error creating Rack Type {model} after {_MAX_RETRIES} retries:" + f"Connection error creating Rack Type {model} after {MAX_RETRIES} retries:" f" {e} (Context: {src_file})" ) self._record_failure( EntityKind.RACK_TYPE, f"{manufacturer_slug}/{model}", - f"Connection error after {_MAX_RETRIES} retries: {e}", + f"Connection error after {MAX_RETRIES} retries: {e}", src_file, ) @@ -1797,9 +1766,7 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): if stale_attributes: self._patch_profile_change(module_type_res, updates) else: - _retry_on_connection_error( - self.netbox.dcim.module_types.update, [{"id": module_type_res.id, **updates}] - ) + call_with_retry(self.netbox.dcim.module_types.update, [{"id": module_type_res.id, **updates}]) self.handle.verbose_log( f"Module Type Updated: {module_type_res.manufacturer.name} - " f"{module_type_res.model} - {module_type_res.id} " @@ -1808,9 +1775,9 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): except pynetbox.RequestError as excep: self.handle.log(f"Error updating Module Type: {excep.error} (Context: {src_file})") return False, False - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( - f"Connection error updating Module Type after {_MAX_RETRIES} retries: {e} (Context: {src_file})" + f"Connection error updating Module Type after {MAX_RETRIES} retries: {e} (Context: {src_file})" ) return False, False return True, True @@ -1826,14 +1793,14 @@ def _patch_profile_change(self, module_type_res, updates): type_id = module_type_res.id cleared = {"id": type_id, "profile": None, _REST_ATTRIBUTES_FIELD: None} try: - _retry_on_connection_error(update, [cleared]) - _retry_on_connection_error(update, [{"id": type_id, **updates}]) + call_with_retry(update, [cleared]) + call_with_retry(update, [{"id": type_id, **updates}]) except Exception: # any failure after the clear leaves an unknown state; restore, then re-raise old_profile = {"name": module_type_res.profile.name} if module_type_res.profile else None restore = {"id": type_id, "profile": old_profile, _REST_ATTRIBUTES_FIELD: module_type_res.attribute_data} for step, payload in (("clear", cleared), ("restore", restore)): try: - _retry_on_connection_error(update, [payload]) + call_with_retry(update, [payload]) except Exception as step_error: # noqa: BLE001 # logged; the original error is re-raised self.handle.log( f"Could not {step} the profile and attributes of Module Type {type_id}: {step_error}" @@ -2026,7 +1993,9 @@ def _process_single_module_type( src_file, ) return False - module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, payload) + module_type_res = create_with_retry( + self.netbox.dcim.module_types, payload, "model", manufacturer=curr_mt["manufacturer"]["slug"] + ) self.counter["module_added"] += 1 is_new = True manufacturer_slug = curr_mt["manufacturer"]["slug"] @@ -2044,14 +2013,14 @@ def _process_single_module_type( src_file, ) return False - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self.handle.log( - f"Connection error creating Module Type after {_MAX_RETRIES} retries: {e} (Context: {src_file})" + f"Connection error creating Module Type after {MAX_RETRIES} retries: {e} (Context: {src_file})" ) self._record_failure( EntityKind.MODULE_TYPE, self._yaml_identity(curr_mt), - f"Connection error after {_MAX_RETRIES} retries: {e}", + f"Connection error after {MAX_RETRIES} retries: {e}", src_file, ) return False @@ -2250,7 +2219,7 @@ def _try_delete_stale_attachment(self, detail, img_path, module_type_res, existi "missing or invalid att_id, skipping upload to avoid duplicates." ) return False - if not _delete_image_attachment(self.url, self.token, att_id, self.ignore_ssl, self.handle): + if not self.device_types.delete_image_attachment(att_id): self.handle.verbose_log( f"Failed to delete stale attachment for " f"'{os.path.basename(img_path)}' on {module_type_res.model}, " @@ -2351,9 +2320,7 @@ def _upload_module_type_images(self, module_type_res, src_file, module_type_exis f"Image '{os.path.basename(img_path)}' already exists for {module_type_res.model}, skipping." ) continue - if self.device_types.upload_image_attachment( - self.url, self.token, img_path, "dcim.moduletype", module_type_res.id - ): + if self.device_types.upload_image_attachment(img_path, "dcim.moduletype", module_type_res.id): existing.add(img_name) _store_image_hashes(self._image_hash_cache, {"image": img_path}, log_fn=self.handle.log) self._persist_hash_cache() @@ -2448,7 +2415,6 @@ def __init__( netbox, handle, counter, - ignore_ssl, *, graphql, repo_path, @@ -2465,7 +2431,6 @@ def __init__( netbox: Connected pynetbox API instance. handle (LogHandler): Sink for creation and error messages. counter (Counter): Shared operation counter updated during creation. - ignore_ssl (bool): Whether SSL certificate verification is disabled. graphql (NetBoxGraphQLClient): GraphQL client for read queries. module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. @@ -2475,7 +2440,6 @@ def __init__( self.netbox = netbox self.handle = handle self.counter = counter - self.ignore_ssl = ignore_ssl self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports @@ -2592,7 +2556,7 @@ def _create_generic( if to_create: try: - created = _retry_on_connection_error(endpoint.create, to_create) + created = create_with_retry(endpoint, to_create, "name", **{f"{parent_key}_id": parent_id}) self.handle.log_ports_created(created, parent_type, component_name) self.counter.update({"components_added": len(created)}) @@ -2615,11 +2579,11 @@ def _create_generic( self._log_component_error( f"Error '{excep.error}' creating {component_name}. Items: {failed_items}{context_str}" ) - except _RETRYABLE_EXCEPTIONS as excep: + except RETRYABLE_EXCEPTIONS as excep: context_str = f" (Context: {context})" if context else "" failed_items = [x["name"] for x in to_create] self._log_component_error( - f"Connection error creating {component_name} after {_MAX_RETRIES} retries: {excep}." + f"Connection error creating {component_name} after {MAX_RETRIES} retries: {excep}." f" Items: {failed_items}{context_str}" ) @@ -2788,14 +2752,14 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, success_count = 0 for update_data in updates: try: - _retry_on_connection_error(endpoint.update, [update_data]) + call_with_retry(endpoint.update, [update_data]) success_count += 1 self.handle.verbose_log(f"Updated {comp_type} (ID: {update_data['id']})") except pynetbox.RequestError as e: self._log_component_error(f"Error updating {comp_type} (ID: {update_data['id']}): {e.error}") - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self._log_component_error( - f"Connection error updating {comp_type} (ID: {update_data['id']}) after {_MAX_RETRIES} retries: {e}" + f"Connection error updating {comp_type} (ID: {update_data['id']}) after {MAX_RETRIES} retries: {e}" ) if success_count: @@ -2901,13 +2865,13 @@ def remove_components(self, device_type_id, component_changes, parent_type="devi success_count = 0 for comp_id in ids_to_delete: try: - _retry_on_connection_error(endpoint.delete, [comp_id]) + delete_with_retry(endpoint, comp_id) success_count += 1 except pynetbox.RequestError as e: self._log_component_error(f"Error removing {comp_type} (ID: {comp_id}): {e.error}") - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self._log_component_error( - f"Connection error removing {comp_type} (ID: {comp_id}) after {_MAX_RETRIES} retries: {e}" + f"Connection error removing {comp_type} (ID: {comp_id}) after {MAX_RETRIES} retries: {e}" ) if success_count: @@ -3114,13 +3078,13 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): if not to_update: return try: - _retry_on_connection_error(self.netbox.dcim.interface_templates.update, to_update) + call_with_retry(self.netbox.dcim.interface_templates.update, to_update) self.handle.verbose_log(f"Bridged {len(to_update)} interfaces.") except pynetbox.RequestError as e: self._log_component_error(f"Error bridging interfaces: {e} (Context: {context})") - except _RETRYABLE_EXCEPTIONS as e: + except RETRYABLE_EXCEPTIONS as e: self._log_component_error( - f"Connection error bridging interfaces after {_MAX_RETRIES} retries: {e} (Context: {context})" + f"Connection error bridging interfaces after {MAX_RETRIES} retries: {e} (Context: {context})" ) @property @@ -3226,98 +3190,114 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co if bridged: self._link_bridges(bridged, parent_id, parent_type, context) - def upload_images(self, baseurl, token, images, device_type): - """Upload front and/or rear image files to the specified NetBox device type. + def upload_images(self, images, device_type_id): + """Upload front and/or rear image files to one NetBox device type. - Sends a PATCH request to the device-type endpoint attaching the provided image files, - increments self.counter["images"] by the number of files sent, and ensures all opened - file handles are closed. Respects self.ignore_ssl to determine SSL verification behavior. + Sends one multipart PATCH to the device type, retried after a transient error, and + adds the number of files sent to ``self.counter["images"]``. Args: - baseurl (str): Base URL of the NetBox instance (e.g. "https://netbox.example.com"). - token (str): API token used for the Authorization header. images (dict): Mapping of form field name to local file path (e.g. {"front_image": "/path/front.jpg", "rear_image": "/path/rear.jpg"}). - device_type (int | str): Identifier of the device type to update in NetBox (used in the endpoint URL). - """ - url = f"{baseurl}/api/dcim/device-types/{device_type}/" - headers = {"Authorization": _build_auth_header(token)} + device_type_id (int): ID of the device type to update. - # Open files with proper cleanup to avoid resource leaks - file_handles = {} - try: - for field, path in images.items(): - file_handles[field] = (os.path.basename(path), open(path, "rb")) # noqa: SIM115 - response = requests.patch( - url, - headers=headers, - files=file_handles, - verify=(not self.ignore_ssl), - timeout=60, - ) - response.raise_for_status() - self.handle.verbose_log(f"Images {images} updated at {url}: {response.status_code}") - self.counter["images"] += len(images) - if self._image_progress: - self._image_progress(len(images)) - except requests.RequestException as e: - self.handle.log(f"Error uploading images for device type {device_type}: {_format_request_error(e)}") - except OSError as e: - self.handle.log(f"Error reading image file for device type {device_type}: {e}") - finally: - for _, fh in file_handles.values(): - with suppress(Exception): - fh.close() + Returns: + bool: True when NetBox stored the images, False when the upload failed. + """ + # One object's URL: Endpoint.update sends a list, which pynetbox never sends as multipart. + request = Request( + key=device_type_id, + base=self.netbox.dcim.device_types.url, + token=self.netbox.token, + http_session=self.netbox.http_session, + ) + with ExitStack() as files: + try: + payload = { + field: (os.path.basename(path), files.enter_context(open(path, "rb"))) + for field, path in images.items() + } + except OSError as e: + self.handle.log(f"Error reading image file for device type {device_type_id}: {e}") + return False + try: + call_with_retry(request.patch, payload) + except pynetbox.RequestError as e: + self.handle.log(f"Error uploading images for device type {device_type_id}: {_format_request_error(e)}") + return False + except RETRYABLE_EXCEPTIONS as e: + self.handle.log( + f"Error uploading images for device type {device_type_id} after {MAX_RETRIES} retries: {e}" + ) + return False + self.handle.verbose_log(f"Images {images} updated on device type {device_type_id}") + self.counter["images"] += len(images) + if self._image_progress: + self._image_progress(len(images)) + return True - def upload_image_attachment(self, baseurl, token, image_path, object_type, object_id): + def upload_image_attachment(self, image_path, object_type, object_id): """Upload an image as an Image Attachment to a NetBox object. - Uses POST /api/extras/image-attachments/ to attach an image to any - NetBox object type (e.g. module types which lack built-in image fields). + Attaches an image to any NetBox object type (e.g. module types, which lack built-in + image fields). After a transient error the attachment is looked up by name on that + object before the upload is sent again, so a lost reply does not create a second one. Args: - baseurl (str): Base URL of the NetBox instance. - token (str): API token for authorization. image_path (str): Local file path of the image to upload. object_type (str): NetBox content type string (e.g. "dcim.moduletype"). - object_id (int | str): ID of the object to attach the image to. + object_id (int): ID of the object to attach the image to. Returns: bool: True if the upload succeeded, False on any error. """ - url = f"{baseurl}/api/extras/image-attachments/" - headers = {"Authorization": _build_auth_header(token)} - data = { - "object_type": object_type, - "object_id": str(object_id), - "name": os.path.splitext(os.path.basename(image_path))[0], - } - + filename = os.path.basename(image_path) + target = f"{object_type} {object_id}" try: - with open(image_path, "rb") as f: - files = {"image": (os.path.basename(image_path), f)} - response = requests.post( - url, - headers=headers, - data=data, - files=files, - verify=(not self.ignore_ssl), - timeout=60, - ) - response.raise_for_status() - self.handle.verbose_log( - f"Image attachment '{os.path.basename(image_path)}' uploaded" - f" for {object_type} {object_id}: {response.status_code}" - ) - self.counter["images"] += 1 - if self._image_progress: - self._image_progress(1) - return True - except requests.RequestException as e: - self.handle.log( - f"Error uploading image attachment for {object_type} {object_id}: {_format_request_error(e)}" - ) - return False + f = open(image_path, "rb") # noqa: SIM115 except OSError as e: self.handle.log(f"Error reading image file {image_path}: {e}") return False + payload = { + "object_type": object_type, + "object_id": object_id, + "name": os.path.splitext(filename)[0], + "image": (filename, f), + } + with f: + try: + create_with_retry( + self.netbox.extras.image_attachments, payload, "name", object_type=object_type, object_id=object_id + ) + except pynetbox.RequestError as e: + self.handle.log(f"Error uploading image attachment for {target}: {_format_request_error(e)}") + return False + except RETRYABLE_EXCEPTIONS as e: + self.handle.log(f"Error uploading image attachment for {target} after {MAX_RETRIES} retries: {e}") + return False + except ValueError as e: + # Endpoint.get raises this when the lookup before a retry matches more than one attachment. + self.handle.log(f"Cannot tell if the image attachment for {target} was created: {e}") + return False + self.handle.verbose_log(f"Image attachment '{filename}' uploaded for {target}") + self.counter["images"] += 1 + if self._image_progress: + self._image_progress(1) + return True + + def delete_image_attachment(self, att_id): + """Delete one NetBox image attachment by ID, retried after a transient error. + + Args: + att_id (int): ID of the image attachment to delete. + + Returns: + bool: True when the attachment is gone, False when NetBox refused or retries ran out. + """ + try: + return delete_with_retry(self.netbox.extras.image_attachments, att_id) + except pynetbox.RequestError as e: + self.handle.log(f"Error deleting image attachment {att_id}: {_format_request_error(e)}") + except RETRYABLE_EXCEPTIONS as e: + self.handle.log(f"Error deleting image attachment {att_id} after {MAX_RETRIES} retries: {e}") + return False diff --git a/core/retry.py b/core/retry.py new file mode 100644 index 00000000..d2592b9a --- /dev/null +++ b/core/retry.py @@ -0,0 +1,99 @@ +"""Retry NetBox writes after a transient error, without repeating a write that committed.""" + +import io +import time + +import pynetbox +import requests + +# Transient errors that warrant a retry: no reply, a cut-off reply, or a reply that is not JSON +RETRYABLE_EXCEPTIONS = ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + pynetbox.core.query.ContentError, +) + +MAX_RETRIES = 3 +RETRY_BACKOFF = (2, 5, 10) # seconds to wait before each retry attempt + +# Keys per lookup request, so the query string stays under common proxy URL limits. +LOOKUP_CHUNK = 50 + + +def call_with_retry(func, *args, find_existing=None, **kwargs): + """Call *func* with retries on transient connection errors. + + Retries up to ``MAX_RETRIES`` times with backoff for each error in ``RETRYABLE_EXCEPTIONS``. + Non-retryable exceptions propagate immediately. + + A transient error can arrive after NetBox committed the write. Before each retry, + *find_existing* (when given) checks for that: a result other than None is returned + in place of a second write. A transient error in that check counts as a failed attempt. + + An upload reads its files to the end, so each attempt first rewinds every file in a dict argument. + """ + for attempt in range(MAX_RETRIES + 1): + try: + if attempt and find_existing is not None: + existing = find_existing() + if existing is not None: + return existing + _rewind_files(args) + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS: + if attempt >= MAX_RETRIES: + raise + time.sleep(RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF) - 1)]) + raise AssertionError("unreachable: the last attempt returns or raises") + + +def create_with_retry(endpoint, payload, key, **scope): + """Create *payload* on *endpoint*, retrying without creating a duplicate. + + *key* names the payload field that, with the *scope* filters, identifies one object + (for example ``key="model", manufacturer=slug``). Before a retry, the object is looked + up by that natural key. When it exists, it is returned as the create would return it: + one Record for a dict payload, a list of Records in payload order for a list payload. + + A bulk create in NetBox is atomic, so a partial match means another writer. It is not + taken as the result; the create is sent again and NetBox rejects or accepts it. + """ + if isinstance(payload, list): + return call_with_retry(endpoint.create, payload, find_existing=lambda: _find_all(endpoint, payload, key, scope)) + return call_with_retry(endpoint.create, payload, find_existing=lambda: endpoint.get(**scope, **{key: payload[key]})) + + +def delete_with_retry(endpoint, object_id): + """Delete one object by id, counting it as deleted when a lost reply hid the commit. + + Returns True. Before a retry, the object is looked up by id; when it is gone, the + earlier attempt committed and the delete is not sent again. + """ + return call_with_retry( + endpoint.delete, [object_id], find_existing=lambda: True if endpoint.get(id=object_id) is None else None + ) + + +def _rewind_files(args): + """Seek each file in a dict argument back to its start, as a bare file or a ``(filename, file)`` pair.""" + for arg in args: + if not isinstance(arg, dict): + continue + for value in arg.values(): + file = value[1] if isinstance(value, tuple) and len(value) > 1 else value + if isinstance(file, io.IOBase): + file.seek(0) + + +def _find_all(endpoint, items, key, scope): + """Return the record for each item in payload order, or None unless every item exists.""" + keys = [item[key] for item in items] + found = { + getattr(record, key): record + for start in range(0, len(keys), LOOKUP_CHUNK) + for record in endpoint.filter(**scope, **{key: keys[start : start + LOOKUP_CHUNK]}) + } + if any(item[key] not in found for item in items): + return None + return [found[item[key]] for item in items] diff --git a/tests/helpers.py b/tests/helpers.py index e69c0733..02e38b21 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,7 +4,10 @@ import json import re import threading -from http.server import BaseHTTPRequestHandler, HTTPServer +from email.parser import BytesParser +from email.policy import HTTP +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import ClassVar from unittest.mock import MagicMock from urllib.parse import parse_qs, urlparse @@ -72,9 +75,12 @@ class FakeNetBox: """A local HTTP server answering the slice of the NetBox REST API the importer uses. Collections are addressed by their URL segment with dashes turned into underscores, - so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates and PATCH - bulk-updates behave the way pynetbox expects, so a real client drives it and the - serialising, filtering and paginating under test are the production ones. + so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates, PATCH + bulk-updates and DELETE bulk-deletes behave the way pynetbox expects, so a real client + drives it and the serialising, filtering and paginating under test are the production ones. + A create that repeats a natural key gets NetBox's 400, as does a delete of a missing id. + A PATCH to one object's URL updates that object. A multipart write stores each file as + a media URL and keeps the received bytes in ``uploads``. ``/graphql/`` answers every query with an empty list, which is enough to bring a real :class:`~core.netbox_api.NetBox` up; the REST half is where the assertions live. @@ -91,23 +97,42 @@ class FakeNetBox: # A nested reference NetBox looks up by its attributes, keyed by field name. _NESTED_REFERENCES = (("manufacturer", "manufacturers"), ("profile", "module_type_profiles")) + # Fields that together identify one record, as NetBox's unique constraints do. + _UNIQUE_TOGETHER: ClassVar[dict[str, tuple[tuple[str, ...], ...]]] = { + "manufacturers": (("slug",),), + "device_types": (("manufacturer", "model"),), + "rack_types": (("manufacturer", "model"),), + "module_types": (("manufacturer", "model"),), + "module_bay_types": (("manufacturer", "name"),), + } + _COMPONENT_UNIQUE = (("device_type", "name"), ("module_type", "name")) + def __init__(self, errors=None, netbox_version="4.7.0", **collections): """Start the server with *collections* seeded, keyed by collection name. *errors* maps a collection name to an HTTP status the server answers it with, so a - test can drive a rejection the importer has to survive. *netbox_version* is what + test can drive a rejection the importer has to survive. A ``(status, text)`` pair + answers with that raw body instead of a JSON detail. *netbox_version* is what ``/api/status/`` reports, which is how the export side decides what it may select. """ self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} self.errors = dict(errors or {}) self.netbox_version = netbox_version + # A GET whose request target is longer than this gets 414, as from a proxy URL limit. + self.max_url_length = None self.requests = [] - # 1-based PATCH numbers that commit, then reply with nothing, a cut-off body, or invalid JSON. - self.lost_patch_responses = set() - self.truncated_patch_responses = set() - self.garbled_patch_responses = set() - self._server = HTTPServer(("127.0.0.1", 0), self._handler()) - threading.Thread(target=self._server.serve_forever, daemon=True).start() + # (verb, 1-based number of that verb) -> "lost", "truncated" or "garbled": the request runs, the reply breaks. + # "dropped": the connection closes before the request runs, so nothing commits. + # "stalled": the request runs, and the reply never comes until close(). + self.broken_replies = {} + # (field, filename, bytes) of each file a multipart write carried, in arrival order. + self.uploads = [] + self._released = threading.Event() # ends every "stalled" reply + # Threaded, so a request can still be answered while a "stalled" reply holds its thread. + self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler()) + self._server.daemon_threads = True + # A short poll lets close() return at once instead of after the default half second. + threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True).start() @property def url(self): @@ -120,6 +145,7 @@ def api(self, token="test-token"): def close(self): """Stop serving and release the listening socket.""" + self._released.set() self._server.shutdown() self._server.server_close() @@ -148,10 +174,12 @@ def matches(self, name, query): for key, values in query.items(): if key in self._IGNORED_FILTERS: continue - field = key[:-3] if key.endswith("_id") else key + # A plain field such as object_id matches itself; device_type_id filters the device_type reference. + field = key[:-3] if key.endswith("_id") and key not in record else key actual = record.get(field) if isinstance(actual, dict): - actual = actual.get("id") + # NetBox filters a nested reference by id with "_id", by slug with "". + actual = actual.get("id" if key.endswith("_id") else "slug") if str(actual) not in values: ok = False break @@ -165,9 +193,46 @@ def _create(self, name, payload): A list payload is a bulk create, which is how the importer adds components, and it answers with a list. """ - if isinstance(payload, list): - return [self._create_one(name, item) for item in payload] - return self._create_one(name, payload) + records = [self._build_record(item) for item in (payload if isinstance(payload, list) else [payload])] + errors = [self._duplicate_error(name, record) for record in records] + if any(errors): + # A bulk create is atomic: one duplicate refuses the whole batch. + raise _Refused(errors if isinstance(payload, list) else errors[0]) + collection = self.collection(name) + for record in records: + record["id"] = 1000 + len(collection) + collection.append(record) + return records if isinstance(payload, list) else records[0] + + def _duplicate_error(self, name, item): + """Return NetBox's error for an item that repeats a stored natural key, else an empty dict.""" + constraints = self._UNIQUE_TOGETHER.get(name, self._COMPONENT_UNIQUE if name.endswith("_templates") else ()) + for fields in constraints: + wanted = [self._reference_id(item.get(field)) for field in fields] + if None in wanted: + continue + if any(wanted == [self._reference_id(r.get(f)) for f in fields] for r in self.collection(name)): + return {"non_field_errors": [f"The fields {', '.join(fields)} must make a unique set."]} + return {} + + @staticmethod + def _reference_id(value): + """Return the id of a nested reference, or the value itself.""" + return value.get("id") if isinstance(value, dict) else value + + def _delete(self, name, payload): + """Remove the records a bulk delete names, refusing the batch as NetBox does if one is missing.""" + wanted = [entry["id"] for entry in payload] + stored = self.collection(name) + missing = [i for i in wanted if not any(r["id"] == i for r in stored)] + if missing: + raise _Refused( + { + "detail": f"{len(missing)} of {len(wanted)} objects could not be found.", + "errors": [{"id": i, "errors": {"id": [f"Object with ID {i} does not exist"]}} for i in missing], + } + ) + stored[:] = [r for r in stored if r["id"] not in wanted] def _resolve_references(self, payload): """Replace each nested reference with the object it names, as NetBox does. @@ -181,18 +246,24 @@ def _resolve_references(self, payload): match = next((r for r in self.collection(collection) if wanted.items() <= r.items()), None) if match is None: raise _Refused({field: [f"Related object not found using the provided attributes: {wanted}"]}) - payload[field] = {"id": match["id"], "name": match.get("name")} + payload[field] = self._nested(match) return payload - def _create_one(self, name, payload): - """Store one new record and return it as NetBox would echo it back.""" - collection = self.collection(name) - record = {"id": 1000 + len(collection), **self._resolve_references(payload)} + @staticmethod + def _nested(record): + """Return *record* as NetBox nests it in another object: id, name, and slug when it has one.""" + nested = {"id": record.get("id"), "name": record.get("name")} + if "slug" in record: + nested["slug"] = record["slug"] + return nested + + def _build_record(self, payload): + """Return one new record as NetBox would echo it back, without storing it.""" + record = self._resolve_references(payload) # NetBox echoes a foreign key back as a nested object, not the id it was given. if isinstance(record.get("manufacturer"), int): owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) - record["manufacturer"] = {"id": record["manufacturer"], "name": owner.get("name")} - collection.append(record) + record["manufacturer"] = {**self._nested(owner), "id": record["manufacturer"]} return record @staticmethod @@ -223,81 +294,146 @@ def _patch(self, name, payload): updated.append(record) return updated - def _reply_to_patch(self, handler, answer): - """Send *answer*, or close the connection when the test chose to lose this PATCH's reply.""" - number = sum(method == "PATCH" for method, _, _ in self.requests) - if number in self.lost_patch_responses: + @staticmethod + def _target(path): + """Return the collection name in *path* and, for one object's URL, its id.""" + segments = urlparse(path).path.rstrip("/").split("/") + object_id = int(segments.pop()) if segments[-1].isdigit() else None + return segments[-1].replace("-", "_"), object_id + + def _collection_name(self, path): + """Return the collection name in *path*, which must be a collection URL.""" + name, object_id = self._target(path) + assert object_id is None, f"the fake answers only PATCH on one object's URL, not {path}" + return name + + def _decode(self, headers, raw): + """Return a request body as a payload: JSON, or a multipart form.""" + if headers.get_content_type() == "multipart/form-data": + return self._form(headers["Content-Type"], raw) + return json.loads(raw or b"{}") + + def _refused(self, handler, name): + """Answer with the status (and raw body, when given) the test set for *name*; return True if it did.""" + error = self.errors.get(name) + if error is None: + return False + if isinstance(error, tuple): + status, text = error + else: + status, text = error, json.dumps({"detail": f"{name} refused with {error}"}) + handler._send(status, text.encode()) + return True + + def _form(self, content_type, raw): + """Return a multipart form body as a payload; each file becomes the media URL NetBox echoes.""" + message = BytesParser(policy=HTTP).parsebytes(f"Content-Type: {content_type}\r\n\r\n".encode() + raw) + payload = {} + for part in message.iter_parts(): + field = part.get_param("name", header="content-disposition") + filename = part.get_filename() + if filename is None: + payload[field] = part.get_payload(decode=True).decode() + continue + self.uploads.append((field, filename, part.get_payload(decode=True))) + payload[field] = f"/media/{filename}" + return payload + + def _broken(self, verb): + """Return how the test chose to break the latest *verb* request, or None.""" + number = sum(method == verb for method, name, _ in self.requests if name != "graphql") + return self.broken_replies.get((verb, number)) + + def _reply_to(self, handler, verb, answer): + """Send *answer*, or break the reply the way the test chose for this request.""" + broken = self._broken(verb) + if broken == "stalled": + self._released.wait() + handler.close_connection = True + return + if broken == "lost": handler.close_connection = True return - if number in self.truncated_patch_responses | self.garbled_patch_responses: + if broken in ("truncated", "garbled"): handler.send_response(answer[0]) handler.send_header("Content-Type", "application/json") - handler.send_header("Content-Length", "1000" if number in self.truncated_patch_responses else "2") + handler.send_header("Content-Length", "1000" if broken == "truncated" else "2") handler.end_headers() handler.wfile.write(b"[{") handler.close_connection = True return handler._reply(*answer) + def _write(self, handler, verb, name, payload, status, apply): + """Record a write, then answer with *apply*'s result or the refusal it raises.""" + self.requests.append((verb, name, payload)) + if self._broken(verb) == "dropped": + handler.close_connection = True + return + if self._refused(handler, name): + return + self._reply_to(handler, verb, self._answer(status, lambda: apply(name, copy.deepcopy(payload)))) + + def _write_patch(self, handler, name, object_id, payload): + """Record a PATCH: a bulk update on a collection URL, or one object's update on its own URL.""" + if object_id is None: + self._write(handler, "PATCH", name, payload, 200, self._patch) + return + self._write(handler, "PATCH", name, {**payload, "id": object_id}, 200, lambda n, p: self._patch(n, p)[0]) + def _handler(self): """Build the request handler class bound to this server's state.""" state = self class Handler(BaseHTTPRequestHandler): def _collection_name(self): - return urlparse(self.path).path.rstrip("/").rsplit("/", 1)[-1].replace("-", "_") + return state._collection_name(self.path) def _body(self): - return json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + return state._decode(self.headers, self.rfile.read(int(self.headers.get("Content-Length", 0)))) def _reply(self, status, payload): - body = json.dumps(payload).encode() + self._send(status, b"" if payload is None else json.dumps(payload).encode()) + + def _send(self, status, body): self.send_response(status) self.send_header("Content-Type", "application/json") + self.send_header("API-Version", ".".join(state.netbox_version.split(".")[:2])) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) - def _refused(self, name): - """Answer with the configured status for *name*, if the test set one.""" - status = state.errors.get(name) - if status is None: - return False - self._reply(status, {"detail": f"{name} refused with {status}"}) - return True - def do_GET(self): parsed = urlparse(self.path) name = self._collection_name() state.requests.append(("GET", name, parsed.query)) + if state.max_url_length is not None and len(self.path) > state.max_url_length: + self._reply(414, {"detail": "Request-URI Too Long"}) + return if name == "status": self._reply(200, {"netbox-version": state.netbox_version}) return - if self._refused(name): + if state._refused(self, name): return results = state.matches(name, parse_qs(parsed.query)) - self._reply(200, {"count": len(results), "next": None, "previous": None, "results": results}) + page = {"count": len(results), "next": None, "previous": None, "results": results} + state._reply_to(self, "GET", (200, page)) def do_POST(self): name = self._collection_name() payload = self._body() - if name == "graphql": - state.requests.append(("POST", name, payload)) - key = state._LIST_KEY.search(payload.get("query", "")) - self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) + if name != "graphql": + state._write(self, "POST", name, payload, 201, state._create) return state.requests.append(("POST", name, payload)) - if self._refused(name): - return - self._reply(*state._answer(201, lambda: state._create(name, copy.deepcopy(payload)))) + key = state._LIST_KEY.search(payload.get("query", "")) + self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) def do_PATCH(self): - name = self._collection_name() - payload = self._body() - state.requests.append(("PATCH", name, payload)) - if self._refused(name): - return - state._reply_to_patch(self, state._answer(200, lambda: state._patch(name, copy.deepcopy(payload)))) + state._write_patch(self, *state._target(self.path), self._body()) + + def do_DELETE(self): + state._write(self, "DELETE", self._collection_name(), self._body(), 204, state._delete) def log_message(self, *args): """Silence the default stderr access log.""" diff --git a/tests/test_image_writes.py b/tests/test_image_writes.py new file mode 100644 index 00000000..dae6af0b --- /dev/null +++ b/tests/test_image_writes.py @@ -0,0 +1,338 @@ +"""Tests for the image writes: device-type image uploads and module-type image attachments. + +A local HTTP server answers a real ``pynetbox`` client. It parses each multipart upload and +keeps the bytes it received, so a retry that sends an empty file fails visibly. It can drop, +cut off or garble a reply, and it refuses a request the way NetBox does. +""" + +import pynetbox +import pytest +from helpers import FakeNetBox, recording_handle + +from core.graphql_client import NetBoxGraphQLClient +from core.netbox_api import DeviceTypes, NetBox + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +SWITCH = {"id": 5, "model": "C9300", "slug": "c9300", "manufacturer": {"id": 1, "name": "Cisco", "slug": "cisco"}} +REPLIES = ["lost", "truncated", "garbled"] +# Bytes that a careless multipart encoder or parser would change: CRLF, a fake boundary, NUL. +IMAGE_BYTES = b"\x89PNG\r\n\x1a\n" + bytes(range(256)) * 20 + b"\r\n--boundary\r\n\x00" + +# The real 400 NetBox >= 4.6.5 returns for an image above its 25 MP cap. +_OVERSIZE_BODY = ( + '{"front_image":["Upload a valid image. The file you uploaded was either not an image or a corrupted image."]}' +) + + +@pytest.fixture(autouse=True) +def no_backoff(monkeypatch): + """Skip the wait between attempts.""" + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server holding one device type.""" + fake = FakeNetBox(device_types=[SWITCH]) + yield fake + fake.close() + + +@pytest.fixture +def device_types(server, tmp_path): + """Build a real DeviceTypes against the local server, with a recording log and a progress sink.""" + handle, console = recording_handle() + dt = DeviceTypes( + server.api(), + handle, + {"images": 0}, + graphql=NetBoxGraphQLClient(server.url, "test-token"), + repo_path=str(tmp_path), + ) + progress = [] + dt._image_progress = progress.append + return dt, console, progress + + +@pytest.fixture +def image(tmp_path): + """Write one image file and return its path.""" + path = tmp_path / "c9300.front.png" + path.write_bytes(IMAGE_BYTES) + return str(path) + + +def requests_to(server, verb, name): + """Count the *verb* requests sent to collection *name*.""" + return sum(v == verb and n == name for v, n, _payload in server.requests) + + +def logged(console): + """Return every emitted log line as one string.""" + return " ".join(console.lines) + + +def break_every(server, verb, count, how): + """Break the next *count* *verb* requests the same way.""" + done = sum(v == verb for v, n, _payload in server.requests if n != "graphql") + server.broken_replies.update({(verb, done + n): how for n in range(1, count + 1)}) + + +class TestUploadImageAttachment: + """An attachment upload creates exactly one attachment, whatever happens to the reply.""" + + def test_an_upload_creates_the_attachment(self, device_types, server, image): + dt, console, progress = device_types + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True + + [stored] = server.collection("image_attachments") + assert stored["object_type"] == "dcim.moduletype" + assert stored["object_id"] == "42" + assert stored["name"] == "c9300.front" + assert server.uploads == [("image", "c9300.front.png", IMAGE_BYTES)] + assert dt.counter["images"] == 1 + assert progress == [1] + assert console.lines == [] + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_broken_reply_does_not_create_a_second_attachment(self, device_types, server, image, reply): + dt, console, progress = device_types + server.broken_replies = {("POST", 1): reply} + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True + + assert len(server.collection("image_attachments")) == 1 + assert requests_to(server, "POST", "image_attachments") == 1 + assert dt.counter["images"] == 1 + assert progress == [1] + assert console.lines == [] + + def test_an_upload_sent_again_carries_the_whole_file(self, device_types, server, image): + """The first attempt read the file to its end; the second must start from byte 0.""" + dt, _, _ = device_types + server.broken_replies = {("POST", 1): "dropped"} + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True + + assert [data for _field, _name, data in server.uploads] == [IMAGE_BYTES, IMAGE_BYTES] + assert len(server.collection("image_attachments")) == 1 + + def test_a_rejected_upload_returns_false_and_logs_netbox_reason(self, device_types, server, image): + dt, console, progress = device_types + server.errors = {"image_attachments": (400, '{"image":["Upload a valid image."]}')} + + assert dt.upload_image_attachment(image, "dcim.moduletype", 7) is False + + assert "Error uploading image attachment for dcim.moduletype 7" in logged(console) + assert "400" in logged(console) + assert "Upload a valid image." in logged(console) + assert dt.counter["images"] == 0 + assert progress == [] + + def test_retries_exhausted_fail_only_that_upload(self, device_types, server, image): + dt, console, _ = device_types + break_every(server, "POST", 4, "dropped") + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is False + assert "after 3 retries" in logged(console) + assert server.collection("image_attachments") == [] + + assert dt.upload_image_attachment(image, "dcim.moduletype", 43) is True + assert [r["object_id"] for r in server.collection("image_attachments")] == ["43"] + + def test_an_ambiguous_lookup_after_a_lost_reply_fails_that_upload(self, device_types, server, image): + """Two attachments with the name: nothing tells which one the lost reply created.""" + dt, console, _ = device_types + old = {"object_type": "dcim.moduletype", "object_id": "42", "name": "c9300.front"} + server.collection("image_attachments").append({**old, "id": 1}) + server.broken_replies = {("POST", 1): "lost"} + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is False + + assert "Cannot tell if the image attachment for dcim.moduletype 42 was created" in logged(console) + assert requests_to(server, "POST", "image_attachments") == 1 + + def test_an_unreadable_file_returns_false(self, device_types, server, tmp_path): + dt, console, _ = device_types + + assert dt.upload_image_attachment(str(tmp_path / "gone.png"), "dcim.moduletype", 42) is False + + assert "Error reading image file" in logged(console) + assert requests_to(server, "POST", "image_attachments") == 0 + + +class TestUploadDeviceTypeImages: + """A device-type image PATCH is idempotent, so a broken reply sends it again in full.""" + + def test_front_and_rear_images_are_set(self, device_types, server, image, tmp_path): + dt, console, progress = device_types + rear = tmp_path / "c9300.rear.png" + rear.write_bytes(b"rear") + + assert dt.upload_images({"front_image": image, "rear_image": str(rear)}, SWITCH["id"]) is True + + [stored] = server.collection("device_types") + assert stored["front_image"] == "/media/c9300.front.png" + assert stored["rear_image"] == "/media/c9300.rear.png" + assert sorted(data for _f, _n, data in server.uploads) == sorted([IMAGE_BYTES, b"rear"]) + assert requests_to(server, "PATCH", "device_types") == 1 + assert dt.counter["images"] == 2 + assert progress == [2] + assert console.lines == [] + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_broken_reply_sends_the_whole_file_again(self, device_types, server, image, reply): + dt, console, progress = device_types + server.broken_replies = {("PATCH", 1): reply} + + assert dt.upload_images({"front_image": image}, SWITCH["id"]) is True + + assert [data for _field, _name, data in server.uploads] == [IMAGE_BYTES, IMAGE_BYTES] + assert server.collection("device_types")[0]["front_image"] == "/media/c9300.front.png" + assert dt.counter["images"] == 1 + assert progress == [1] + assert console.lines == [] + + def test_a_rejected_upload_logs_netbox_reason(self, device_types, server, image): + dt, console, progress = device_types + server.errors = {"device_types": (400, _OVERSIZE_BODY)} + + assert dt.upload_images({"front_image": image}, 42) is False + + assert "Error uploading images for device type 42" in logged(console) + assert "400" in logged(console) + assert "not an image or a corrupted image" in logged(console) + assert dt.counter["images"] == 0 + assert progress == [] + + def test_a_server_error_logs_its_body(self, device_types, server, image): + """A 500 surfaces the body too: this is the case issue #102 could not diagnose.""" + dt, console, _ = device_types + server.errors = {"device_types": (500, "PermissionError: [Errno 13] Permission denied: '/media'")} + + assert dt.upload_images({"front_image": image}, 30) is False + + assert "500" in logged(console) + assert "Permission denied" in logged(console) + + def test_a_long_error_body_is_truncated(self, device_types, server, image): + """A DEBUG-mode HTML traceback page is capped instead of flooding the log.""" + dt, console, _ = device_types + server.errors = {"device_types": (500, "" + ("x" * 5000) + "")} + + dt.upload_images({"front_image": image}, 1) + + assert "truncated" in logged(console) + assert len(logged(console)) < 1000 + + def test_control_characters_in_the_body_are_escaped(self, device_types, server, image): + """A response body cannot inject line breaks that forge more log records.""" + dt, console, _ = device_types + forged = '{"detail":"boom"}\r\n[12:34:56] Device Type Created: FORGED - 999\x1b[31m' + server.errors = {"device_types": (400, forged)} + + dt.upload_images({"front_image": image}, 1) + + assert "\n" not in logged(console) + assert "\r" not in logged(console) + assert "\x1b" not in logged(console) + assert "\\r\\n" in logged(console) + assert "FORGED" in logged(console) + + def test_retries_exhausted_fail_only_that_device_type(self, device_types, server, image): + dt, console, _ = device_types + break_every(server, "PATCH", 4, "dropped") + + assert dt.upload_images({"front_image": image}, SWITCH["id"]) is False + assert "Error uploading images for device type 5 after 3 retries" in logged(console) + assert "front_image" not in server.collection("device_types")[0] + + assert dt.upload_images({"front_image": image}, SWITCH["id"]) is True + assert server.collection("device_types")[0]["front_image"] == "/media/c9300.front.png" + + def test_an_unreadable_file_sends_nothing(self, device_types, server, image, tmp_path): + dt, console, _ = device_types + + assert dt.upload_images({"front_image": image, "rear_image": str(tmp_path / "gone.png")}, 5) is False + + assert "Error reading image file for device type 5" in logged(console) + assert requests_to(server, "PATCH", "device_types") == 0 + + +class TestDeleteImageAttachment: + """A delete whose reply broke counts as done; a delete NetBox refuses does not.""" + + @pytest.fixture(autouse=True) + def stored(self, server): + """Hold one attachment to delete.""" + server.collection("image_attachments").append({"id": 12, "name": "c9300.front", "object_id": "42"}) + + def test_a_delete_removes_the_attachment(self, device_types, server): + dt, console, _ = device_types + + assert dt.delete_image_attachment(12) is True + + assert server.collection("image_attachments") == [] + assert console.lines == [] + + @pytest.mark.parametrize("reply", ["lost", "dropped"]) + def test_a_broken_reply_counts_as_deleted(self, device_types, server, reply): + dt, console, _ = device_types + server.broken_replies = {("DELETE", 1): reply} + + assert dt.delete_image_attachment(12) is True + + assert server.collection("image_attachments") == [] + assert console.lines == [] + + def test_a_first_attempt_delete_of_a_missing_attachment_fails(self, device_types, server): + dt, console, _ = device_types + + assert dt.delete_image_attachment(99) is False + + assert "Error deleting image attachment 99" in logged(console) + assert "could not be found" in logged(console) + assert len(server.collection("image_attachments")) == 1 + + def test_a_refused_delete_logs_netbox_reason(self, device_types, server): + dt, console, _ = device_types + server.errors = {"image_attachments": (409, '{"detail":"Attachment is in use."}')} + + assert dt.delete_image_attachment(12) is False + + assert "409" in logged(console) + assert "Attachment is in use." in logged(console) + + def test_retries_exhausted_return_false(self, device_types, server): + dt, console, _ = device_types + break_every(server, "DELETE", 1, "dropped") + break_every(server, "GET", 3, "lost") + + assert dt.delete_image_attachment(12) is False + + assert "Error deleting image attachment 12 after 3 retries" in logged(console) + + +class TestImageHashes: + """A failed upload stores no hash, so --verify-images uploads the changed file next run.""" + + @pytest.fixture + def netbox(self, make_config, mock_pynetbox, server, tmp_path): + """Build a real NetBox whose device types talk to the local server.""" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, _console = recording_handle() + nb = NetBox(make_config(netbox_url=server.url, repo_path=str(tmp_path)), handle) + nb.netbox = nb.device_types.netbox = server.api() + return nb + + @pytest.mark.parametrize(("errors", "stored"), [({}, True), ({"device_types": 400}, False)]) + def test_a_hash_is_stored_only_after_the_upload(self, netbox, server, image, errors, stored): + server.errors = errors + dt = pynetbox.core.response.Record({**SWITCH, "front_image": None}, server.api(), None) + + netbox._handle_existing_device_type(dt, {"model": "C9300"}, "cisco", {"front_image": image}, True, None, False) + + assert (image in netbox._image_hash_cache) is stored diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index b2084a1b..2838adc9 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -83,7 +83,6 @@ def _make(module_bay_types_supported=True): server.api(), handle, {}, - False, graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), repo_path=str(catalog_root), module_bay_types_supported=module_bay_types_supported, diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index e629b393..fee56d7d 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -146,6 +146,51 @@ def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): assert "mx304-re" in str(exc.value) assert not server.sent("POST", "module_bay_types") + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) + def test_retries_exhausted_fail_the_definition_not_the_run(self, catalog, monkeypatch, reply): + """The create commits but its reply breaks, and every lookup before a retry is dropped.""" + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + cat, server = catalog() + server.broken_replies = {("POST", 1): reply, **{("GET", n): "lost" for n in (3, 4, 5)}} + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["MX304-RE"]) + assert "MX304-RE" in str(exc.value) + assert len(server.sent("POST", "module_bay_types")) == 1 + assert len(cat.ids_for("juniper", ["MX304-LMIC"])) == 1 + + +@pytest.mark.real_http +class TestRetries: + """A dropped connection is retried, and a create whose reply broke is not repeated.""" + + @pytest.fixture(autouse=True) + def no_backoff(self, monkeypatch): + """Skip the wait between attempts.""" + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + + def test_a_dropped_lookup_is_retried(self, catalog): + cat, server = catalog() + server.broken_replies = {("GET", 1): "lost", ("GET", 2): "lost"} + [bay_type_id] = cat.ids_for("juniper", ["MX304-RE"]) + assert [r["id"] for r in server.collection("module_bay_types")] == [bay_type_id] + + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) + def test_a_module_bay_type_create_with_a_broken_reply_is_not_repeated(self, catalog, reply): + cat, server = catalog() + server.broken_replies = {("POST", 1): reply} + [bay_type_id] = cat.ids_for("juniper", ["MX304-RE"]) + assert [r["id"] for r in server.collection("module_bay_types")] == [bay_type_id] + assert len(server.sent("POST", "module_bay_types")) == 1 + + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) + def test_a_manufacturer_create_with_a_broken_reply_is_not_repeated(self, catalog, reply): + cat, server = catalog(manufacturers=()) + server.broken_replies = {("POST", 1): reply} + cat.ids_for("juniper", ["MX304-RE"]) + assert [r["slug"] for r in server.collection("manufacturers")] == ["juniper"] + assert len(server.sent("POST", "manufacturers")) == 1 + assert len(server.collection("module_bay_types")) == 1 + @pytest.mark.real_http class TestCatalogReading: diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index d2ac5fc6..6068a868 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -8,7 +8,6 @@ import pynetbox import pytest -import requests from helpers import FakeNetBox, recording_handle from core.graphql_client import _to_dotdict @@ -212,15 +211,16 @@ def test_a_rejected_profile_change_keeps_the_old_profile_and_attributes(self, ne assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} - @pytest.mark.parametrize("lost", [{1, 2, 3, 4}, {2, 3, 4, 5}], ids=["clear", "write"]) - def test_a_profile_change_with_lost_responses_keeps_the_old_profile_and_attributes( - self, netbox, server, definition, monkeypatch, lost + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) + @pytest.mark.parametrize("broken", [{1, 2, 3, 4}, {2, 3, 4, 5}], ids=["clear", "write"]) + def test_a_profile_change_with_broken_replies_keeps_the_old_profile_and_attributes( + self, netbox, server, definition, monkeypatch, broken, reply ): - """Each PATCH commits but its reply never arrives, so every retry is exhausted.""" - monkeypatch.setattr("core.netbox_api.time.sleep", lambda _seconds: None) + """Each PATCH commits but its reply breaks, so every retry is exhausted.""" + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) nb, _ = netbox record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) - server.lost_patch_responses = lost + server.broken_replies = {("PATCH", number): reply for number in broken} assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (False, False) @@ -228,29 +228,52 @@ def test_a_profile_change_with_lost_responses_keeps_the_old_profile_and_attribut assert stored_type["profile"] == {"id": 4, "name": "Fan"} assert stored_type["attribute_data"] == {"rpm": 3000} - @pytest.mark.parametrize( - ("reply", "error"), - [("truncated", requests.exceptions.ChunkedEncodingError), ("garbled", pynetbox.core.query.ContentError)], - ) - def test_a_broken_clear_reply_restores_the_old_profile_and_attributes( - self, netbox, server, definition, reply, error - ): + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) + @pytest.mark.parametrize("broken", [1, 2], ids=["clear", "write"]) + def test_a_profile_change_retries_a_broken_reply(self, netbox, server, definition, monkeypatch, broken, reply): + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) nb, _ = netbox record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) - setattr(server, f"{reply}_patch_responses", {1}) + server.broken_replies = {("PATCH", broken): reply} - with pytest.raises(error): - nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (True, True) [stored_type] = server.collection("module_types") - assert stored_type["profile"] == {"id": 4, "name": "Fan"} - assert stored_type["attribute_data"] == {"rpm": 3000} + assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} + assert stored_type["attribute_data"] == SFP_ATTRIBUTES + assert len(server.sent("PATCH", "module_types")) == 3 + + @pytest.mark.parametrize("reply", ["truncated", "garbled"]) + def test_an_update_retries_a_broken_reply(self, netbox, server, definition, monkeypatch, reply): + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + nb, _ = netbox + record = existing(server, {"id": 3, "name": "Transceiver"}, {}) + server.broken_replies = {("PATCH", 1): reply} + + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (True, True) + + assert server.collection("module_types")[0]["attribute_data"] == SFP_ATTRIBUTES + assert len(server.sent("PATCH", "module_types")) == 2 + + @pytest.mark.parametrize("reply", ["truncated", "garbled"]) + def test_an_update_whose_replies_stay_broken_fails_only_that_module_type( + self, netbox, server, definition, monkeypatch, reply + ): + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + nb, console = netbox + record = existing(server, {"id": 3, "name": "Transceiver"}, {}) + server.broken_replies = {("PATCH", number): reply for number in range(1, 5)} + + assert nb._try_update_module_type(definition(), record, "cisco/SFP-10G-SR.yaml") == (False, False) + + assert len(server.sent("PATCH", "module_types")) == 4 + assert any("after 3 retries" in line for line in console.lines) def test_a_failed_recovery_clear_still_restores_the_old_profile(self, netbox, server, definition): nb, console = netbox record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) curr_mt = definition(SFP_YAML.replace("profile: Transceiver", "profile: No Such Profile")) - server.garbled_patch_responses = {3} + server.broken_replies = {("PATCH", number): "garbled" for number in range(3, 7)} assert nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") == (False, False) diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index a2f61af4..ddd4b1a4 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -15,7 +15,6 @@ NetBox, NetBoxError, SSLVerificationError, - _delete_image_attachment, _FrontPortRecordWithMappings, ) from core.outcomes import EntityKind, Outcome @@ -80,7 +79,6 @@ def _factory(nb_api=None, handle=None, counter=None, **kwargs): nb_api if nb_api is not None else MagicMock(), handle if handle is not None else mock_handle, counter if counter is not None else MagicMock(), - False, graphql=kwargs.pop("graphql", graphql_client), repo_path=kwargs.pop("repo_path", mock_settings.repo_path), **kwargs, @@ -273,30 +271,6 @@ def test_redundant_image_upload(mock_settings, mock_pynetbox, mock_handle): nb.device_types.upload_images.assert_not_called() -def test_upload_images_success_logs_verbose_only( - mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle -): - mock_nb_api = mock_pynetbox.api.return_value - - image_file = tmp_path / "front.jpg" - image_file.write_bytes(b"fake") - - dt = make_device_types(nb_api=mock_nb_api) - mock_handle.log.reset_mock() - mock_handle.verbose_log.reset_mock() - - with patch("core.netbox_api.requests.patch") as mock_patch: - response = MagicMock() - response.status_code = 200 - response.raise_for_status.return_value = None - mock_patch.return_value = response - - dt.upload_images("http://mock-netbox", "token", {"front_image": str(image_file)}, 123) - - assert any("Images" in call.args[0] for call in mock_handle.verbose_log.call_args_list) - mock_handle.log.assert_not_called() - - def test_filter_new_module_types_returns_only_missing_items(mock_settings, mock_pynetbox): existing_a = MagicMock() module_types = [ @@ -1003,49 +977,6 @@ def test_creates_new_device_type_with_components( mock_nb_api.dcim.power_port_templates.create.assert_called_once() -class TestUploadImageAttachment: - """Tests for TestUploadImageAttachment.""" - - def test_success_returns_true(self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path): - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img_path = tmp_path / "img.png" - img_path.write_bytes(b"fake") - - with patch("core.netbox_api.requests.post") as mock_post: - resp = MagicMock() - resp.status_code = 201 - resp.raise_for_status.return_value = None - mock_post.return_value = resp - result = dt.upload_image_attachment("http://nb", "token", str(img_path), "dcim.moduletype", 42) - - assert result is True - - def test_request_error_returns_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path - ): - import requests as req_lib - - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img_path = tmp_path / "img.png" - img_path.write_bytes(b"fake") - - with patch("core.netbox_api.requests.post") as mock_post: - mock_post.side_effect = req_lib.RequestException("timeout") - result = dt.upload_image_attachment("http://nb", "token", str(img_path), "dcim.moduletype", 42) - - assert result is False - - def test_os_error_returns_false(self, mock_settings, mock_pynetbox, graphql_client, make_device_types): - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - result = dt.upload_image_attachment("http://nb", "token", "/nonexistent/img.png", "dcim.moduletype", 42) - assert result is False - - class TestImageDirForYaml: """Tests for the _image_dir_for_yaml module-level helper.""" @@ -1473,39 +1404,6 @@ def test_existing_dt_with_image_not_reuploaded( mock_handle.verbose_log.assert_any_call(f"Front image already exists for {existing_dt.model}, skipping upload.") -class TestUploadImagesProgress: - """Tests for TestUploadImagesProgress.""" - - def test_image_progress_callback_called( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path - ): - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img_path = tmp_path / "front.jpg" - img_path.write_bytes(b"fake") - - progress_calls = [] - dt._image_progress = lambda n: progress_calls.append(n) - - with patch("core.netbox_api.requests.patch") as mock_patch: - resp = MagicMock() - resp.status_code = 200 - resp.raise_for_status.return_value = None - mock_patch.return_value = resp - dt.upload_images("http://nb", "token", {"front_image": str(img_path)}, 1) - - assert progress_calls == [1] - - def test_upload_images_os_error_logged( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle - ): - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - dt.upload_images("http://nb", "token", {"front_image": "/nonexistent/img.jpg"}, 1) - mock_handle.log.assert_called() - - class TestCountDeviceTypeImagesEdge: """Tests for TestCountDeviceTypeImagesEdge.""" @@ -4065,93 +3963,6 @@ def test_calls_create_generic_with_module_parent( assert call_args[0]["module_type"] == 3 -# --------------------------------------------------------------------------- -# upload_images RequestException and file-close exception (lines 2039-2046) -# --------------------------------------------------------------------------- - - -class TestUploadImagesErrors: - """Tests for error branches in upload_images.""" - - def test_request_exception_logged( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """requests.RequestException during upload is caught and logged.""" - import requests as _req2 - - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - with patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req2.RequestException - mock_req.patch.side_effect = _req2.RequestException("timeout") - dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) - - assert mock_handle.log.called - - def test_file_close_exception_swallowed( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """Exception from fh.close() in the finally block is silently swallowed.""" - import requests as _req2 - - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - # Make the file object's close() raise an exception - fake_fh = MagicMock() - fake_fh.close.side_effect = OSError("cannot close") - - with patch("builtins.open", return_value=fake_fh), patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req2.RequestException - mock_req.patch.side_effect = _req2.RequestException("server error") - # Should NOT raise despite close() raising - dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) - - # The RequestException log should still have been called - assert mock_handle.log.called - - -# --------------------------------------------------------------------------- -# upload_image_attachment: _image_progress callback, OSError (lines 2090, 2095-2097) -# --------------------------------------------------------------------------- - - -class TestUploadImageAttachmentProgress: - """Tests for upload_image_attachment _image_progress and error paths.""" - - def test_image_progress_callback_called_on_success( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path - ): - """_image_progress is called with 1 on a successful upload.""" - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img = tmp_path / "img.png" - img.write_bytes(b"fake") - - progress_calls = [] - dt._image_progress = lambda n: progress_calls.append(n) - - with patch("core.netbox_api.requests.post") as mock_post: - resp = MagicMock() - resp.status_code = 201 - resp.raise_for_status.return_value = None - mock_post.return_value = resp - result = dt.upload_image_attachment("http://nb", "token", str(img), "dcim.moduletype", 42) - - assert result is True - assert progress_calls == [1] - - # --------------------------------------------------------------------------- # Corner-case tests for high-complexity functions (cognitive complexity > 15) # --------------------------------------------------------------------------- @@ -4373,58 +4184,9 @@ def test_module_outlet_without_power_port_key_skips_link( call_args = mock_nb_api.dcim.power_outlet_templates.create.call_args[0][0] assert call_args[0]["name"] == "PO1" - -class TestUploadImageAttachmentExceptions: - """Tests for exception paths in upload_image_attachment.""" - - def test_request_exception_returns_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """requests.RequestException during POST returns False (lines 2095-2097).""" - import requests as _req3 - - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img = tmp_path / "img.png" - img.write_bytes(b"fake") - - with patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req3.RequestException - mock_req.post.side_effect = _req3.RequestException("conn error") - result = dt.upload_image_attachment("http://nb", "token", str(img), "dcim.moduletype", 42) - - assert result is False - assert mock_handle.log.called - # "other_endpoint" was dropped; no exception raised -class TestUploadImagesRequestException: - """Dedicated tests to ensure upload_images RequestException path is covered.""" - - def test_upload_images_request_exception_lines_covered( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """Verify lines 2039-2040 (RequestException catch in upload_images) are executed.""" - import requests as _req4 - - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - - img = tmp_path / "front2.jpg" - img.write_bytes(b"data") - mock_handle.log.reset_mock() - - # Patch the whole requests namespace so except clause matches - with patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req4.RequestException - mock_req.patch.side_effect = _req4.RequestException("network error") - dt.upload_images("http://nb", "tok", {"front_image": str(img)}, 99) - - assert mock_handle.log.called - - # --------------------------------------------------------------------------- # TestGetExistingRackTypes # --------------------------------------------------------------------------- @@ -4950,18 +4712,18 @@ def test_retryable_exception_returns_false_and_logs(self, mock_settings, mock_py module_type_res = self._make_module_type_res() module_type_res.part_number = "OLD-123" - with patch("core.netbox_api.time.sleep") as mock_sleep: + with patch("core.retry.time.sleep") as mock_sleep: ok, updated = nb._try_update_module_type(curr_mt, module_type_res, "test.yaml") assert ok is False assert updated is False mock_handle.log.assert_called() # Prove the retry loop actually ran the full budget with the correct backoff. - from core.netbox_api import _MAX_RETRIES, _RETRY_BACKOFF + from core.retry import MAX_RETRIES, RETRY_BACKOFF - assert nb.netbox.dcim.module_types.update.call_count == _MAX_RETRIES + 1 - assert mock_sleep.call_count == _MAX_RETRIES - assert [call.args[0] for call in mock_sleep.call_args_list] == list(_RETRY_BACKOFF[:_MAX_RETRIES]) + assert nb.netbox.dcim.module_types.update.call_count == MAX_RETRIES + 1 + assert mock_sleep.call_count == MAX_RETRIES + assert [call.args[0] for call in mock_sleep.call_args_list] == list(RETRY_BACKOFF[:MAX_RETRIES]) # --------------------------------------------------------------------------- @@ -4989,6 +4751,7 @@ def test_retryable_exception_on_create_returns_false( mock_handle.log.reset_mock() mock_nb_api.dcim.module_types.create.side_effect = requests.exceptions.ConnectionError("network down") + mock_nb_api.dcim.module_types.get.return_value = None # no earlier attempt committed curr_mt = { "manufacturer": {"slug": "cisco"}, @@ -4996,7 +4759,7 @@ def test_retryable_exception_on_create_returns_false( "slug": "cm-retryable", } - with patch("core.netbox_api.time.sleep") as mock_sleep: + with patch("core.retry.time.sleep") as mock_sleep: result = nb._process_single_module_type( curr_mt, "test.yaml", @@ -5008,11 +4771,11 @@ def test_retryable_exception_on_create_returns_false( assert result is False mock_handle.log.assert_called() # Prove the retry loop actually ran the full budget with the correct backoff. - from core.netbox_api import _MAX_RETRIES, _RETRY_BACKOFF + from core.retry import MAX_RETRIES, RETRY_BACKOFF - assert mock_nb_api.dcim.module_types.create.call_count == _MAX_RETRIES + 1 - assert mock_sleep.call_count == _MAX_RETRIES - assert [call.args[0] for call in mock_sleep.call_args_list] == list(_RETRY_BACKOFF[:_MAX_RETRIES]) + assert mock_nb_api.dcim.module_types.create.call_count == MAX_RETRIES + 1 + assert mock_sleep.call_count == MAX_RETRIES + assert [call.args[0] for call in mock_sleep.call_args_list] == list(RETRY_BACKOFF[:MAX_RETRIES]) # --------------------------------------------------------------------------- @@ -5597,28 +5360,6 @@ def test_store_image_hashes_skips_unreadable_files(self, tmp_path): assert str(good) in cache assert str(missing) not in cache - def test_delete_image_attachment_success(self, mock_settings, mock_handle): - from core.netbox_api import _delete_image_attachment - - response = MagicMock() - response.raise_for_status = MagicMock() - with patch("core.netbox_api.requests.delete", return_value=response) as mock_delete: - result = _delete_image_attachment("http://nb", mock_settings.netbox_token, 12, False, mock_handle) - assert result is True - - mock_delete.assert_called_once() - - def test_delete_image_attachment_logs_request_errors(self, mock_settings, mock_handle): - import requests - - from core.netbox_api import _delete_image_attachment - - with patch("core.netbox_api.requests.delete", side_effect=requests.RequestException("boom")): - result = _delete_image_attachment("http://nb", mock_settings.netbox_token, 12, False, mock_handle) - assert result is False - - assert any("Error deleting image attachment 12" in str(call) for call in mock_handle.log.call_args_list) - def test_load_module_type_properties_falls_back_without_a_schema(self, tmp_path): """A checkout with no schema directory yields the hardcoded list, with nothing swallowed.""" from core.netbox_api import _MODULE_TYPE_PROPERTIES_FALLBACK, _load_module_type_properties @@ -5825,7 +5566,7 @@ def test_missing_server_image_skips_upload_when_delete_fails( with ( patch("core.netbox_api._check_image_url", return_value="missing"), - patch("core.netbox_api._delete_image_attachment", return_value=False), + patch.object(nb.device_types, "delete_image_attachment", return_value=False), ): nb._upload_module_type_images(mt_res, str(src), existing_images) @@ -5873,7 +5614,7 @@ def test_changed_module_image_is_deleted_and_reuploaded(self, mock_settings, moc with ( patch("core.netbox_api._check_image_url", return_value="present"), patch("core.netbox_api._is_image_hash_changed", return_value=True), - patch("core.netbox_api._delete_image_attachment", return_value=True), + patch.object(nb.device_types, "delete_image_attachment", return_value=True), patch("core.netbox_api._save_image_hash_cache"), ): nb._upload_module_type_images(mt_res, str(src), existing_images) @@ -6046,7 +5787,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.manufacturers.create.side_effect = requests.exceptions.ConnectionError("offline") - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): nb.create_manufacturers([{"name": "Cisco", "slug": "cisco"}]) assert any("Connection error creating manufacturers" in str(c) for c in mock_handle.log.call_args_list) @@ -6145,7 +5886,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( with ( patch("core.netbox_api.classify_device_type_update_failure", return_value=resolution), - patch("core.netbox_api.time.sleep"), + patch("core.retry.time.sleep"), ): ok, returned = nb._try_resolve_and_retry_device_type_update(dt, {}, {"slug": "m1"}, MagicMock()) @@ -6231,7 +5972,7 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( component_changes=[], ) - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): nb._handle_existing_device_type(dt, {}, "cisco", {}, False, dt_change, False) assert any("Connection error updating device type Router" in str(c) for c in mock_handle.log.call_args_list) @@ -6245,8 +5986,9 @@ def test_create_new_device_type_logs_retryable_error(self, mock_settings, mock_p mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.create.side_effect = requests.exceptions.ConnectionError("offline") + nb.netbox.dcim.device_types.get.return_value = None # no earlier attempt committed - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): dt, should_continue = nb._create_new_device_type( {"manufacturer": {"slug": "cisco"}, "model": "Router"}, "router.yaml", @@ -6446,7 +6188,7 @@ def test_create_generic_logs_retryable_exception( dcim = mock_pynetbox.api.return_value.dcim dcim.interface_templates.create.side_effect = requests.exceptions.ConnectionError("offline") - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): dt._create_generic(BY_YAML_KEY["interfaces"], [{"name": "eth0"}], 1) assert any("Connection error creating Interface" in str(c) for c in mock_handle.log.call_args_list) @@ -6473,7 +6215,7 @@ def test_apply_updates_for_type_logs_retryable_exception( ) ] - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): dt._apply_updates_for_type("interfaces", changes, {}, 1, "device") assert any("Connection error updating interfaces (ID: 9)" in str(c) for c in mock_handle.log.call_args_list) @@ -6493,7 +6235,7 @@ def test_remove_components_logs_retryable_exception( dt.netbox.dcim.interface_templates.delete.side_effect = requests.exceptions.ConnectionError("offline") changes = [ComponentChange("interfaces", "eth0", ChangeType.COMPONENT_REMOVED)] - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): dt.remove_components(1, changes) assert any("Connection error removing interfaces (ID: 9)" in str(c) for c in mock_handle.log.call_args_list) @@ -6513,7 +6255,7 @@ def test_create_interfaces_logs_retryable_bridge_update_exception( dt.netbox.dcim.interface_templates.update.side_effect = requests.exceptions.ConnectionError("offline") interfaces = [{"name": "eth0", "type": "virtual", "bridge": "eth1"}, {"name": "eth1", "type": "virtual"}] - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): dt.create_components("interfaces", interfaces, 1, context="ctx.yaml") assert any("Connection error bridging interfaces" in str(c) for c in mock_handle.log.call_args_list) @@ -6545,7 +6287,7 @@ def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock nb = NetBox(mock_settings, mock_handle) existing = DotDict({"id": 5, "model": "AR1300", "u_height": 40}) - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): nb.create_rack_types( [{"manufacturer": {"slug": "apc"}, "model": "AR1300", "u_height": 42, "src": "rack.yaml"}], all_rack_types={"apc": {"AR1300": existing}}, @@ -6562,9 +6304,10 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = requests.exceptions.ConnectionError( "offline" ) + mock_pynetbox.api.return_value.dcim.rack_types.get.return_value = None # no earlier attempt committed nb = NetBox(mock_settings, mock_handle) - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): nb.create_rack_types( [{"manufacturer": {"slug": "apc"}, "model": "AR1300", "slug": "apc-ar1300", "src": "rack.yaml"}], all_rack_types={}, @@ -6594,7 +6337,7 @@ def test_upload_module_type_images_discards_missing_attachment_before_failed_upl with ( patch("core.netbox_api._check_image_url", return_value="missing"), - patch("core.netbox_api._delete_image_attachment", return_value=True), + patch.object(nb.device_types, "delete_image_attachment", return_value=True), ): nb._upload_module_type_images(mt_res, str(src), existing_images) @@ -6623,7 +6366,7 @@ def test_upload_module_type_images_skips_changed_image_when_delete_fails( with ( patch("core.netbox_api._check_image_url", return_value="present"), patch("core.netbox_api._is_image_hash_changed", return_value=True), - patch("core.netbox_api._delete_image_attachment", return_value=False), + patch.object(nb.device_types, "delete_image_attachment", return_value=False), ): nb._upload_module_type_images(mt_res, str(src), existing_images) @@ -6633,9 +6376,7 @@ def test_upload_module_type_images_skips_changed_image_when_delete_fails( # --------------------------------------------------------------------------- -# Image-upload failures must surface NetBox's HTTP status and response body. -# Exercised against a real local HTTP server so requests, raise_for_status and -# response parsing all run for real; only NetBox itself is substituted. +# A real local HTTP server with one canned reply, for the image-URL checks above. # --------------------------------------------------------------------------- @@ -6675,127 +6416,6 @@ def _netbox_returning(status, body, content_type="application/json"): # The real 400 NetBox >= 4.6.5 returns for an image above its 25 MP cap. -_OVERSIZE_BODY = ( - '{"front_image":["Upload a valid image. The file you uploaded was either not an image or a corrupted image."]}' -) - - -class TestImageUploadErrorDetail: - """Failed uploads must log NetBox's status code and response body, not just str(exc).""" - - def test_upload_images_logs_status_and_response_body( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """A 400 from NetBox surfaces the field-level reason it returned.""" - dt = make_device_types(nb_api=mock_pynetbox.api.return_value) - img = tmp_path / "front.jpg" - img.write_bytes(b"not-really-a-jpeg") - mock_handle.log.reset_mock() - - with _netbox_returning(400, _OVERSIZE_BODY) as url: - dt.upload_images(url, "token", {"front_image": str(img)}, 42) - - logged = " ".join(str(c.args[0]) for c in mock_handle.log.call_args_list) - assert "400" in logged - assert "not an image or a corrupted image" in logged - - def test_upload_images_logs_body_for_server_error( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """A 500 surfaces the body too — this is the case issue #102 could not diagnose.""" - dt = make_device_types(nb_api=mock_pynetbox.api.return_value) - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - with _netbox_returning(500, "PermissionError: [Errno 13] Permission denied: '/opt/netbox/netbox/media'") as url: - dt.upload_images(url, "token", {"front_image": str(img)}, 30) - - logged = " ".join(str(c.args[0]) for c in mock_handle.log.call_args_list) - assert "500" in logged - assert "Permission denied" in logged - - def test_upload_image_attachment_logs_status_and_response_body( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """The image-attachment path (module types) reports the same detail.""" - dt = make_device_types(nb_api=mock_pynetbox.api.return_value) - img = tmp_path / "module.png" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - with _netbox_returning(400, '{"image":["Upload a valid image."]}') as url: - ok = dt.upload_image_attachment(url, "token", str(img), "dcim.moduletype", 7) - - assert ok is False - logged = " ".join(str(c.args[0]) for c in mock_handle.log.call_args_list) - assert "400" in logged - assert "Upload a valid image." in logged - - def test_upload_images_success_is_unaffected( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """A 200 still counts the upload and logs nothing at error level.""" - counter = {"images": 0} - dt = make_device_types(nb_api=mock_pynetbox.api.return_value, counter=counter) - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - with _netbox_returning(200, "{}") as url: - dt.upload_images(url, "token", {"front_image": str(img)}, 1) - - assert counter["images"] == 1 - assert not mock_handle.log.called - - def test_delete_image_attachment_logs_response_body(self): - """Attachment deletion reports NetBox's reason rather than a bare status line.""" - handle = MagicMock() - - with _netbox_returning(409, '{"detail":"Attachment is in use."}') as url: - ok = _delete_image_attachment(url, "token", 5, False, handle) - - assert ok is False - logged = " ".join(str(c.args[0]) for c in handle.log.call_args_list) - assert "409" in logged - assert "Attachment is in use." in logged - - def test_long_error_body_is_truncated( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """A DEBUG-mode HTML traceback page is capped instead of flooding the log.""" - dt = make_device_types(nb_api=mock_pynetbox.api.return_value) - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - with _netbox_returning(500, "" + ("x" * 5000) + "") as url: - dt.upload_images(url, "token", {"front_image": str(img)}, 1) - - logged = " ".join(str(c.args[0]) for c in mock_handle.log.call_args_list) - assert "truncated" in logged - assert len(logged) < 1000 - - def test_control_characters_in_body_are_escaped( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, tmp_path, mock_handle - ): - """A response body cannot inject line breaks that forge additional log records.""" - dt = make_device_types(nb_api=mock_pynetbox.api.return_value) - img = tmp_path / "front.jpg" - img.write_bytes(b"fake") - mock_handle.log.reset_mock() - - forged = '{"detail":"boom"}\r\n[12:34:56] Device Type Created: FORGED - 999\x1b[31m' - with _netbox_returning(400, forged) as url: - dt.upload_images(url, "token", {"front_image": str(img)}, 1) - - logged = " ".join(str(c.args[0]) for c in mock_handle.log.call_args_list) - assert "\n" not in logged - assert "\r" not in logged - assert "\x1b" not in logged - # The text is still reported, just neutralised. - assert "\\r\\n" in logged - assert "FORGED" in logged @pytest.mark.real_http @@ -6841,7 +6461,6 @@ def _device_types(url, handle): MagicMock(), handle, MagicMock(), - False, graphql=NetBoxGraphQLClient(url, "token", page_size=10), repo_path="/tmp/repo", max_threads=2, @@ -7417,7 +7036,7 @@ def test_component_update_transport_failure_reports_the_transport_error( "interfaces": [{"name": "LAN", "type": "2.5gbase-t", "description": "uplink"}], "src": "/repo/device-types/TP-Link/EAP670.yaml", } - with patch("core.netbox_api.time.sleep"): + with patch("core.retry.time.sleep"): nb.create_device_types( [device_type], update=True, change_report=ChangeReport(modified_device_types=[change]) ) @@ -7447,7 +7066,7 @@ def test_component_removal_transport_failure_reports_the_transport_error( dt.components.record("interface_templates", "device", 6119, {"OLD": stale}) mock_nb_api.dcim.interface_templates.delete.side_effect = requests.exceptions.ConnectionError("network down") - with patch("core.netbox_api.time.sleep"), dt.collect_component_errors() as errors: + with patch("core.retry.time.sleep"), dt.collect_component_errors() as errors: dt.remove_components( 6119, [ComponentChange("interfaces", "OLD", ChangeType.COMPONENT_REMOVED)], diff --git a/tests/test_write_retries.py b/tests/test_write_retries.py new file mode 100644 index 00000000..0f0faacc --- /dev/null +++ b/tests/test_write_retries.py @@ -0,0 +1,241 @@ +"""Tests for retrying a NetBox write whose reply broke after the write committed. + +A local HTTP server answers a real ``pynetbox`` client. It commits each write, then drops, +cuts off or garbles the reply when a test asks it to, and it refuses a duplicate the way +NetBox does. A retry that sends the write again therefore fails visibly. +""" + +import pynetbox +import pytest +from helpers import FakeNetBox, recording_handle + +from core.change_detector import ChangeType, ComponentChange +from core.component_registry import BY_YAML_KEY +from core.netbox_api import NetBox +from core.retry import create_with_retry + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +CISCO = {"id": 1, "name": "Cisco", "slug": "cisco"} +SWITCH = {"id": 5, "model": "C9300", "slug": "c9300", "manufacturer": {"id": 1, "name": "Cisco", "slug": "cisco"}} +REPLIES = ["lost", "truncated", "garbled"] + + +@pytest.fixture(autouse=True) +def no_backoff(monkeypatch): + """Skip the wait between attempts.""" + monkeypatch.setattr("core.retry.time.sleep", lambda _seconds: None) + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server holding one manufacturer and one device type.""" + fake = FakeNetBox(manufacturers=[CISCO], device_types=[SWITCH]) + yield fake + fake.close() + + +@pytest.fixture +def netbox(make_config, mock_pynetbox, server, tmp_path): + """Build a real NetBox against the local server.""" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, console = recording_handle() + nb = NetBox(make_config(netbox_url=server.url, repo_path=str(tmp_path)), handle) + + # connect_api ran against the patched pynetbox; every call under test uses a real client. + api = server.api() + nb.netbox = api + nb.device_types.netbox = api + nb.device_types.components.netbox = api + nb.device_types.components.ensure_ready() + return nb, console + + +def requests_to(server, verb, name): + """Count the *verb* requests sent to collection *name*.""" + return sum(v == verb and n == name for v, n, _payload in server.requests) + + +def break_reads_after_now(server, count=3): + """Drop the reply of the next *count* GETs, so each lookup before a retry fails too.""" + done = sum(verb == "GET" for verb, _name, _payload in server.requests) + server.broken_replies.update({("GET", done + n): "lost" for n in range(1, count + 1)}) + + +class TestCreate: + """A create whose reply broke returns the object it committed, and creates no duplicate.""" + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_device_type_create_is_not_repeated(self, netbox, server, reply): + nb, _ = netbox + server.broken_replies = {("POST", 1): reply} + + dt, skip = nb._create_new_device_type( + {"manufacturer": {"slug": "cisco"}, "model": "C9500", "slug": "c9500"}, "c9500.yaml" + ) + + assert skip is False + stored = [r for r in server.collection("device_types") if r["model"] == "C9500"] + assert len(stored) == 1 + assert dt.id == stored[0]["id"] + assert dt.manufacturer.name == "Cisco" + assert requests_to(server, "POST", "device_types") == 1 + assert nb.counter["added"] == 1 + assert nb.outcomes.failures() == [] + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_module_type_create_is_not_repeated(self, netbox, server, reply): + nb, _ = netbox + server.broken_replies = {("POST", 1): reply} + all_module_types = {} + + ok = nb._process_single_module_type( + {"manufacturer": {"slug": "cisco"}, "model": "C9300-NM-8X"}, "nm.yaml", all_module_types, {}, False + ) + + assert ok is True + [stored] = server.collection("module_types") + assert all_module_types["cisco"]["C9300-NM-8X"].id == stored["id"] + assert requests_to(server, "POST", "module_types") == 1 + assert nb.counter["module_added"] == 1 + assert nb.outcomes.failures() == [] + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_rack_type_create_is_not_repeated(self, netbox, server, reply): + nb, _ = netbox + server.broken_replies = {("POST", 1): reply} + rack = {"manufacturer": {"slug": "cisco"}, "model": "R42", "slug": "r42", "form_factor": "4-post-cabinet"} + + nb.create_rack_types([{**rack, "src": "r42.yaml"}], all_rack_types={}) + + assert len(server.collection("rack_types")) == 1 + assert requests_to(server, "POST", "rack_types") == 1 + assert nb.counter["rack_type_added"] == 1 + assert nb.outcomes.failures() == [] + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_manufacturer_bulk_create_is_not_repeated(self, netbox, server, reply): + nb, console = netbox + server.broken_replies = {("POST", 1): reply} + + nb.create_manufacturers([{"name": "Juniper"}, {"name": "Nokia"}]) + + assert sorted(r["slug"] for r in server.collection("manufacturers")) == ["cisco", "juniper", "nokia"] + assert requests_to(server, "POST", "manufacturers") == 1 + assert nb.counter["manufacturer"] == 2 + assert not any("Error creating manufacturers" in line for line in console.lines) + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_component_bulk_create_is_not_repeated(self, netbox, server, reply): + nb, _ = netbox + server.broken_replies = {("POST", 1): reply} + items = [{"name": f"Gi1/0/{n}", "type": "1000base-t"} for n in (1, 2, 3)] + + with nb.device_types.collect_component_errors() as errors: + nb.device_types._create_generic(BY_YAML_KEY["interfaces"], items, SWITCH["id"]) + + assert errors == [] + names = [r["name"] for r in server.collection("interface_templates")] + assert names == ["Gi1/0/1", "Gi1/0/2", "Gi1/0/3"] + assert requests_to(server, "POST", "interface_templates") == 1 + assert nb.counter["components_added"] == 3 + + def test_a_large_bulk_create_is_looked_up_in_short_requests(self, netbox, server): + """Hundreds of names in one lookup URL would pass a proxy's URL limit and fail with 414.""" + nb, _ = netbox + server.max_url_length = 2000 + server.broken_replies = {("POST", 1): "lost"} + items = [{"name": f"GigabitEthernet1/0/{n}", "type": "1000base-t"} for n in range(1, 201)] + + with nb.device_types.collect_component_errors() as errors: + nb.device_types._create_generic(BY_YAML_KEY["interfaces"], items, SWITCH["id"]) + + assert errors == [] + assert len(server.collection("interface_templates")) == 200 + assert requests_to(server, "POST", "interface_templates") == 1 + assert nb.counter["components_added"] == 200 + + def test_a_partial_bulk_match_is_sent_again_not_taken_as_the_result(self, server): + """Another writer holds one of the names: NetBox refuses the batch, and nothing is guessed.""" + server.collection("interface_templates").append({"id": 77, "name": "Gi1/0/1", "device_type": {"id": 5}}) + server.broken_replies = {("POST", 1): "lost"} + items = [{"name": "Gi1/0/1", "device_type": 5}, {"name": "Gi1/0/2", "device_type": 5}] + + with pytest.raises(pynetbox.RequestError): + create_with_retry(server.api().dcim.interface_templates, items, "name", device_type_id=5) + + assert [r["name"] for r in server.collection("interface_templates")] == ["Gi1/0/1"] + assert requests_to(server, "POST", "interface_templates") == 2 + + def test_retries_exhausted_fail_only_that_device_type(self, netbox, server): + nb, _ = netbox + server.broken_replies = {("POST", 1): "lost"} + break_reads_after_now(server) + + dt, skip = nb._create_new_device_type( + {"manufacturer": {"slug": "cisco"}, "model": "C9500", "slug": "c9500"}, "c9500.yaml" + ) + nxt, _ = nb._create_new_device_type( + {"manufacturer": {"slug": "cisco"}, "model": "C9600", "slug": "c9600"}, "c9600.yaml" + ) + + assert (dt, skip) == (None, True) + [failure] = nb.outcomes.failures() + assert "C9500" in failure.identity + assert "after 3 retries" in failure.reason + assert requests_to(server, "POST", "device_types") == 2 # one per device type; none is sent twice + assert nxt.model == "C9600" + + +class TestDelete: + """A delete whose reply broke counts as done; a delete NetBox refuses does not.""" + + def test_a_delete_with_a_lost_reply_counts_as_removed(self, netbox, server): + nb, _ = netbox + server.collection("interface_templates").append({"id": 50, "name": "OLD", "device_type": {"id": 5}}) + server.broken_replies = {("DELETE", 1): "lost"} + + with nb.device_types.collect_component_errors() as errors: + nb.device_types.remove_components( + SWITCH["id"], [ComponentChange("interfaces", "OLD", ChangeType.COMPONENT_REMOVED)] + ) + + assert errors == [] + assert server.collection("interface_templates") == [] + assert requests_to(server, "DELETE", "interface_templates") == 1 + assert nb.counter["components_removed"] == 1 + + def test_a_first_attempt_delete_of_a_missing_object_is_an_error(self, netbox, server): + nb, _ = netbox + stale = pynetbox.core.response.Record({"id": 88, "name": "OLD"}, server.api(), None) + nb.device_types.components.record("interface_templates", "device", SWITCH["id"], {"OLD": stale}) + + with nb.device_types.collect_component_errors() as errors: + nb.device_types.remove_components( + SWITCH["id"], [ComponentChange("interfaces", "OLD", ChangeType.COMPONENT_REMOVED)] + ) + + [error] = errors + assert "Error removing interfaces (ID: 88)" in error + assert requests_to(server, "DELETE", "interface_templates") == 1 + assert nb.counter["components_removed"] == 0 + + +class TestTimeout: + """A stalled reply times out, so the run does not hang on it.""" + + def test_a_stalled_create_times_out_and_is_not_repeated(self, make_config, server, tmp_path, monkeypatch): + monkeypatch.setattr("core.netbox_api._REST_TIMEOUT", (5, 0.5)) + # A real client from connect_api, so the session under test is the one the importer builds. + nb = NetBox(make_config(netbox_url=server.url, repo_path=str(tmp_path)), recording_handle()[0]) + server.broken_replies = {("POST", 1): "stalled"} + + dt, skip = nb._create_new_device_type( + {"manufacturer": {"slug": "cisco"}, "model": "C9500", "slug": "c9500"}, "c9500.yaml" + ) + + assert skip is False + assert dt.model == "C9500" + assert len([r for r in server.collection("device_types") if r["model"] == "C9500"]) == 1 + assert requests_to(server, "POST", "device_types") == 1 From 47e87233dfe26317c0db5c6154b02ef93740c49f Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 07:40:58 +0200 Subject: [PATCH 29/39] test: keep FakeNetBox ids unique and skip the write-retry backoff FakeNetBox gave a new record the id 1000 + len(collection). After a delete, or with a seeded id of 1000 or more, a create reused an id that a stored record still held. Each collection now counts up from its highest seeded, stored or assigned id. Two tests drove core.retry through all retries and slept 2+5+10 s each. An autouse conftest fixture now patches core.retry.time.sleep for every test except integration tests, so a new retry test cannot add the wait. The full suite drops from 81 s to 38 s. --- tests/conftest.py | 10 ++++++++++ tests/helpers.py | 7 ++++++- tests/test_suite_hygiene.py | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 32544ae5..19d929c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -125,6 +125,16 @@ def mock_graphql_requests(request): yield mock_session.post +@pytest.fixture(autouse=True) +def no_write_retry_backoff(request): + """Skip the 2+5+10 s wait between NetBox write retries; integration tests keep it.""" + if request.node.get_closest_marker("integration"): + yield + return + with patch("core.retry.time.sleep"): + yield + + @pytest.fixture def no_retry_backoff(): """Drop the client's 1+2+4s retry back-off, leaving the retry count and error type intact.""" diff --git a/tests/helpers.py b/tests/helpers.py index 02e38b21..ac4ff0a5 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -116,6 +116,7 @@ def __init__(self, errors=None, netbox_version="4.7.0", **collections): ``/api/status/`` reports, which is how the export side decides what it may select. """ self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} + self._last_ids = {} # collection name -> highest id a create assigned self.errors = dict(errors or {}) self.netbox_version = netbox_version # A GET whose request target is longer than this gets 414, as from a proxy URL limit. @@ -199,9 +200,13 @@ def _create(self, name, payload): # A bulk create is atomic: one duplicate refuses the whole batch. raise _Refused(errors if isinstance(payload, list) else errors[0]) collection = self.collection(name) + # Ids stay above every seeded, stored and deleted id, as NetBox's sequences do. + last = max([self._last_ids.get(name, 999), *(r["id"] for r in collection)]) for record in records: - record["id"] = 1000 + len(collection) + last += 1 + record["id"] = last collection.append(record) + self._last_ids[name] = last return records if isinstance(payload, list) else records[0] def _duplicate_error(self, name, item): diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index c456b12b..2208b117 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -8,6 +8,8 @@ import sys import tomllib +import pytest + _TESTS_DIR = pathlib.Path(__file__).resolve().parent _ROOT = _TESTS_DIR.parent @@ -120,3 +122,40 @@ def test_no_test_function_contains_an_orphaned_docstring(): assert not offenders, "Bare string statement inside a test function, likely a lost `def` header:\n" + "\n".join( offenders ) + + +@pytest.mark.real_http +def test_a_fake_netbox_create_never_reuses_a_stored_id(): + """The fake must not give a new record the id of a seeded or still-stored one.""" + import pynetbox + from helpers import FakeNetBox + + server = FakeNetBox(manufacturers=[{"id": 1000, "name": "Juniper", "slug": "juniper"}]) + try: + api = pynetbox.api(server.url, token="test-token") + first = api.dcim.manufacturers.create(name="Cisco", slug="cisco") + second = api.dcim.manufacturers.create(name="Nokia", slug="nokia") + api.dcim.manufacturers.delete([first.id]) + third = api.dcim.manufacturers.create(name="Arista", slug="arista") + finally: + server.close() + + ids = [1000, first.id, second.id, third.id] + assert len(set(ids)) == len(ids), ids + + +def test_the_suite_skips_the_write_retry_backoff(): + """A test that drives write retries must not wait 2+5+10 s of real backoff.""" + import time + + import requests + + from core.retry import call_with_retry + + def offline(): + raise requests.exceptions.ConnectionError("offline") + + started = time.monotonic() + with pytest.raises(requests.exceptions.ConnectionError): + call_with_retry(offline) + assert time.monotonic() - started < 1 From d39416cc3240c7f049566e2440daca1498d2ac1d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 07:44:59 +0200 Subject: [PATCH 30/39] fix(repo): bind git to the library checkout on a pull A hook runner or a shell can export GIT_DIR and GIT_WORK_TREE for another checkout. GitPython passes the process environment to git, so pull_repo fetched and ran "checkout -B" against that checkout and left the library unchanged. open_repo() opens a repository and sets GIT_DIR and GIT_WORK_TREE on its git wrapper to that repository. A suite check rejects a Repo() call in core/ outside open_repo(). A clone with the same environment stops with a fatal git error, so it needs no change. --- core/repo.py | 9 ++++++++- tests/test_repo.py | 22 ++++++++++++++++++++++ tests/test_suite_hygiene.py | 27 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/core/repo.py b/core/repo.py index 2c261f1d..f8fb0790 100644 --- a/core/repo.py +++ b/core/repo.py @@ -80,6 +80,13 @@ def find_class(self, module, name): _INDEX_MAX_BYTES = 10 * 1024 * 1024 # 10 MiB — DTL index files are typically <1 MiB +def open_repo(path) -> Repo: + """Open the repository at *path* with git bound to it over an inherited GIT_DIR or GIT_WORK_TREE.""" + repo = Repo(path) + repo.git.update_environment(GIT_DIR=repo.git_dir, GIT_WORK_TREE=os.path.abspath(path)) + return repo + + def _resolve_index_path(base_dir: str, stem: str) -> "str | None": """Return the path to a DTL index file, preferring the JSON form over the legacy pickle. @@ -591,7 +598,7 @@ def pull_repo(self): self.handle.log( "Package devicetype-library is already installed, " + f"updating {self.get_absolute_path()}" ) - self.repo = Repo(self.repo_path) + self.repo = open_repo(self.repo_path) origin_url = self.repo.remotes.origin.url remote_label = origin_url diff --git a/tests/test_repo.py b/tests/test_repo.py index c33af6b2..fd9584f8 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -322,6 +322,28 @@ def test_pulls_when_target_is_an_existing_clone(self, tmp_path): assert (target / "device-types" / "TestVendor" / "second.yaml").is_file() + def test_a_pull_ignores_an_inherited_git_work_tree(self, tmp_path, monkeypatch): + """An exported GIT_DIR or GIT_WORK_TREE for another checkout must not receive the checkout.""" + source = tmp_path / "source" + source.mkdir() + source_repo = self._make_source_repo(source) + target = tmp_path / "repo" + target.mkdir() + self._init_dtl(source, target) + foreign = tmp_path / "foreign" + foreign.mkdir() + foreign_head = self._make_source_repo(foreign).head.commit.hexsha + (source / "device-types" / "TestVendor" / "second.yaml").write_text("manufacturer: TestVendor\nmodel: Second\n") + self._commit(source_repo, "device-types/TestVendor/second.yaml", "second device") + monkeypatch.setenv("GIT_DIR", str(foreign / ".git")) + monkeypatch.setenv("GIT_WORK_TREE", str(foreign)) + + self._init_dtl(source, target) + + assert (target / "device-types" / "TestVendor" / "second.yaml").is_file() + assert not (foreign / "device-types" / "TestVendor" / "second.yaml").exists() + assert GitRepo(foreign).head.commit.hexsha == foreign_head + def test_pulls_when_target_is_a_git_worktree(self, tmp_path): """A worktree holds .git as a file, so testing for a .git directory would clone over it.""" source = tmp_path / "source" diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index 2208b117..351e8da9 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -159,3 +159,30 @@ def offline(): with pytest.raises(requests.exceptions.ConnectionError): call_with_retry(offline) assert time.monotonic() - started < 1 + + +def _bare_repo_calls(tree): + """Yield the line of each ``Repo(...)`` call outside ``open_repo``.""" + allowed = set() + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "open_repo": + allowed.update(id(inner) for inner in ast.walk(node)) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "Repo" + and id(node) not in allowed + ): + yield node.lineno + + +def test_core_opens_git_repositories_only_through_open_repo(): + """A bare Repo(path) inherits GIT_DIR and GIT_WORK_TREE, so git can act on another checkout.""" + offenders = [ + f"{path.relative_to(_ROOT)}:{lineno}" + for path in sorted((_ROOT / "core").rglob("*.py")) + for lineno in _bare_repo_calls(ast.parse(path.read_text(encoding="utf-8"))) + ] + + assert not offenders, "Use core.repo.open_repo instead of Repo():\n" + "\n".join(offenders) From 536f1147ad26f8c13566d6802f326b5baa7676a3 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 07:53:28 +0200 Subject: [PATCH 31/39] fix(repo): ignore every inherited git location variable in open_repo GitPython reads GIT_COMMON_DIR, GIT_OBJECT_DIRECTORY and GIT_WORK_TREE from os.environ when it builds a Repo, and git reads GIT_INDEX_FILE for each command. With these set for another checkout, open_repo() read that checkout's remotes, and a pull could write its index. open_repo() now builds the Repo without these variables and binds all five on the git wrapper to the library. FakeNetBox now seeds its id counter from the seeded records, so a seed deleted before the first create does not give its id to a new record. --- core/repo.py | 20 +++++++++++++++++--- tests/helpers.py | 5 +++-- tests/test_repo.py | 5 +++++ tests/test_suite_hygiene.py | 7 +++++-- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/core/repo.py b/core/repo.py index f8fb0790..f01a12a0 100644 --- a/core/repo.py +++ b/core/repo.py @@ -80,10 +80,24 @@ def find_class(self, module, name): _INDEX_MAX_BYTES = 10 * 1024 * 1024 # 10 MiB — DTL index files are typically <1 MiB +# Variables that point git at a repository; a hook runner exports them for its own checkout. +_GIT_LOCATION_VARS = ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_COMMON_DIR") + + def open_repo(path) -> Repo: - """Open the repository at *path* with git bound to it over an inherited GIT_DIR or GIT_WORK_TREE.""" - repo = Repo(path) - repo.git.update_environment(GIT_DIR=repo.git_dir, GIT_WORK_TREE=os.path.abspath(path)) + """Open the repository at *path* with git bound to it over inherited ``_GIT_LOCATION_VARS``.""" + inherited = {name: os.environ.pop(name) for name in _GIT_LOCATION_VARS if name in os.environ} + try: + repo = Repo(path) # GitPython reads these variables from os.environ here + finally: + os.environ.update(inherited) + repo.git.update_environment( + GIT_DIR=repo.git_dir, + GIT_WORK_TREE=os.path.abspath(path), + GIT_INDEX_FILE=os.path.join(repo.git_dir, "index"), + GIT_COMMON_DIR=repo.common_dir, + GIT_OBJECT_DIRECTORY=os.path.join(repo.common_dir, "objects"), + ) return repo diff --git a/tests/helpers.py b/tests/helpers.py index ac4ff0a5..ef3f783a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -116,7 +116,8 @@ def __init__(self, errors=None, netbox_version="4.7.0", **collections): ``/api/status/`` reports, which is how the export side decides what it may select. """ self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} - self._last_ids = {} # collection name -> highest id a create assigned + # Collection name -> highest id seeded or assigned, so a deleted id is never handed out again. + self._last_ids = {name: max([999, *(r["id"] for r in records)]) for name, records in self.collections.items()} self.errors = dict(errors or {}) self.netbox_version = netbox_version # A GET whose request target is longer than this gets 414, as from a proxy URL limit. @@ -200,7 +201,7 @@ def _create(self, name, payload): # A bulk create is atomic: one duplicate refuses the whole batch. raise _Refused(errors if isinstance(payload, list) else errors[0]) collection = self.collection(name) - # Ids stay above every seeded, stored and deleted id, as NetBox's sequences do. + # A test can also append records directly, so stored ids count too. last = max([self._last_ids.get(name, 999), *(r["id"] for r in collection)]) for record in records: last += 1 diff --git a/tests/test_repo.py b/tests/test_repo.py index fd9584f8..362ce372 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -335,14 +335,19 @@ def test_a_pull_ignores_an_inherited_git_work_tree(self, tmp_path, monkeypatch): foreign_head = self._make_source_repo(foreign).head.commit.hexsha (source / "device-types" / "TestVendor" / "second.yaml").write_text("manufacturer: TestVendor\nmodel: Second\n") self._commit(source_repo, "device-types/TestVendor/second.yaml", "second device") + foreign_index = (foreign / ".git" / "index").read_bytes() monkeypatch.setenv("GIT_DIR", str(foreign / ".git")) monkeypatch.setenv("GIT_WORK_TREE", str(foreign)) + monkeypatch.setenv("GIT_INDEX_FILE", str(foreign / ".git" / "index")) + monkeypatch.setenv("GIT_COMMON_DIR", str(foreign / ".git")) + monkeypatch.setenv("GIT_OBJECT_DIRECTORY", str(foreign / ".git" / "objects")) self._init_dtl(source, target) assert (target / "device-types" / "TestVendor" / "second.yaml").is_file() assert not (foreign / "device-types" / "TestVendor" / "second.yaml").exists() assert GitRepo(foreign).head.commit.hexsha == foreign_head + assert (foreign / ".git" / "index").read_bytes() == foreign_index def test_pulls_when_target_is_a_git_worktree(self, tmp_path): """A worktree holds .git as a file, so testing for a .git directory would clone over it.""" diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index 351e8da9..a84bdd16 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -130,9 +130,12 @@ def test_a_fake_netbox_create_never_reuses_a_stored_id(): import pynetbox from helpers import FakeNetBox - server = FakeNetBox(manufacturers=[{"id": 1000, "name": "Juniper", "slug": "juniper"}]) + server = FakeNetBox( + manufacturers=[{"id": 1000, "name": "Juniper", "slug": "juniper"}, {"id": 1001, "name": "HP", "slug": "hp"}] + ) try: api = pynetbox.api(server.url, token="test-token") + api.dcim.manufacturers.delete([1001]) first = api.dcim.manufacturers.create(name="Cisco", slug="cisco") second = api.dcim.manufacturers.create(name="Nokia", slug="nokia") api.dcim.manufacturers.delete([first.id]) @@ -140,7 +143,7 @@ def test_a_fake_netbox_create_never_reuses_a_stored_id(): finally: server.close() - ids = [1000, first.id, second.id, third.id] + ids = [1000, 1001, first.id, second.id, third.id] assert len(set(ids)) == len(ids), ids From 5931e43d3d69ed08d31e076ffd2a6594c4caf0fa Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:42:17 +0200 Subject: [PATCH 32/39] fix(slugs): find definitions that the upstream index does not list (#149) --- README.md | 4 +- core/import_run.py | 83 ++++------ core/repo.py | 215 ++++++++++++------------- pyproject.toml | 1 + tests/conftest.py | 7 + tests/test_import_run.py | 114 +++++++++++++- tests/test_nb_dt_import.py | 24 ++- tests/test_repo.py | 315 ++++++++++++++++++++++--------------- 8 files changed, 451 insertions(+), 312 deletions(-) diff --git a/README.md b/README.md index 5b004a04..4c7e6a92 100644 --- a/README.md +++ b/README.md @@ -443,7 +443,9 @@ least one of `device-types/`, `module-types/`, or `rack-types/`, and an import s error if it does not, rather than reporting an empty library as nothing to do. Nothing in an import writes to `REPO_PATH`, so a read-only mount works. `REPO_BRANCH` is -ignored in this mode, and a run that sets both says so. +ignored in this mode, and a run that sets both says so. `--slugs` parses every definition file +in this mode: the device index in `tests/` is used only when git can show which files changed +after the index was written. ```shell REPO_URL=local REPO_PATH=/srv/devicetype-library uv run nb-dt-import.py diff --git a/core/import_run.py b/core/import_run.py index c99a6ef0..f57c2318 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -29,7 +29,7 @@ class RunSelection: devices_path: str modules_path: str racks_path: str - slug_resolved: dict | None + slug_matches: dict | None @dataclass(frozen=True) @@ -618,7 +618,7 @@ def discover(self): else: vendors = all_vendors - vendors, slug_resolved = self._narrow_by_slug(vendors) + vendors, slug_matches = self._match_slugs(vendors, devices_path, modules_path, racks_path) if self.config.vendors and not vendors: raise VendorSelectionError( f"No vendors matched the combination of --vendors and --slugs: {', '.join(self.config.vendors)}" @@ -629,63 +629,46 @@ def discover(self): devices_path=devices_path, modules_path=modules_path, racks_path=racks_path, - slug_resolved=slug_resolved, + slug_matches=slug_matches, ) - def _narrow_by_slug(self, vendors): - """Narrow vendors with the repository slug index when available.""" - if not self.config.slugs: - return vendors, None + def _match_slugs(self, vendors, devices_path, modules_path, racks_path): + """Parse the types that match --slugs, and keep only the vendors that have some. - slug_resolved = self.repo.resolve_slug_files(self.config.slugs) - if slug_resolved is None: - self.reporter.verbose_log("Slug index unavailable; falling back to full file scan.") + Returns the narrowed vendors and ``{vendor_slug: (device_types, module_types, rack_types)}``. + The device index, when available, limits which device files are parsed. + """ + slugs = self.config.slugs + if not slugs: return vendors, None - matched_vendor_slugs = ( - set(slug_resolved["device_files"]) - | (slug_resolved["module_vendors"] or set()) - | (slug_resolved["rack_vendors"] or set()) - ) - if not matched_vendor_slugs: - self.reporter.verbose_log("Slug index returned no matches; falling back to full file scan.") - return vendors, None - narrowed = [vendor for vendor in vendors if vendor["slug"] in matched_vendor_slugs] - self.reporter.verbose_log( - f"Slug index resolved {sum(len(files) for files in slug_resolved['device_files'].values())} " - f"device file(s) across {len(matched_vendor_slugs)} vendor(s)." - ) - return narrowed, slug_resolved + device_files = self.repo.resolve_slug_files(slugs) + if device_files is None: + self.reporter.verbose_log("Slug index unavailable; parsing every device type file.") + + matches = {} + for vendor in vendors: + if device_files is None: + device_types = _parse_vendor_files(self.repo, devices_path, vendor["name"], slugs) + else: + files = device_files.get(vendor["slug"], []) + device_types = self.repo.parse_files(files, slugs=slugs) if files else [] + module_types = _parse_vendor_files(self.repo, modules_path, vendor["name"], slugs) + rack_types = _parse_vendor_files(self.repo, racks_path, vendor["name"], slugs) + if device_types or module_types or rack_types: + matches[vendor["slug"]] = (device_types, module_types, rack_types) + + self.reporter.verbose_log(f"--slugs matched types from {len(matches)} vendor(s).") + return [vendor for vendor in vendors if vendor["slug"] in matches], matches def plan_vendor(self, selection, vendor): """Parse source files and return an inspectable vendor plan.""" - slug_resolved = selection.slug_resolved - if slug_resolved is not None: - device_files = slug_resolved["device_files"].get(vendor["slug"], []) - device_types = self.repo.parse_files(device_files) if device_files else [] - else: - device_types = _parse_vendor_files( - self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] - ) - - module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None - if module_hint is not None and vendor["slug"] not in module_hint: - module_types = [] - else: - module_types = _parse_vendor_files( - self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] - ) - - rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None - if rack_hint is not None and vendor["slug"] not in rack_hint: - rack_types = [] + if selection.slug_matches is not None: + device_types, module_types, rack_types = selection.slug_matches[vendor["slug"]] else: - rack_types = _parse_vendor_files( - self.repo, - selection.racks_path, - vendor["name"], - self.config.slugs or [], - ) + device_types = _parse_vendor_files(self.repo, selection.devices_path, vendor["name"], []) + module_types = _parse_vendor_files(self.repo, selection.modules_path, vendor["name"], []) + rack_types = _parse_vendor_files(self.repo, selection.racks_path, vendor["name"], []) return VendorPlan( vendor=vendor, diff --git a/core/repo.py b/core/repo.py index f01a12a0..56617239 100644 --- a/core/repo.py +++ b/core/repo.py @@ -4,8 +4,9 @@ import json import os import pickle +import posixpath from collections.abc import Sequence -from glob import glob +from glob import escape, glob from re import sub as re_sub from urllib.parse import urlparse @@ -117,37 +118,39 @@ def _resolve_index_path(base_dir: str, stem: str) -> "str | None": return None -def _vendor_slugs_from_index( - index_path: "str | None", slugs_lower: list, slug_format, subdir_filter: "str | None" = None -) -> "set | None": - """Load a (model_name, vendor_dir) index and return the set of vendor slugs matching *slugs_lower*. +def _vendor_of(directory: str) -> str: + """Return the vendor folder name of a definition directory such as ``device-types/Nokia``.""" + return directory.rsplit("/", 1)[-1] - *index_path* may point at a ``.json`` (current) or ``.pickle`` (legacy) file; the loader is - chosen by extension. *subdir_filter*, if given, requires the vendor_dir to contain that - substring. Returns ``None`` when the index is unavailable (missing or unreadable), so callers - can distinguish "no matches" (empty set) from "hint unavailable" (None). + +def _reached_through_link(abs_path: str, real_root: str, relpath: str) -> bool: + """Return True when *abs_path* does not resolve to *relpath* (POSIX separators) under *real_root*.""" + return os.path.realpath(abs_path) != os.path.join(real_root, *relpath.split("/")) + + +def _paths_changed_since_index(repo_root: str, index_path: str, type_dir: str) -> "set[str] | None": + """Return the repo-relative paths under *type_dir* that differ from the commit that last wrote *index_path*. + + Returns ``None`` when the index cannot describe the checkout: *repo_root* is not a git work + tree, its history is shallow, the index is not committed, or the index differs from that commit. """ - if index_path is None: - return None + index_relpath = os.path.relpath(index_path, repo_root).replace("\\", "/") try: - entries = _safe_index_load(index_path) - except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 + git = open_repo(repo_root).git + git.update_environment(GIT_OPTIONAL_LOCKS="0") # read-only: do not refresh .git/index + # In a shallow clone, git log reports the shallow boundary, not the commit that wrote the index. + if git.rev_parse("--is-shallow-repository") != "false": + return None + commit = git.log("-1", "--format=%H", "--", index_relpath) + if not commit: + return None + # A tree diff against the working tree catches fork edits, upstream merges and uncommitted edits. + changed = git.diff("--name-only", "--no-renames", "-z", commit, "--", type_dir, index_relpath) + except (exc.InvalidGitRepositoryError, exc.NoSuchPathError, exc.GitCommandError): return None - result = set() - for model_name, vendor_dir in entries: - if subdir_filter and subdir_filter not in vendor_dir.replace("\\", "/"): - continue - if not any(s in model_name.casefold() for s in slugs_lower): - continue - vendor_name = vendor_dir.replace("\\", "/").split("/")[-1] - result.add(slug_format(vendor_name)) - return result - + paths = {path for path in changed.split("\0") if path} -def _safe_abs_path(repo_root: str, relpath: str) -> "str | None": - """Return the absolute path for *relpath* inside *repo_root*, or None if it escapes the root.""" - abs_path = os.path.normpath(os.path.join(repo_root, *relpath.replace("\\", "/").split("/"))) - return abs_path if abs_path.startswith(os.path.normpath(repo_root) + os.sep) else None + return None if index_relpath in paths else paths def _validate_index_data(data, source: str): @@ -462,33 +465,33 @@ def parse_single_file(file): `src` set to the file path. str: Error string beginning with "Error:" describing YAML parsing or other failure. """ - with open(file) as stream: - try: + try: + with open(file) as stream: data = yaml.safe_load(stream) - manufacturer = data["manufacturer"] - # Use only slug for manufacturer lookup - more resilient to case mismatches - # (e.g., RuggedCOM vs RuggedCom in upstream data) - data["manufacturer"] = {"slug": re_sub(r"\W+", "-", manufacturer.lower())} - profile = data.get("profile") - if isinstance(profile, str): - data["profile"] = {"name": profile} - elif profile is not None and not (isinstance(profile, dict) and isinstance(profile.get("name"), str)): - return f"Error: profile must be a name or a mapping with a name, not {profile!r}" - attribute_data = data.get("attribute_data") - if attribute_data is not None and not isinstance(attribute_data, dict): - return f"Error: attribute_data must be a mapping, not {attribute_data!r}" - for field in ("profile", "attribute_data"): - if not _is_strict_json(data.get(field)): - return f"Error: {field} must hold JSON values only (string keys, finite numbers): {data[field]!r}" - data["src"] = file - err = normalize_port_mappings(data) - if err: - return err - return data - except yaml.YAMLError as excep: - return f"Error: {excep}" - except Exception as e: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 - return f"Error: {e}" + manufacturer = data["manufacturer"] + # Use only slug for manufacturer lookup - more resilient to case mismatches + # (e.g., RuggedCOM vs RuggedCom in upstream data) + data["manufacturer"] = {"slug": re_sub(r"\W+", "-", manufacturer.lower())} + profile = data.get("profile") + if isinstance(profile, str): + data["profile"] = {"name": profile} + elif profile is not None and not (isinstance(profile, dict) and isinstance(profile.get("name"), str)): + return f"Error: profile must be a name or a mapping with a name, not {profile!r}" + attribute_data = data.get("attribute_data") + if attribute_data is not None and not isinstance(attribute_data, dict): + return f"Error: attribute_data must be a mapping, not {attribute_data!r}" + for field in ("profile", "attribute_data"): + if not _is_strict_json(data.get(field)): + return f"Error: {field} must hold JSON values only (string keys, finite numbers): {data[field]!r}" + data["src"] = file + err = normalize_port_mappings(data) + if err: + return err + return data + except yaml.YAMLError as excep: + return f"Error: {excep}" + except Exception as e: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 + return f"Error: {e}" class DTLRepo: @@ -688,84 +691,72 @@ def get_devices(self, base_path, vendors: Sequence[str] | None = None): if folder.casefold() != "testing": discovered_vendors.append({"name": folder, "slug": self.slug_format(folder)}) for extension in self.yaml_extensions: - files.extend(glob(os.path.join(base_path, folder, f"*.{extension}"))) + files.extend(glob(os.path.join(escape(os.path.join(base_path, folder)), f"*.{extension}"))) return files, discovered_vendors def resolve_slug_files(self, slugs): - """Use the upstream slug indexes to resolve YAML file paths for slug/model matches. + """Use the upstream device index to select the device type files that can match *slugs*. - The DTL repo ships three index files under ``tests/``. Upstream replaced the original - ``known-*.pickle`` files with ``known-*.json`` (GHSA-492p-5wp7-2w7c); both carry the same - data shape, and we prefer JSON, falling back to pickle for older pinned checkouts: + The DTL repo ships ``tests/known-slugs.json`` (``known-slugs.pickle`` in older checkouts, + GHSA-492p-5wp7-2w7c): a list of ``(manufacturer_prefixed_slug, relpath)`` for device + types, where ``relpath`` is relative to the repo root, e.g. ``device-types/Nokia/7750-SR-7s.yaml``. + Matching uses the same case-insensitive substring check as :meth:`parse_files`. - * ``known-slugs.json`` — list of ``(manufacturer_prefixed_slug, relpath)`` for - device types. ``relpath`` is relative to the repo root, e.g. - ``device-types/Nokia/7750-SR-7s.yaml``. - * ``known-modules.json`` — list of ``(model_name, vendor_dir)`` for module - types. Only the vendor directory is stored, not the file name. - * ``known-racks.json`` — same format as known-modules. + The index comes from upstream master, so a fork or branch can hold files it does not + list, can lack files it lists, and can change files it lists. The files on disk are the + source of truth: an indexed file that is gone is dropped, and a file that the index does + not list, or that differs from the commit that last wrote the index, is always selected. - Matching uses a **case-insensitive substring** check identical to the runtime - :meth:`parse_files` filter so that partial slug/model searches work the same way. + The module and rack indexes are not used: they record file stems, not models, so they + cannot tell which files hold a matching model. Args: slugs (list[str]): User-supplied slug/model substrings (``--slugs``). Returns: - dict or None: ``None`` when the device index is unavailable (caller falls back - to the normal glob path). Otherwise a dict with the keys: - - ``"device_files"`` - ``{vendor_slug: [abs_path, ...]}`` for devices resolved via the index. - ``"module_vendors"`` - ``{vendor_slug}`` — set of vendor slugs that may contain matching module - types, or ``None`` when the module index was unavailable (caller should - fall back to full glob+parse instead of skipping). - ``"rack_vendors"`` - ``{vendor_slug}`` — same for rack types; ``None`` means unavailable. + dict or None: ``{vendor_slug: [abs_path, ...]}``, to parse with *slugs*, or ``None`` + when the device index is missing, unreadable, or cannot be compared with the checkout. + ``REPO_URL=local`` always gives ``None``, because that mode runs no git command. """ + if is_local_repo_url(self.url): + return None repo_root = self.get_absolute_path() - tests_dir = os.path.join(repo_root, "tests") - device_index = _resolve_index_path(tests_dir, "known-slugs") - module_index = _resolve_index_path(tests_dir, "known-modules") - rack_index = _resolve_index_path(tests_dir, "known-racks") - + device_index = _resolve_index_path(os.path.join(repo_root, "tests"), "known-slugs") if device_index is None: return None - - slugs_lower = [s.casefold() for s in slugs] - - # --- device types -------------------------------------------------- - device_files: dict = {} # vendor_slug -> [abs_path] + changed = _paths_changed_since_index(repo_root, device_index, "device-types") + if changed is None: + return None try: known_slugs = _safe_index_load(device_index) - except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 + except Exception: # an unreadable index is skipped, not fatal # noqa: BLE001 return None + slug_by_relpath = {relpath.replace("\\", "/"): entry_slug for entry_slug, relpath in known_slugs} - for entry_slug, relpath in known_slugs: - if not any(s in entry_slug.casefold() for s in slugs_lower): - continue - parts = relpath.replace("\\", "/").split("/") - if len(parts) < 3: - continue - vendor_name = parts[1] - vendor_slug = self.slug_format(vendor_name) - abs_path = _safe_abs_path(repo_root, relpath) - if abs_path is None: - continue - device_files.setdefault(vendor_slug, []).append(abs_path) - - # --- module types -------------------------------------------------- - module_vendors = _vendor_slugs_from_index(module_index, slugs_lower, self.slug_format) - - # --- rack types ---------------------------------------------------- - rack_vendors = _vendor_slugs_from_index(rack_index, slugs_lower, self.slug_format, subdir_filter="rack-types") - - return { - "device_files": device_files, - "module_vendors": module_vendors, - "rack_vendors": rack_vendors, - } + slugs_lower = [s.casefold() for s in slugs] + real_root = os.path.realpath(repo_root) + device_files: dict = {} # vendor_slug -> [abs_path] + for relpath, abs_path in self._definition_files(self.get_devices_path()): + entry_slug = slug_by_relpath.get(relpath) + # An unindexed or changed file can hold any slug, so the caller's parse_files filter decides. + # A link target can change while the link blob does not, so a file reached through a link counts as changed. + if ( + entry_slug is None + or relpath in changed + or _reached_through_link(abs_path, real_root, relpath) + or any(s in entry_slug.casefold() for s in slugs_lower) + ): + vendor_slug = self.slug_format(_vendor_of(posixpath.dirname(relpath))) + device_files.setdefault(vendor_slug, []).append(abs_path) + return device_files + + def _definition_files(self, type_root): + """Return ``(posix relpath, abs_path)`` for each definition file on disk under *type_root*.""" + if not os.path.isdir(type_root): + return [] + files, _ = self.get_devices(type_root) + repo_root = self.get_absolute_path() + return [(os.path.relpath(path, repo_root).replace("\\", "/"), path) for path in files] def discover_vendors(self, devices_path, modules_path, racks_path): """Discover all vendor directories across device-types/, module-types/, and rack-types/. diff --git a/pyproject.toml b/pyproject.toml index c1cad3cc..0ad9f2de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ norecursedirs = ["tests/integration"] markers = [ "integration: end-to-end tests that require a live NetBox instance (set NETBOX_URL + NETBOX_TOKEN)", "real_http: drives a local HTTP server, so the GraphQL session mock is not applied", + "real_git: runs real git commands, so the core.repo.Repo mock is not applied", ] [tool.coverage.run] diff --git a/tests/conftest.py b/tests/conftest.py index 19d929c5..dc744fef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,13 @@ def mock_git_repo(request): if request.node.get_closest_marker("integration"): yield None return + if request.node.get_closest_marker("real_git"): + # A hook runner exports GIT_DIR and friends for the outer repo. + monkeypatch = request.getfixturevalue("monkeypatch") + for var in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_COMMON_DIR"): + monkeypatch.delenv(var, raising=False) + yield None + return with patch("core.repo.Repo") as mock_repo: mock_remote = MagicMock() mock_remote.url = "https://example.com/repo.git" diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 26b09352..86f768a7 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -3,8 +3,11 @@ from collections import Counter from contextlib import contextmanager from types import SimpleNamespace +from typing import ClassVar import pytest +from git import Actor +from git import Repo as GitRepo from helpers import recording_handle as _recording_handle from core.errors import VendorSelectionError @@ -238,11 +241,11 @@ def test_discover_raises_a_typed_fatal_error_when_no_vendor_matches(make_config, def test_discover_raises_a_typed_fatal_error_when_slug_selection_removes_the_vendor(make_config, tmp_path): class SlugResolvedRepository(_RepositoryBoundary): def resolve_slug_files(self, _slugs): - return { - "device_files": {"other-vendor": []}, - "module_vendors": set(), - "rack_vendors": set(), - } + return {"other-vendor": []} + + @staticmethod + def parse_files(_files, slugs=None): + return [] repo = SlugResolvedRepository(tmp_path) run = ImportRun( @@ -323,6 +326,107 @@ def test_one_type_root_plans_without_reading_the_absent_ones(self, present, make assert sum(parsed.values()) == 1, parsed +@pytest.mark.real_git +class TestSlugSelectionWithAStaleIndex: + """The upstream index lags a fork or branch, so --slugs must also find what only the disk holds.""" + + INDEX: ClassVar[dict] = { + "known-slugs": [["acme-old-box", "device-types/Acme/old-box.yaml"]], + "known-modules": [["old-card", "module-types/Acme"]], + "known-racks": [["old-rack", "rack-types/Acme"]], + } + + @staticmethod + def _write(root, relpath, model, slug=None): + path = root / relpath + path.parent.mkdir(parents=True, exist_ok=True) + manufacturer = relpath.split("/")[1] + path.write_text(f"manufacturer: {manufacturer}\nmodel: {model}\n" + (f"slug: {slug}\n" if slug else "")) + + def _plan(self, tmp_path, make_config, index, slugs, vendors=(), after_clone=None): + """Commit *tmp_path* with *index*, clone it, run *after_clone* on the clone, and return the planned models.""" + import json + + (tmp_path / "tests").mkdir() + for stem, entries in index.items(): + (tmp_path / "tests" / f"{stem}.json").write_text(json.dumps(entries)) + git_repo = GitRepo.init(tmp_path, initial_branch="master") + git_repo.git.add("-A") + git_repo.index.commit("library", author=Actor("t", "t@example.com"), committer=Actor("t", "t@example.com")) + clone = tmp_path.parent / f"{tmp_path.name}-clone" + config = make_config( + repo_url=f"file://{tmp_path}", repo_path=str(clone), repo_branch="master", slugs=slugs, vendors=vendors + ) + repo = DTLRepo(config, LogHandler(False)) + if after_clone is not None: + after_clone(clone) + assert repo.resolve_slug_files(slugs) is not None, "the test must reach the device index" + run = ImportRun(config, repo, _NetBoxBoundary(), LogHandler(False), _ProgressFactory()) + selection = run.discover() + models = set() + for vendor in selection.vendors: + plan = run.plan_vendor(selection, vendor) + models |= {t["model"] for t in [*plan.device_types, *plan.module_types, *plan.rack_types]} + return models + + def test_definitions_missing_from_the_index_are_found(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + self._write(tmp_path, "device-types/Acme/new-box.yaml", "New Box", "acme-new-box") + self._write(tmp_path, "device-types/Acme/other.yaml", "Other", "acme-other") + self._write(tmp_path, "module-types/Acme/old-card.yaml", "Old Card") + self._write(tmp_path, "module-types/Beta/new-card.yaml", "New Card") + self._write(tmp_path, "rack-types/Acme/old-rack.yaml", "Old Rack") + self._write(tmp_path, "rack-types/Gamma/new-rack.yaml", "New Rack") + + models = self._plan(tmp_path, make_config, self.INDEX, ("box", "card", "rack")) + + assert models == {"Old Box", "New Box", "Old Card", "New Card", "Old Rack", "New Rack"} + + def test_a_module_is_found_when_its_file_stem_does_not_hold_the_model(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + self._write(tmp_path, "module-types/Acme/Patch Antenna.yaml", "Patch Antenna") + self._write(tmp_path, "module-types/Datto/AP-ANT-1.yaml", "Patch Antenna") + index = { + **self.INDEX, + "known-modules": [["Patch Antenna", "module-types/Acme"], ["AP-ANT-1", "module-types/Datto"]], + } + + assert self._plan(tmp_path, make_config, index, ("Patch Antenna",), vendors=("datto",)) == {"Patch Antenna"} + + def test_an_indexed_file_whose_slug_changed_after_the_index_is_found(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + self._write(tmp_path, "device-types/Acme/other.yaml", "Other", "acme-other") + + def rename(clone): + self._write(clone, "device-types/Acme/old-box.yaml", "Renamed Box", "acme-renamed-box") + + models = self._plan(tmp_path, make_config, self.INDEX, ("renamed",), vendors=("acme",), after_clone=rename) + + assert models == {"Renamed Box"} + + def test_an_indexed_file_that_is_gone_from_disk_is_skipped(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + index = { + **self.INDEX, + "known-slugs": [*self.INDEX["known-slugs"], ["acme-gone-box", "device-types/Acme/gone.yaml"]], + } + + assert self._plan(tmp_path, make_config, index, ("box",)) == {"Old Box"} + + def test_a_yml_file_is_not_taken_for_the_indexed_yaml_file_of_the_same_name(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + self._write(tmp_path, "device-types/Acme/old-box.yml", "Yml Box", "acme-yml-box") + self._write(tmp_path, "device-types/Acme/new-box.yaml", "New Box", "acme-new-box") + + assert self._plan(tmp_path, make_config, self.INDEX, ("yml",)) == {"Yml Box"} + + def test_a_broken_definition_link_is_skipped(self, tmp_path, make_config): + self._write(tmp_path, "device-types/Acme/old-box.yaml", "Old Box", "acme-old-box") + (tmp_path / "device-types/Acme/dangling.yaml").symlink_to(tmp_path / "device-types/Acme/missing.yaml") + + assert self._plan(tmp_path, make_config, self.INDEX, ("box",)) == {"Old Box"} + + class _ModuleNetBox: """Report one new module type so the banner is emitted.""" diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index bb3af27d..1740514a 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -844,7 +844,9 @@ def test_vendors_and_slugs_flags_log_lines(self, nb_dt_import, capsys): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_repo = _make_mock_repo() + mock_repo = _make_mock_repo( + device_types=[{"manufacturer": {"slug": "cisco"}, "model": "WS-C3750", "slug": "cisco-ws-c3750"}] + ) mock_repo.discover_vendors.return_value = [{"name": "Cisco", "slug": "cisco"}] MockRepo.return_value = mock_repo MockNetBox.return_value = _make_mock_netbox() @@ -1809,11 +1811,7 @@ def test_main_clears_environment_slugs_before_running_the_export(self, nb_dt_imp def test_main_uses_slug_fast_path_device_files(self, nb_dt_import): mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Cisco", "slug": "cisco"}] - mock_repo.resolve_slug_files.return_value = { - "device_files": {"cisco": ["resolved.yaml"]}, - "module_vendors": set(), - "rack_vendors": set(), - } + mock_repo.resolve_slug_files.return_value = {"cisco": ["resolved.yaml"]} mock_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["resolved.yaml"] else [] ) @@ -1951,8 +1949,8 @@ def test_build_argument_parser_sets_expected_defaults_and_flags(self, nb_dt_impo assert parsed.export_diff_dir == "exports/" assert parsed.force_export_overwrite is True - def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_config): - """Execution applies a resolved plan and advances past an empty plan.""" + def test_import_run_processes_slug_fast_path_and_drops_unmatched_vendor(self, make_config): + """Execution applies the matched vendor's plan, and discovery drops a vendor with no match.""" config = make_config(slugs=("x",)) handle = MagicMock() progress = MagicMock() @@ -1971,11 +1969,7 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c else [] ) netbox = _make_mock_netbox() - slug_resolved = { - "device_files": {"empty": [], "cisco": ["resolved.yaml"]}, - "module_vendors": {"cisco"}, - "rack_vendors": set(), - } + slug_resolved = {"empty": [], "cisco": ["resolved.yaml"]} vendors = [{"slug": "empty", "name": "Empty"}, {"slug": "cisco", "name": "Cisco"}] dtl_repo.discover_vendors.return_value = vendors dtl_repo.resolve_slug_files.return_value = slug_resolved @@ -2012,8 +2006,8 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c {"manufacturer": {"slug": "cisco"}, "model": "M", "slug": "m"} ] mock_process_rack_types.assert_called_once() - assert mock_process_rack_types.call_args.args[4] == [] # cisco not in rack_vendors → empty - assert progress.advance.call_args_list == [((7,),), ((7,),)] + assert mock_process_rack_types.call_args.args[4] == [] # no cisco rack type matches the slug + assert progress.advance.call_args_list == [((7,),)] mock_finalize.assert_called_once_with(progress, {}) assert any(call.args[0] == ["resolved.yaml"] for call in dtl_repo.parse_files.call_args_list) diff --git a/tests/test_repo.py b/tests/test_repo.py index 362ce372..c9b621f8 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -17,6 +17,7 @@ GitInvalidRepositoryError, InvalidGitURLError, InvalidRepoPathError, + _reached_through_link, _resolve_index_path, _safe_index_load, _safe_json_load, @@ -250,23 +251,13 @@ def test_the_branch_is_left_unused(self, tmp_path, mock_git_repo): mock_git_repo.assert_not_called() +@pytest.mark.real_git class TestDTLRepoRealGit: """Clone/pull branching driven against a real local Git repository (no git mocks).""" # Pinned so the fixture never depends on ambient git config or on getpass.getuser(). ACTOR = Actor("DTL test", "dtl-test@example.invalid") - @pytest.fixture(autouse=True) - def mock_git_repo(self): - """Override the global autouse git mock so these tests exercise real git.""" - return - - @pytest.fixture(autouse=True) - def clear_ambient_git_env(self, monkeypatch): - """Drop git's per-command variables: a hook runner exports GIT_DIR for the outer repo.""" - for var in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_COMMON_DIR"): - monkeypatch.delenv(var, raising=False) - @classmethod def _commit(cls, repo, path, message): """Stage *path* in *repo* and commit it under the fixed test identity.""" @@ -1546,6 +1537,23 @@ def __reduce__(self): _safe_pickle_load(str(path)) +def _write_definitions(root, *relpaths): + """Create the definition files an index entry points at, so the disk agrees with it.""" + for relpath in relpaths: + path = root / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("manufacturer: Test\nmodel: Test\n") + + +def _commit_library(root): + """Commit everything under *root*, so the device index describes the checkout.""" + git_repo = GitRepo.init(root, initial_branch="master") + git_repo.git.add("-A") + git_repo.index.commit("library", author=Actor("t", "t@example.com"), committer=Actor("t", "t@example.com")) + return git_repo + + +@pytest.mark.real_git class TestResolveSlugFiles: """Tests for the pickle-based slug fast path.""" @@ -1568,6 +1576,7 @@ def _make_repo(self): def test_returns_none_when_pickle_missing(self, tmp_path): """Returns None gracefully when the device pickle doesn't exist.""" + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1583,10 +1592,9 @@ def test_returns_device_files_for_matching_slug(self, tmp_path): tests_dir.mkdir() known = {("nokia-7750-sr-7s", "device-types/Nokia/7750-SR-7s.yaml")} (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(known)) - # Create an empty module/rack pickle so those code paths don't crash - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) + _write_definitions(tmp_path, "device-types/Nokia/7750-SR-7s.yaml") + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1594,9 +1602,9 @@ def test_returns_device_files_for_matching_slug(self, tmp_path): result = repo.resolve_slug_files(["nokia-7750-sr-7s"]) assert result is not None - assert "nokia" in result["device_files"] + assert "nokia" in result expected_path = str(tmp_path / "device-types" / "Nokia" / "7750-SR-7s.yaml") - assert expected_path in result["device_files"]["nokia"] + assert expected_path in result["nokia"] def test_substring_match(self, tmp_path): """Partial slug matches (substring) are found.""" @@ -1610,9 +1618,9 @@ def test_substring_match(self, tmp_path): ("cisco-catalyst-9200", "device-types/Cisco/Catalyst-9200.yaml"), } (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(known)) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) + _write_definitions(tmp_path, *(relpath for _, relpath in known)) + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1620,21 +1628,20 @@ def test_substring_match(self, tmp_path): result = repo.resolve_slug_files(["7750-sr"]) assert result is not None - assert "nokia" in result["device_files"] - assert len(result["device_files"]["nokia"]) == 2 - assert "cisco" not in result["device_files"] + assert "nokia" in result + assert len(result["nokia"]) == 2 + assert "cisco" not in result def test_no_match_returns_empty_dict(self, tmp_path): - """No matching slug returns empty device_files (not None).""" + """No matching slug returns an empty dict (not None).""" import pickle tests_dir = tmp_path / "tests" tests_dir.mkdir() known = {("cisco-catalyst-9200", "device-types/Cisco/Catalyst-9200.yaml")} (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(known)) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1642,14 +1649,14 @@ def test_no_match_returns_empty_dict(self, tmp_path): result = repo.resolve_slug_files(["nokia-7750-sr-7s"]) assert result is not None - assert result["device_files"] == {} - assert result["module_vendors"] == set() + assert result == {} def test_corrupted_device_pickle_returns_none(self, tmp_path): tests_dir = tmp_path / "tests" tests_dir.mkdir() (tests_dir / "known-slugs.pickle").write_bytes(b"not-a-pickle") + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1662,91 +1669,15 @@ def test_short_device_relpath_is_skipped(self, tmp_path): tests_dir = tmp_path / "tests" tests_dir.mkdir() (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps({("nokia-7750-sr-7s", "device-types/Nokia.yaml")})) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" result = repo.resolve_slug_files(["7750-sr"]) - assert result["device_files"] == {} - - def test_corrupted_module_pickle_is_ignored(self, tmp_path): - import pickle - - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-modules.pickle").write_bytes(b"bad-module-pickle") - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) - - repo = self._make_repo() - repo.repo_path = str(tmp_path) - repo.cwd = "" - - result = repo.resolve_slug_files(["module"]) - - assert result["module_vendors"] is None - - def test_matching_module_pickle_adds_vendor_slug(self, tmp_path): - import pickle - - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps({("7750 line card", "module-types/Nokia")})) - (tests_dir / "known-racks.pickle").write_bytes(pickle.dumps(set())) - - repo = self._make_repo() - repo.repo_path = str(tmp_path) - repo.cwd = "" - - result = repo.resolve_slug_files(["7750"]) - - assert result["module_vendors"] == {"nokia"} - - def test_corrupted_rack_pickle_is_ignored(self, tmp_path): - import pickle - - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes(b"bad-rack-pickle") - - repo = self._make_repo() - repo.repo_path = str(tmp_path) - repo.cwd = "" - - result = repo.resolve_slug_files(["rack"]) - - assert result["rack_vendors"] is None - - def test_rack_pickle_only_uses_rack_type_entries(self, tmp_path): - import pickle - - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "known-slugs.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-modules.pickle").write_bytes(pickle.dumps(set())) - (tests_dir / "known-racks.pickle").write_bytes( - pickle.dumps( - { - ("42U Rack", "rack-types/APC"), - ("Not A Rack", "module-types/Nokia"), - } - ) - ) - - repo = self._make_repo() - repo.repo_path = str(tmp_path) - repo.cwd = "" - - result = repo.resolve_slug_files(["rack"]) - - assert result["rack_vendors"] == {"apc"} + assert result == {} class TestIndexLoaders: @@ -1787,6 +1718,7 @@ def test_safe_index_load_dispatches_on_extension(self, tmp_path): assert _safe_index_load(str(pickle_path)) == {("p", "q")} +@pytest.mark.real_git class TestResolveSlugFilesJson: """Tests for the JSON slug fast path (current upstream format).""" @@ -1802,9 +1734,9 @@ def test_returns_device_files_for_matching_slug(self, tmp_path): tests_dir = tmp_path / "tests" tests_dir.mkdir() self._write_json(tests_dir, "known-slugs", [("nokia-7750-sr-7s", "device-types/Nokia/7750-SR-7s.yaml")]) - self._write_json(tests_dir, "known-modules", []) - self._write_json(tests_dir, "known-racks", []) + _write_definitions(tmp_path, "device-types/Nokia/7750-SR-7s.yaml") + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1813,22 +1745,7 @@ def test_returns_device_files_for_matching_slug(self, tmp_path): assert result is not None expected_path = str(tmp_path / "device-types" / "Nokia" / "7750-SR-7s.yaml") - assert expected_path in result["device_files"]["nokia"] - - def test_matching_module_json_adds_vendor_slug(self, tmp_path): - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - self._write_json(tests_dir, "known-slugs", []) - self._write_json(tests_dir, "known-modules", [("7750 line card", "module-types/Nokia")]) - self._write_json(tests_dir, "known-racks", []) - - repo = self._make_repo() - repo.repo_path = str(tmp_path) - repo.cwd = "" - - result = repo.resolve_slug_files(["7750"]) - - assert result["module_vendors"] == {"nokia"} + assert expected_path in result["nokia"] def test_json_preferred_over_pickle(self, tmp_path): """When both formats exist, JSON wins — pickle here would yield a different match.""" @@ -1838,25 +1755,24 @@ def test_json_preferred_over_pickle(self, tmp_path): tests_dir.mkdir() self._write_json(tests_dir, "known-slugs", [("nokia-7750-sr-7s", "device-types/Nokia/7750-SR-7s.yaml")]) (tests_dir / "known-slugs.pickle").write_bytes( - pickle.dumps({("cisco-catalyst-9200", "device-types/Cisco/Catalyst-9200.yaml")}) + pickle.dumps({("other-slug", "device-types/Nokia/7750-SR-7s.yaml")}) ) - self._write_json(tests_dir, "known-modules", []) - self._write_json(tests_dir, "known-racks", []) + _write_definitions(tmp_path, "device-types/Nokia/7750-SR-7s.yaml") + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" - result = repo.resolve_slug_files(["nokia"]) - - assert "nokia" in result["device_files"] - assert "cisco" not in result["device_files"] + assert "nokia" in repo.resolve_slug_files(["nokia"]) + assert repo.resolve_slug_files(["other-slug"]) == {} def test_corrupted_device_json_returns_none(self, tmp_path): tests_dir = tmp_path / "tests" tests_dir.mkdir() (tests_dir / "known-slugs.json").write_text("{not valid json") + _commit_library(tmp_path) repo = self._make_repo() repo.repo_path = str(tmp_path) repo.cwd = "" @@ -1864,6 +1780,147 @@ def test_corrupted_device_json_returns_none(self, tmp_path): assert repo.resolve_slug_files(["nokia"]) is None +@pytest.mark.real_git +class TestDeviceIndexAgainstTheCheckout: + """The device index describes the commit that last wrote it, so later changes must be parsed.""" + + def _library(self, root): + import json + + (root / "tests").mkdir(parents=True) + (root / "tests" / "known-slugs.json").write_text( + json.dumps( + [["acme-old-box", "device-types/Acme/old-box.yaml"], ["acme-other", "device-types/Acme/other.yaml"]] + ) + ) + for name, slug in (("old-box", "acme-old-box"), ("other", "acme-other")): + path = root / "device-types" / "Acme" / f"{name}.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + continue + path.write_text(f"manufacturer: Acme\nmodel: {name}\nslug: {slug}\n") + return _commit_library(root) + + def _resolve(self, root, slugs): + repo = TestResolveSlugFiles._make_repo(self) + repo.repo_path = str(root) + repo.cwd = "" + return repo.resolve_slug_files(slugs) + + def test_an_unchanged_indexed_file_that_does_not_match_is_not_selected(self, tmp_path): + self._library(tmp_path) + + assert self._resolve(tmp_path, ["old-box"]) == {"acme": [str(tmp_path / "device-types/Acme/old-box.yaml")]} + + def test_an_uncommitted_slug_change_is_selected(self, tmp_path): + self._library(tmp_path) + (tmp_path / "device-types/Acme/old-box.yaml").write_text("manufacturer: Acme\nmodel: Box\nslug: acme-new\n") + + assert self._resolve(tmp_path, ["acme-new"]) == {"acme": [str(tmp_path / "device-types/Acme/old-box.yaml")]} + + def test_an_inherited_git_work_tree_does_not_hide_a_change(self, tmp_path, monkeypatch): + """A hook runner can export GIT_DIR and GIT_WORK_TREE for another checkout.""" + target, foreign = tmp_path / "target", tmp_path / "foreign" + self._library(target) + self._library(foreign) + (target / "device-types/Acme/old-box.yaml").write_text("manufacturer: Acme\nmodel: Box\nslug: acme-new\n") + monkeypatch.setenv("GIT_DIR", str(foreign / ".git")) + monkeypatch.setenv("GIT_WORK_TREE", str(foreign)) + + assert self._resolve(target, ["acme-new"]) == {"acme": [str(target / "device-types/Acme/old-box.yaml")]} + + def test_a_slug_change_committed_after_the_index_is_selected(self, tmp_path): + git_repo = self._library(tmp_path) + (tmp_path / "device-types/Acme/other.yaml").write_text("manufacturer: Acme\nmodel: Other\nslug: acme-new\n") + git_repo.index.add(["device-types/Acme/other.yaml"]) + git_repo.index.commit("fork edit", author=Actor("t", "t@example.com"), committer=Actor("t", "t@example.com")) + + assert self._resolve(tmp_path, ["acme-new"]) == {"acme": [str(tmp_path / "device-types/Acme/other.yaml")]} + + def test_an_index_that_differs_from_its_commit_is_not_used(self, tmp_path): + self._library(tmp_path) + (tmp_path / "tests" / "known-slugs.json").write_text("[]") + + assert self._resolve(tmp_path, ["old-box"]) is None + + def test_a_checkout_that_is_not_a_git_work_tree_is_not_compared(self, tmp_path): + self._library(tmp_path) + import shutil + + shutil.rmtree(tmp_path / ".git") + + assert self._resolve(tmp_path, ["old-box"]) is None + + def test_a_shallow_clone_is_not_compared(self, tmp_path): + self._library(tmp_path / "source") + GitRepo.clone_from(f"file://{tmp_path / 'source'}", tmp_path / "shallow", depth=1) + + assert self._resolve(tmp_path / "shallow", ["old-box"]) is None + + def test_a_symlinked_definition_is_always_selected(self, tmp_path): + target = tmp_path / "shared" / "other.yaml" + target.parent.mkdir() + target.write_text("manufacturer: Acme\nmodel: Other\nslug: acme-other\n") + (tmp_path / "device-types/Acme").mkdir(parents=True) + (tmp_path / "device-types/Acme/other.yaml").symlink_to("../../shared/other.yaml") + self._library(tmp_path) + target.write_text("manufacturer: Acme\nmodel: Other\nslug: acme-new\n") + + assert self._resolve(tmp_path, ["acme-new"]) == {"acme": [str(tmp_path / "device-types/Acme/other.yaml")]} + + def test_a_definition_under_a_symlinked_directory_is_always_selected(self, tmp_path): + shared = tmp_path / "shared" / "Acme" + shared.mkdir(parents=True) + (tmp_path / "device-types").mkdir() + (tmp_path / "device-types/Acme").symlink_to("../shared/Acme") + self._library(tmp_path) + (shared / "other.yaml").write_text("manufacturer: Acme\nmodel: Other\nslug: acme-new\n") + + selected = self._resolve(tmp_path, ["acme-new"])["acme"] + assert sorted(selected) == [str(tmp_path / f"device-types/Acme/{n}.yaml") for n in ("old-box", "other")] + + def test_a_local_library_runs_no_git_and_does_not_use_the_index(self, tmp_path): + self._library(tmp_path) + repo = _dtl_repo(MagicMock(repo_url="local", repo_branch="master"), str(tmp_path), LogHandler(False)) + + with patch("core.repo.Repo", side_effect=AssertionError("git must not run")): + assert repo.resolve_slug_files(["old-box"]) is None + + def test_a_repo_path_with_glob_characters_lists_its_files(self, tmp_path): + root = tmp_path / "lib[local]" + self._library(root) + + assert self._resolve(root, ["old-box"]) == {"acme": [str(root / "device-types/Acme/old-box.yaml")]} + + +class TestReachedThroughLink: + """The link check compares native paths, so a POSIX index relpath matches on Windows too.""" + + def test_a_plain_windows_path_is_not_a_link(self, monkeypatch): + import ntpath + + monkeypatch.setattr(os, "path", ntpath) + abs_path = r"C:\dtl\device-types\Acme\other.yaml" + + assert not _reached_through_link(abs_path, r"C:\dtl", "device-types/Acme/other.yaml") + + def test_a_windows_path_that_resolves_elsewhere_is_a_link(self, monkeypatch): + import ntpath + + monkeypatch.setattr(os, "path", ntpath) + + assert _reached_through_link(r"C:\shared\other.yaml", r"C:\dtl", "device-types/Acme/other.yaml") + + def test_a_windows_path_that_differs_only_in_case_is_a_link(self, monkeypatch): + """A case-sensitive directory can hold both names, so only an exact match is safe to trust.""" + import ntpath + + monkeypatch.setattr(os, "path", ntpath) + abs_path = r"C:\dtl\DEVICE-TYPES\Acme\other.yaml" + + assert _reached_through_link(abs_path, r"C:\dtl", "device-types/Acme/other.yaml") + + class TestAnExplicitlyEmptyStanza: """`port-mappings: []` is an author saying "none", which is not the same as saying nothing.""" From 20c2a337db3ea7374e92382da9a5fa61db20103d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 10:13:20 +0200 Subject: [PATCH 33/39] fix(export): fill default port-mapping positions before the compare The serializer always writes front_port_position and rear_port_position. The importer reads an omitted position as 1. The compare used exact equality, so a stanza entry without positions read as differing on every run, and its sort key put an omitted position apart from 1. --- core/export.py | 1 + tests/test_exporter.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/core/export.py b/core/export.py index d08b2d4c..53a91326 100644 --- a/core/export.py +++ b/core/export.py @@ -212,6 +212,7 @@ def _sort_port_mappings(d: dict) -> dict: mappings = d.get("port-mappings") if not isinstance(mappings, list) or not all(isinstance(mapping, dict) for mapping in mappings): return d + mappings = [{"front_port_position": 1, "rear_port_position": 1, **mapping} for mapping in mappings] fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 7bfa5bd1..45584624 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -297,6 +297,46 @@ def test_a_default_positions_does_not_make_every_front_port_differ(self): assert _repo_supersedes(repo, nb), "an omitted positions is the default, not a difference" + def test_default_port_mapping_positions_do_not_trigger_export(self): + """A stanza entry that omits a mapping position means the default 1, not a difference.""" + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = { + "manufacturer": "Acme", + "model": "Panel", + "slug": "acme-panel", + "u_height": 1, + "is_full_depth": False, + "front-ports": [{"name": "FP1", "type": "lc-upc", "positions": 2}], + "port-mappings": [ + {"front_port": "FP1", "rear_port": "RP1"}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2"}, + ], + } + record = DotDict( + id=1, manufacturer=DotDict(name="Acme"), model="Panel", slug="acme-panel", u_height=1, is_full_depth=False + ) + port = DotDict( + name="FP1", + type="lc-upc", + positions=2, + mappings=[ + DotDict(rear_port=DotDict(name="RP2"), front_port_position=2, rear_port_position=1), + DotDict(rear_port=DotDict(name="RP1"), front_port_position=1, rear_port_position=1), + ], + ) + serialized = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + + assert serialized["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 1}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2", "rear_port_position": 1}, + ] + assert _repo_supersedes(repo, serialized), "omitted mapping positions must use the default before sorting" + + repo["port-mappings"][0]["rear_port_position"] = 2 + assert not _repo_supersedes(repo, serialized), "a real mapping position change must trigger export" + def test_a_non_default_positions_still_differs(self): """Only the default may be treated as absent, or a real change would be suppressed.""" repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} From a9d718897bbeba742c4663a6a08f93830c5ad503 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 10:14:28 +0200 Subject: [PATCH 34/39] refactor(module-types): compare type relations through _relation_change _type_relation_changes had its own copy of the relation compare. The copies had diverged: only the module-type copy canonicalized the NetBox owner slug, and only the component copy logged a malformed declaration. Module types now call _relation_change, which canonicalizes the owner slug for both callers and names a module type by its model in the log. --- core/change_detector.py | 7 ++-- core/netbox_api.py | 53 +++++------------------------- tests/test_module_bay_type_sync.py | 42 ++++++++++++++++++++--- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 57ca4d3f..cdbc63c4 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -73,7 +73,8 @@ def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=No if not _is_relation_list(declared): if handle is not None: handle.log( - f"Ignored {prop} on {yaml_comp.get('name', 'Unknown')!r}: expected a list of names, got {declared!r}" + f"Ignored {prop} on {yaml_comp.get('name') or yaml_comp.get('model', 'Unknown')!r}: " + f"expected a list of names, got {declared!r}" ) return None netbox_value = getattr(netbox_comp, prop, _MISSING) @@ -113,6 +114,8 @@ def _relation_identities(value): A read path that returns only ``id`` and ``name`` cannot answer the scope question, so the caller falls back to comparing names rather than inventing an identity. """ + from core.module_bay_types import manufacturer_slug + identities = set() for item in value or []: slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) @@ -120,7 +123,7 @@ def _relation_identities(value): owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) if not slug or not owner: return None - identities.add((owner, slug)) + identities.add((manufacturer_slug(owner), slug)) return frozenset(identities) diff --git a/core/netbox_api.py b/core/netbox_api.py index 651ce08f..1c02c2c1 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -16,7 +16,7 @@ import requests from pynetbox.core.query import Request -from core.change_detector import ChangeDetector, ChangeType +from core.change_detector import ChangeDetector, ChangeType, _relation_change from core.compat import parse_netbox_version, supports_module_bay_types from core.component_cache import ComponentCache from core.component_registry import ( @@ -58,31 +58,6 @@ def __init__(self, ignore_ssl_errors: bool, cause=None): MINIMUM_NETBOX_VERSION = (4, 3) -def _relation_identities_differ(catalog, manufacturer, declared, related, *, names_differ): - """Return True when the assigned objects are not the ones the references resolve to. - - Comparing names alone reports equality when NetBox holds a same-named class from - another manufacturer's scope, so the identity is compared where both sides can supply - one. Where they cannot, the name comparison the caller already made stands. - """ - from core.module_bay_types import ModuleBayTypeError, manufacturer_slug - - if not catalog or not manufacturer: - return names_differ - current = set() - for item in related: - slug = getattr(item, "slug", None) - owner = getattr(getattr(item, "manufacturer", None), "slug", None) - if not slug or not owner: - return names_differ - current.add((manufacturer_slug(owner), slug)) - try: - return catalog.identities_for(manufacturer, declared) != frozenset(current) - except ModuleBayTypeError: - # A matching name must not make an unresolvable reference look applied. - return True - - class NetBoxError(FatalError): """A fatal error reported by the NetBox integration.""" @@ -1676,26 +1651,16 @@ def _type_relation_changes(self, module_type, existing_module): if not self.module_bay_types: return changes for field in MODULE_TYPE_RELATIONS: - if field not in module_type: - continue - netbox_value = getattr(existing_module, field, _MISSING) - if netbox_value is _MISSING: - continue - related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] - declared = module_type[field] - if not isinstance(declared, list) or any(not isinstance(x, str) or not x.strip() for x in declared): - # Malformed or bare key: leave the relation unmanaged rather than clear it. - continue - wanted = sorted(set(declared)) - current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) - if _relation_identities_differ( + change = _relation_change( + field, + module_type, + existing_module, self.device_types.module_bay_type_catalog, self.device_types._manufacturer_slug(module_type.get("manufacturer")), - declared, - related, - names_differ=wanted != current, - ): - changes.append((field, current, wanted)) + self.handle, + ) + if change is not None: + changes.append((change.property_name, change.old_value, change.new_value)) return changes def _resolve_type_relations(self, payload): diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index 2838adc9..25857e2e 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -129,6 +129,29 @@ def test_the_right_class_is_left_alone(self, netbox): assert nb._type_relation_changes(module_type, existing) == [] + def test_noncanonical_owner_slugs_do_not_make_relations_differ(self, netbox): + """NetBox may return the owner slug in another case; the catalog identity is canonical.""" + nb, _ = netbox + assigned = related("MX304-RE", "mx304-re", "Juniper") + existing = NetBoxRecord(7, model="JNP304-RE", module_bay_types=[assigned]) + nb.device_types.components.record( + "module_bay_templates", + "module", + 7, + {"Slot 0": NetBoxRecord(55, name="Slot 0", module_bay_types=[assigned])}, + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["MX304-RE"], + "module-bays": [{"name": "Slot 0", "module_bay_types": ["MX304-RE"]}], + } + + actionable, _, changes = nb.filter_actionable_module_types([module_type], {"juniper": {"JNP304-RE": existing}}) + + assert changes == [], "owner slug case must not make module or component relations differ" + assert actionable == [] + def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): """Juniper owns MX304-RE. A Generic object of that name is not the one referenced.""" nb, _ = netbox @@ -193,13 +216,22 @@ def test_an_omitted_key_leaves_the_relation_unmanaged(self, netbox): assert nb._type_relation_changes({"model": "JNP304-RE"}, existing) == [] - def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox): - """A bare key parses as None; clearing on it would drop a restriction nobody removed.""" - nb, _ = netbox - existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) - module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": None} + @pytest.mark.parametrize("declared", [None, "QSFP", [" "]], ids=["null", "string", "blank-name"]) + def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox, declared): + """Log malformed declarations without changing the existing restriction.""" + nb, console = netbox + existing = NetBoxRecord(7, model="JNP304-RE", module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": declared} assert nb._type_relation_changes(module_type, existing) == [] + actionable, _, changes = nb.filter_actionable_module_types([module_type], {"juniper": {"JNP304-RE": existing}}) + + assert actionable == [] + assert changes == [] + assert any( + "Ignored module_bay_types on 'JNP304-RE'" in line and f"expected a list of names, got {declared!r}" in line + for line in console.lines + ), "malformed module-type relations must be logged as ignored" def test_names_are_compared_when_the_definition_names_no_manufacturer(self, netbox): """Without an owning manufacturer there is no scope, so the name comparison stands.""" From 2f69cf82579921ad909136236a234459d0a1b092 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 10:33:47 +0200 Subject: [PATCH 35/39] fix(export): compare relation-name lists without their order The serializer writes module_bay_types sorted. A library file that lists the same names in another order read as differing, so the definition was exported although nothing changed. The compare now sorts relation-name lists on both sides and keeps duplicates. --- core/export.py | 18 ++++++++++++++++-- tests/test_exporter.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/core/export.py b/core/export.py index 53a91326..d09e613b 100644 --- a/core/export.py +++ b/core/export.py @@ -190,6 +190,7 @@ def _repo_supersedes(repo_yaml: dict, nb_serialized: dict) -> bool: Lists of named components are matched element-wise by ``name``: each NB component must be present in the repo with the same fields/values, but the repo may carry additional components or extra per-component fields. + Relation-name lists are sorted without removing duplicates. """ # Normalize manufacturer to canonical slug so dict-form repo values @@ -216,8 +217,21 @@ def _sort_port_mappings(d: dict) -> dict: fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} - nrepo = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml)))) - nnb = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized)))) + relations = _relation_names() + + def _sort_relations(d: dict) -> dict: + """Sort relation-name lists on the type and its components, keeping duplicates.""" + result = dict(d) + for key, value in d.items(): + if key in relations: + if isinstance(value, list) and all(isinstance(name, str) for name in value): + result[key] = sorted(value) + elif isinstance(value, list): + result[key] = [_sort_relations(item) if isinstance(item, dict) else item for item in value] + return result + + nrepo = _sort_relations(_sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml))))) + nnb = _sort_relations(_sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized))))) return _is_subset(nnb, nrepo) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 45584624..df1f6589 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -277,6 +277,45 @@ def test_an_empty_relation_does_not_make_every_definition_differ(self): assert populated["module-bays"][0]["module_bay_types"] == ["Control"] assert not _repo_supersedes(repo, populated), "a new restriction must trigger export" + def test_module_bay_relation_order_does_not_trigger_export(self): + """Relation order is cosmetic, but different names and duplicate counts must differ.""" + from core.nb_serializer import serialize_device_type + + record = _make_dt() + bay = _Stub(name="Slot 0", position="0", module_bay_types=[_Stub(name="A"), _Stub(name="B")]) + serialized = serialize_device_type(record, {record.id: {"module_bay_templates": [bay]}}) + assert serialized["module-bays"][0]["module_bay_types"] == ["A", "B"] + repo = { + **serialized, + "module-bays": [{"name": "Slot 0", "position": "0", "module_bay_types": ["B", "A"]}], + } + + assert _repo_supersedes(repo, serialized), "relation order must not trigger export" + assert _repo_supersedes(serialized, repo), "normalize relation order on both sides" + + for changed in (["A", "C"], ["B", "A", "A"], ["A"], "A", ["A", 1], ["A", {"name": "B"}]): + repo["module-bays"][0]["module_bay_types"] = changed + assert not _repo_supersedes(repo, serialized) + assert not _repo_supersedes(serialized, repo) + + def test_module_type_relation_order_does_not_trigger_export(self): + """A module type relation ignores order and preserves real differences.""" + from core.nb_serializer import serialize_module_type + + record = _make_mt() + record.module_bay_types = [_Stub(name="A"), _Stub(name="B")] + serialized = serialize_module_type(record, {}) + assert serialized["module_bay_types"] == ["A", "B"] + repo = {**serialized, "module_bay_types": ["B", "A"]} + + assert _repo_supersedes(repo, serialized), "relation order must not trigger export" + assert _repo_supersedes(serialized, repo), "normalize relation order on both sides" + + for changed in (["A", "C"], ["B", "A", "A"], ["A"], "A", ["A", 1], ["A", {"name": "B"}]): + repo["module_bay_types"] = changed + assert not _repo_supersedes(repo, serialized) + assert not _repo_supersedes(serialized, repo) + def test_a_default_positions_does_not_make_every_front_port_differ(self): """The export writes the schema-required positions; it must not re-export the library. From 4c78394ee1d1319923be0f3fee9c68285d2cdda3 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 10:34:47 +0200 Subject: [PATCH 36/39] fix(images): do not take an old attachment as the retried upload Attachment names are not unique. When an old same-named attachment stayed after the stale one was deleted, and the upload failed before NetBox committed it, the retry lookup returned the old attachment as the new upload. The caller then stored the new image hash, so later runs never uploaded the image. The upload now records the matching attachment ids first and accepts only a new match on a retry. --- core/netbox_api.py | 23 ++++++++---- tests/test_image_writes.py | 72 ++++++++++++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 1c02c2c1..397bd909 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -3205,8 +3205,9 @@ def upload_image_attachment(self, image_path, object_type, object_id): """Upload an image as an Image Attachment to a NetBox object. Attaches an image to any NetBox object type (e.g. module types, which lack built-in - image fields). After a transient error the attachment is looked up by name on that - object before the upload is sent again, so a lost reply does not create a second one. + image fields). Record matching attachment IDs before uploading because names are not + unique. After a transient error, accept only a new matching attachment. Multiple + new matches are ambiguous and fail the upload instead of reporting success. Args: image_path (str): Local file path of the image to upload. @@ -3231,9 +3232,20 @@ def upload_image_attachment(self, image_path, object_type, object_id): } with f: try: - create_with_retry( - self.netbox.extras.image_attachments, payload, "name", object_type=object_type, object_id=object_id - ) + endpoint = self.netbox.extras.image_attachments + filters = {"object_type": object_type, "object_id": object_id, "name": payload["name"]} + existing_ids = call_with_retry(lambda: {attachment.id for attachment in endpoint.filter(**filters)}) + + def find_created(): + """Return the sole matching attachment created after the initial lookup.""" + matches = [ + attachment for attachment in endpoint.filter(**filters) if attachment.id not in existing_ids + ] + if len(matches) > 1: + raise ValueError("Multiple new image attachments match the upload") + return matches[0] if matches else None + + call_with_retry(endpoint.create, payload, find_existing=find_created) except pynetbox.RequestError as e: self.handle.log(f"Error uploading image attachment for {target}: {_format_request_error(e)}") return False @@ -3241,7 +3253,6 @@ def upload_image_attachment(self, image_path, object_type, object_id): self.handle.log(f"Error uploading image attachment for {target} after {MAX_RETRIES} retries: {e}") return False except ValueError as e: - # Endpoint.get raises this when the lookup before a retry matches more than one attachment. self.handle.log(f"Cannot tell if the image attachment for {target} was created: {e}") return False self.handle.verbose_log(f"Image attachment '{filename}' uploaded for {target}") diff --git a/tests/test_image_writes.py b/tests/test_image_writes.py index dae6af0b..8e7d9acf 100644 --- a/tests/test_image_writes.py +++ b/tests/test_image_writes.py @@ -11,6 +11,7 @@ from core.graphql_client import NetBoxGraphQLClient from core.netbox_api import DeviceTypes, NetBox +from core.retry import MAX_RETRIES # The suite patches requests.Session by default, which would stop every request below. pytestmark = pytest.mark.real_http @@ -98,13 +99,21 @@ def test_an_upload_creates_the_attachment(self, device_types, server, image): assert console.lines == [] @pytest.mark.parametrize("reply", REPLIES) - def test_a_broken_reply_does_not_create_a_second_attachment(self, device_types, server, image, reply): + @pytest.mark.parametrize("old_count", [0, 1, 2]) + def test_a_broken_reply_does_not_create_a_second_attachment(self, device_types, server, image, reply, old_count): + """A lost reply recovers the new attachment even when its name already exists.""" dt, console, progress = device_types + old = {"object_type": "dcim.moduletype", "object_id": "42", "name": "c9300.front"} + server.collection("image_attachments").extend({**old, "id": index + 1} for index in range(old_count)) server.broken_replies = {("POST", 1): reply} assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True - assert len(server.collection("image_attachments")) == 1 + stored = server.collection("image_attachments") + assert len(stored) == old_count + 1 + [created] = [attachment for attachment in stored if attachment["id"] > old_count] + assert created["image"] == "/media/c9300.front.png" + assert server.uploads == [("image", "c9300.front.png", IMAGE_BYTES)] assert requests_to(server, "POST", "image_attachments") == 1 assert dt.counter["images"] == 1 assert progress == [1] @@ -120,6 +129,26 @@ def test_an_upload_sent_again_carries_the_whole_file(self, device_types, server, assert [data for _field, _name, data in server.uploads] == [IMAGE_BYTES, IMAGE_BYTES] assert len(server.collection("image_attachments")) == 1 + def test_a_retry_does_not_take_an_old_attachment(self, device_types, server, image): + """An old duplicate cannot stand in for an upload that did not commit.""" + dt, console, progress = device_types + old = {"object_type": "dcim.moduletype", "object_id": "42", "name": "c9300.front"} + server.collection("image_attachments").extend([{**old, "id": 1}, {**old, "id": 2}]) + assert dt.delete_image_attachment(1) is True + server.broken_replies = {("POST", 1): "dropped"} + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True + + assert requests_to(server, "POST", "image_attachments") == 2 + stored = server.collection("image_attachments") + assert len(stored) == 2 + [created] = [attachment for attachment in stored if attachment["id"] != 2] + assert created["image"] == "/media/c9300.front.png" + assert [data for _field, _name, data in server.uploads] == [IMAGE_BYTES, IMAGE_BYTES] + assert dt.counter["images"] == 1 + assert progress == [1] + assert console.lines == [] + def test_a_rejected_upload_returns_false_and_logs_netbox_reason(self, device_types, server, image): dt, console, progress = device_types server.errors = {"image_attachments": (400, '{"image":["Upload a valid image."]}')} @@ -143,11 +172,42 @@ def test_retries_exhausted_fail_only_that_upload(self, device_types, server, ima assert dt.upload_image_attachment(image, "dcim.moduletype", 43) is True assert [r["object_id"] for r in server.collection("image_attachments")] == ["43"] - def test_an_ambiguous_lookup_after_a_lost_reply_fails_that_upload(self, device_types, server, image): - """Two attachments with the name: nothing tells which one the lost reply created.""" + @pytest.mark.parametrize("reply", REPLIES) + def test_a_broken_initial_lookup_is_retried(self, device_types, server, image, reply): + """One transport error in the initial lookup is retried, then the upload goes ahead.""" + dt, _, _ = device_types + break_every(server, "GET", 1, reply) + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is True + + assert requests_to(server, "POST", "image_attachments") == 1 + assert len(server.collection("image_attachments")) == 1 + + @pytest.mark.parametrize("reply", REPLIES) + def test_a_broken_initial_lookup_fails_before_uploading(self, device_types, server, image, reply): + """An initial lookup that fails on every attempt logs a failure and sends no upload.""" + dt, console, progress = device_types + break_every(server, "GET", MAX_RETRIES + 1, reply) + + assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is False + + assert f"Error uploading image attachment for dcim.moduletype 42 after {MAX_RETRIES} retries" in logged(console) + assert requests_to(server, "POST", "image_attachments") == 0 + assert dt.counter["images"] == 0 + assert progress == [] + + def test_an_ambiguous_lookup_after_a_lost_reply_fails_that_upload(self, device_types, server, image, monkeypatch): + """Two new attachments with the same name leave the committed upload ambiguous.""" dt, console, _ = device_types - old = {"object_type": "dcim.moduletype", "object_id": "42", "name": "c9300.front"} - server.collection("image_attachments").append({**old, "id": 1}) + reply_to = server._reply_to + + def reply_after_another_create(handler, verb, answer): + """Simulate another writer that commits before the upload reply is lost.""" + if verb == "POST": + server.collection("image_attachments").append({**answer[1], "id": 1}) + reply_to(handler, verb, answer) + + monkeypatch.setattr(server, "_reply_to", reply_after_another_create) server.broken_replies = {("POST", 1): "lost"} assert dt.upload_image_attachment(image, "dcim.moduletype", 42) is False From 8e5aee0401848b81d700d4042523fd0c0053d514 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 10:49:20 +0200 Subject: [PATCH 37/39] fix(module-bay-types): keep the catalog identity on retry and symlinks A lost create reply was recovered by name only, so a same-named record with another slug from another writer was taken without the slug check. The recovery now matches the catalog slug. os.walk did not enter symlinked directories and gave no error, so a symlinked manufacturer directory was left out and a same-named Generic entry resolved instead. The catalog now follows symlinks, as the device-type glob does, and refuses a directory it already visited. --- core/module_bay_types.py | 9 ++++-- tests/test_module_bay_types.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 92457dff..7e6f94b9 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -163,7 +163,12 @@ def _unreadable(exc): ) from exc entries = {} - for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable): + visited_paths = set() + for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable, followlinks=True): + real_path = os.path.realpath(root) + if real_path in visited_paths: + raise ModuleBayCatalogError(f"Module bay type catalog directory {root!r} repeats a visited directory") + visited_paths.add(real_path) for filename in sorted(files): if not filename.endswith((".yaml", ".yml")): continue @@ -240,7 +245,7 @@ def _netbox_id(self, entry): f"creating module bay type {entry['name']!r}", self._netbox.dcim.module_bay_types, payload, - "name", + "slug", manufacturer_id=manufacturer_id, ) self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index fee56d7d..b78f927b 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -174,6 +174,27 @@ def test_a_dropped_lookup_is_retried(self, catalog): [bay_type_id] = cat.ids_for("juniper", ["MX304-RE"]) assert [r["id"] for r in server.collection("module_bay_types")] == [bay_type_id] + def test_a_competing_slug_is_not_recovered_after_a_dropped_create(self, catalog, monkeypatch): + """Reject a competing record whose name matches but whose slug differs from the catalog.""" + cat, server = catalog() + competing = {"id": 88, "name": "MX304-RE", "slug": "mx304_re", "manufacturer": {"id": 1}} + server.broken_replies = {("POST", 1): "dropped"} + real_write = server._write + + def write_with_competitor(handler, verb, name, payload, status, apply): + """Insert the competing record after lookup and before the first create fails.""" + if verb == "POST" and name == "module_bay_types" and not server.collection(name): + server.collection(name).append(competing) + return real_write(handler, verb, name, payload, status, apply) + + monkeypatch.setattr(server, "_write", write_with_competitor) + + with pytest.raises(ModuleBayTypeError, match="NetBox rejected creating module bay type 'MX304-RE'"): + cat.ids_for("juniper", ["MX304-RE"]) + + assert server.collection("module_bay_types") == [competing] + assert len(server.sent("POST", "module_bay_types")) == 2 + @pytest.mark.parametrize("reply", ["lost", "truncated", "garbled"]) def test_a_module_bay_type_create_with_a_broken_reply_is_not_repeated(self, catalog, reply): cat, server = catalog() @@ -196,6 +217,35 @@ def test_a_manufacturer_create_with_a_broken_reply_is_not_repeated(self, catalog class TestCatalogReading: """The catalog is read from real files on disk, including the shapes that are rejected.""" + def test_a_symlinked_manufacturer_directory_wins_over_generic(self, tmp_path, catalog): + """Resolve an entry from a linked manufacturer directory before the Generic fallback.""" + root = tmp_path / "symlink-library" + linked = tmp_path / "linked" + write_module_bay_type(linked, "Juniper", "juniper-qsfp-dd", "QSFP-DD") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD") + (root / "module-bay-types" / "Juniper").symlink_to( + linked / "module-bay-types" / "Juniper", target_is_directory=True + ) + cat, server = catalog(root=root) + + assert cat.identities_for("juniper", ["QSFP-DD"]) == frozenset({("juniper", "juniper-qsfp-dd")}) + assert len(cat.ids_for("juniper", ["QSFP-DD"])) == 1 + assert server.sent("POST", "module_bay_types") == [ + {"name": "QSFP-DD", "slug": "juniper-qsfp-dd", "manufacturer": 1} + ] + + @pytest.mark.timeout(2) + def test_a_symlink_cycle_is_refused_at_load(self, tmp_path, catalog): + """Reject a linked directory cycle before catalog resolution can continue.""" + write_module_bay_type(tmp_path, "Generic", "qsfp-dd", "QSFP-DD") + directory = tmp_path / "module-bay-types" / "Juniper" + directory.mkdir() + (directory / "cycle").symlink_to(directory, target_is_directory=True) + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("juniper", ["QSFP-DD"]) + def test_entry_without_a_description_is_created_without_one(self, tmp_path, catalog): write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") cat, server = catalog(root=tmp_path) From b5cb77978285d65687b9b4cb6b0f4b3e1ceb098a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 11:05:15 +0200 Subject: [PATCH 38/39] fix(relations): treat an unscoped module bay type as its own identity NetBox allows a module bay type with no manufacturer. Such an item made _relation_identities fall back to names, so an unscoped class named like the catalog's scoped class read as equal and the bay kept the wrong class. A queried null manufacturer is now the identity (None, slug). Only an item without a manufacturer field falls back. --- core/change_detector.py | 12 +++++- tests/test_module_bay_type_sync.py | 65 +++++++++++++++++++++++++----- tests/test_relation_scope.py | 25 ++++++++++++ 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index cdbc63c4..4d6d0c56 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -113,15 +113,23 @@ def _relation_identities(value): A read path that returns only ``id`` and ``name`` cannot answer the scope question, so the caller falls back to comparing names rather than inventing an identity. + An explicit null manufacturer identifies an unscoped object as ``(None, slug)``. """ from core.module_bay_types import manufacturer_slug identities = set() for item in value or []: slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) - manufacturer = item.get("manufacturer") if isinstance(item, dict) else getattr(item, "manufacturer", None) + manufacturer = ( + item.get("manufacturer", _MISSING) if isinstance(item, dict) else getattr(item, "manufacturer", _MISSING) + ) + if not slug or manufacturer is _MISSING: + return None + if manufacturer is None: + identities.add((None, slug)) + continue owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) - if not slug or not owner: + if not owner: return None identities.add((manufacturer_slug(owner), slug)) return frozenset(identities) diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index 25857e2e..364f33dd 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -14,9 +14,9 @@ import pytest from helpers import FakeNetBox, recording_handle, write_module_bay_type -from core.change_detector import ChangeType, ComponentChange, PropertyChange +from core.change_detector import ChangeDetector, ChangeType, ComponentChange, PropertyChange from core.component_registry import BY_YAML_KEY -from core.graphql_client import NetBoxGraphQLClient +from core.graphql_client import NetBoxGraphQLClient, _to_dotdict from core.module_bay_types import ModuleBayTypeCatalog from core.netbox_api import DeviceTypes, NetBox from core.outcomes import EntityKind, Outcome @@ -160,6 +160,39 @@ def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + def test_an_unscoped_class_is_reported_and_patched_as_catalog_ids(self, netbox, server): + """Replace an unscoped class with the catalog class that has the same name and slug.""" + nb, _ = netbox + assigned = {"id": 9, "name": "MX304-RE", "slug": "mx304-re", "manufacturer": None} + server.collection("module_bay_types").append(assigned) + stored = {"id": 7, "model": "JNP304-RE", "manufacturer": JUNIPER, "module_bay_types": [assigned]} + server.collection("module_types").append(stored) + existing = _to_dotdict(stored) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + actionable, _, changes = nb.filter_actionable_module_types([module_type], {"juniper": {"JNP304-RE": existing}}) + + assert actionable == [module_type] + assert changes[0][2] == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + assert nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") == (True, True) + [scoped] = [item for item in server.collection("module_bay_types") if item.get("manufacturer")] + assert scoped["manufacturer"]["slug"] == "juniper" + assert scoped["id"] != assigned["id"] + assert server.sent("PATCH", "module_types") == [{"id": 7, "module_bay_types": [scoped["id"]]}] + assert stored["module_bay_types"] == [scoped["id"]] + + def test_matching_names_are_left_alone_when_manufacturer_is_absent(self, netbox): + """A relation without a manufacturer field still uses the name comparison.""" + nb, _ = netbox + assigned = NetBoxRecord(9, name="MX304-RE", slug="mx304-re") + existing = NetBoxRecord(7, model="JNP304-RE", module_bay_types=[assigned]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + actionable, _, changes = nb.filter_actionable_module_types([module_type], {"juniper": {"JNP304-RE": existing}}) + + assert actionable == [] + assert changes == [] + def test_names_are_compared_when_netbox_returned_no_identity(self, netbox): """A record carrying only a name cannot answer the scope question; names still can.""" nb, _ = netbox @@ -547,17 +580,29 @@ def _change(name, new_value): ) def test_a_changed_restriction_is_patched_as_ids(self, make_device_types, server): + """Detect an unscoped class and replace it with the scoped catalog id.""" device_types, _ = make_device_types() - server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) - device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + assigned = {"id": 9, "name": "MX304-LMIC", "slug": "mx304-lmic", "manufacturer": None} + server.collection("module_bay_types").append(assigned) + stored = {"id": 55, "name": "FPC 0", "module_bay_types": [assigned]} + server.collection("module_bay_templates").append(stored) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": _to_dotdict(stored)}) + definition = { + "manufacturer": {"slug": "juniper"}, + "module-bays": [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + } - device_types._apply_updates_for_type( - "module-bays", [self._change("FPC 0", ["MX304-LMIC"])], {"manufacturer": {"slug": "juniper"}}, 3, "device" - ) + changes = ChangeDetector(device_types, device_types.handle)._compare_components(definition, 3) - assert server.sent("PATCH", "module_bay_templates") == [ - {"id": 55, "module_bay_types": [created_id(server, "mx304-lmic")]} - ] + assert len(changes) == 1 + assert changes[0].change_type == ChangeType.COMPONENT_CHANGED + device_types._apply_updates_for_type("module-bays", changes, definition, 3, "device") + + [scoped] = [item for item in server.collection("module_bay_types") if item.get("manufacturer")] + assert scoped["manufacturer"]["slug"] == "juniper" + assert scoped["id"] != assigned["id"] + assert server.sent("PATCH", "module_bay_templates") == [{"id": 55, "module_bay_types": [scoped["id"]]}] + assert stored["module_bay_types"] == [scoped["id"]] def test_a_rejected_patch_is_collected_rather_than_raised(self, make_device_types, server): """The bay resolves; NetBox refuses the write. That must land in the entity's report.""" diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py index caee3dd0..c39326f1 100644 --- a/tests/test_relation_scope.py +++ b/tests/test_relation_scope.py @@ -6,8 +6,10 @@ """ import pytest +from pynetbox.core.response import Record from core.change_detector import ChangeDetector +from core.graphql_client import DotDict from core.module_bay_types import ModuleBayTypeCatalog @@ -80,6 +82,19 @@ def test_wrong_scope_assignment_is_detected(two_scope_catalog): ) +@pytest.mark.parametrize("record_type", [dict, DotDict, Record], ids=["dict", "dotdict", "record"]) +def test_unscoped_assignment_is_detected(two_scope_catalog, record_type): + """A null manufacturer identifies an unscoped class even when its name and slug match.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + fields = {"id": 9, "url": None, "name": "X", "slug": "acme-x", "manufacturer": None} + assigned = Record(fields, None, None) if record_type is Record else record_type(fields) + netbox_comp = NetBoxBay("Slot 0", [assigned]) + + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + + assert [(c.property_name, c.old_value, c.new_value) for c in changes] == [("module_bay_types", ["X"], ["X"])] + + def test_right_scope_assignment_is_left_alone(two_scope_catalog): """The same bay already holding Acme's X is correct and must not be rewritten.""" catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) @@ -170,6 +185,16 @@ def test_an_unresolvable_reference_reaches_the_write_path(self, two_scope_catalo class TestNameComparisonFallback: """Where an identity cannot be had, comparing names is still better than doing nothing.""" + @pytest.mark.parametrize("record_type", [dict, DotDict, Record], ids=["dict", "dotdict", "record"]) + def test_matching_names_are_left_alone_when_manufacturer_is_absent(self, two_scope_catalog, record_type): + """A missing manufacturer field leaves the scope unknown even when a slug was fetched.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + fields = {"id": 9, "url": None, "name": "X", "slug": "acme-x"} + assigned = Record(fields, None, None) if record_type is Record else record_type(fields) + netbox_comp = NetBoxBay("Slot 0", [assigned]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + def test_names_are_compared_when_netbox_returned_no_slug(self, two_scope_catalog): """A read path returning only id and name cannot answer the scope question.""" detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) From a87ed133f1c2179a8659b6409271cdeaa600be11 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 18 Oct 2026 11:06:21 +0200 Subject: [PATCH 39/39] fix(module-types): send an explicit attribute_data null as null on update The update path changed attribute_data: null to {}. A profile schema with required fields and no top-level type, as NetBox's built-in hard_disk and memory profiles have, rejects {} but accepts null. The create path already sent null. The update now sends the declared value. --- core/netbox_api.py | 8 ++--- tests/test_module_type_profile.py | 60 +++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 397bd909..584a51d7 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1627,12 +1627,12 @@ def _profile_changes(module_type, existing_module): if wanted != current: changes.append(("profile", current, wanted)) if "attribute_data" in module_type and hasattr(existing_module, "attribute_data"): - declared = module_type["attribute_data"] or {} + declared = module_type["attribute_data"] stored = existing_module.attribute_data if not declared: # NetBox replaces, not merges, the stored attributes with an empty value. if stored: - changes.append(("attribute_data", stored, {})) + changes.append(("attribute_data", stored, declared)) return changes stored = stored if isinstance(stored, dict) else {} for key, wanted in declared.items(): @@ -1703,7 +1703,7 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): if field == "profile": updates["profile"] = curr_mt["profile"] stored = getattr(module_type_res, "attribute_data", None) - declared = curr_mt.get("attribute_data") or {} + declared = curr_mt.get("attribute_data") if "attribute_data" in curr_mt: updates[_REST_ATTRIBUTES_FIELD] = declared # A non-empty PATCH keeps every stored key; an undeclared set must end as no attributes, as on create. @@ -1712,7 +1712,7 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): or (bool(declared) and (not isinstance(stored, dict) or bool(stored.keys() - declared))) ) else: - updates[_REST_ATTRIBUTES_FIELD] = curr_mt["attribute_data"] or {} + updates[_REST_ATTRIBUTES_FIELD] = curr_mt["attribute_data"] if self.module_bay_types: from core.module_bay_types import ModuleBayTypeError diff --git a/tests/test_module_type_profile.py b/tests/test_module_type_profile.py index 6068a868..be3aae21 100644 --- a/tests/test_module_type_profile.py +++ b/tests/test_module_type_profile.py @@ -163,17 +163,42 @@ def test_a_boolean_and_a_number_are_different_values(self, netbox, server, defin assert log[0][2] == [("attribute_data.hot_swap", 1, True)] - @pytest.mark.parametrize("declared", ["{}", "null"]) - def test_explicit_empty_attributes_clear_the_stored_ones(self, netbox, server, definition, declared): + @pytest.mark.parametrize(("declared", "wanted"), [("{}", {}), ("null", None)], ids=["object", "null"]) + def test_explicit_empty_attributes_clear_the_stored_ones(self, netbox, server, definition, declared, wanted): + """Preserve the declared empty value in the PATCH and avoid another update after clearing.""" nb, _ = netbox record = existing(server, {"id": 4, "name": "Fan"}, {"rpm": 3000}) curr_mt = definition(f"manufacturer: Cisco\nmodel: SFP-10G-SR\nattribute_data: {declared}\n") - _, _, log = nb.filter_actionable_module_types([curr_mt], {"cisco": {"SFP-10G-SR": record}}) - assert log[0][2] == [("attribute_data", {"rpm": 3000}, {})] + actionable, _, log = nb.filter_actionable_module_types([curr_mt], {"cisco": {"SFP-10G-SR": record}}) + + assert actionable == [curr_mt] + assert nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") == (True, True) + assert server.sent("PATCH", "module_types") == [{"id": 7, "attributes": wanted}] + assert log[0][2] == [("attribute_data", {"rpm": 3000}, wanted)] + [stored_type] = server.collection("module_types") + assert stored_type["attribute_data"] == wanted + + actionable, _, log = nb.filter_actionable_module_types( + [curr_mt], {"cisco": {"SFP-10G-SR": _to_dotdict(stored_type)}} + ) + + assert actionable == [] + assert log == [] + + @pytest.mark.parametrize("declared", ["{}", "null"]) + def test_empty_attributes_match_stored_null(self, netbox, server, definition, declared): + """An empty object or null declaration leaves stored null attributes unchanged.""" + nb, _ = netbox + record = existing(server, {"id": 3, "name": "Transceiver"}) + curr_mt = definition(SFP_YAML.split("attribute_data")[0] + f"attribute_data: {declared}\n") + + actionable, _, log = nb.filter_actionable_module_types([curr_mt], {"cisco": {"SFP-10G-SR": record}}) - nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") - assert server.sent("PATCH", "module_types") == [{"id": 7, "attributes": {}}] + assert actionable == [] + assert log == [] + assert nb._try_update_module_type(curr_mt, record, "cisco/SFP-10G-SR.yaml") == (True, False) + assert not server.sent("PATCH", "module_types") class TestUpdate: @@ -282,16 +307,31 @@ def test_a_failed_recovery_clear_still_restores_the_old_profile(self, netbox, se assert stored_type["attribute_data"] == {"rpm": 3000} assert any("Could not clear" in line for line in console.lines) - def test_a_profile_change_keeps_explicit_empty_attributes(self, netbox, server, definition): + @pytest.mark.parametrize("stored", [{}, {"rpm": 3000}], ids=["empty", "populated"]) + @pytest.mark.parametrize(("declared", "wanted"), [("{}", {}), ("null", None)], ids=["object", "null"]) + def test_a_profile_change_keeps_explicit_empty_attributes( + self, netbox, server, definition, stored, declared, wanted + ): + """An explicit empty value replaces old attributes in one PATCH when the profile changes.""" nb, _ = netbox - record = existing(server, {"id": 4, "name": "Fan"}, {}) - curr_mt = definition(SFP_YAML.split("attribute_data")[0] + "attribute_data: {}\n") + record = existing(server, {"id": 4, "name": "Fan"}, stored) + curr_mt = definition(SFP_YAML.split("attribute_data")[0] + f"attribute_data: {declared}\n") assert nb._try_update_module_type(curr_mt, record, "x.yaml") == (True, True) + assert server.sent("PATCH", "module_types") == [ + {"id": 7, "profile": {"name": "Transceiver"}, "attributes": wanted} + ] [stored_type] = server.collection("module_types") assert stored_type["profile"] == {"id": 3, "name": "Transceiver"} - assert stored_type["attribute_data"] == {} + assert stored_type["attribute_data"] == wanted + + actionable, _, log = nb.filter_actionable_module_types( + [curr_mt], {"cisco": {"SFP-10G-SR": _to_dotdict(stored_type)}} + ) + + assert actionable == [] + assert log == [] def test_a_profile_only_change_drops_empty_old_attributes(self, netbox, server, definition): """A new profile can reject {} (required keys) but accepts no attributes, as on create."""