diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 6b689306..c7b5b82b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -7,7 +7,9 @@ on: branches: - development paths: + - .dockerignore - .github/** + - Dockerfile - module/** - netbox-sync.py - requirements.txt @@ -26,14 +28,14 @@ jobs: uses: actions/checkout@v6 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Lower case docker image name id: docker_image - uses: ASzc/change-string-case-action@v6 + uses: ASzc/change-string-case-action@v8 with: string: ${{ github.repository }} @@ -46,7 +48,7 @@ jobs: replace: ${{ vars.DOCKER_HUB_USERNAME }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -71,7 +73,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ vars.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..7c870977 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,46 @@ +name: Test + +on: + push: + branches: [ "main", "development" ] + pull_request: + branches: [ "main", "development" ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + + env: + # vcsim serves the captured vCenter inventories in tests/fixtures/vcsim + GOVMOMI_VERSION: v0.56.0 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + + - name: Install vcsim + run: | + curl -fsSL -o vcsim.tar.gz \ + "https://github.com/vmware/govmomi/releases/download/${GOVMOMI_VERSION}/vcsim_Linux_x86_64.tar.gz" + tar -xzf vcsim.tar.gz vcsim + sudo install -m 0755 vcsim /usr/local/bin/vcsim + vcsim -h 2>&1 | head -1 || true + + - name: Run tests + run: pytest -v diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..127f58c6 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,6 @@ + +* @bb-ricardo + + +/sources/hetzner/ @SerhiiZahuba +docs/source_hetzner.md @SerhiiZahuba diff --git a/Dockerfile b/Dockerfile index b32d8333..0852fa81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,43 @@ -FROM python:3.13-slim AS builder +FROM python:3.14-slim AS builder COPY requirements.txt . ARG VENV=/opt/netbox-sync/venv # Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends git gcc libc-dev && \ - rm -rf /var/lib/apt/lists/* && \ - python3 -m venv $VENV && \ +RUN python3 -m venv $VENV && \ $VENV/bin/python3 -m pip install --upgrade pip && \ $VENV/bin/pip install -r requirements.txt && \ - $VENV/bin/pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git && \ + $VENV/bin/pip install vmware-vcenter==9.1.1.0 && \ + $VENV/bin/python3 -m pip uninstall -y pip && \ find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf -FROM python:3.13-slim AS netbox-sync +FROM python:3.14-slim AS netbox-sync ARG VENV=/opt/netbox-sync/venv # Copy installed packages COPY --from=builder $VENV $VENV -# Add netbox-sync user -RUN groupadd --gid 1000 netbox-sync && \ - useradd --uid 1000 --gid netbox-sync --shell /bin/sh \ - --no-create-home --system netbox-sync +# Copy application files +WORKDIR /app +COPY . . + +# Install the security updates published since the base image was built, +# drop pip (not needed at runtime) and add the netbox-sync user. +# The code belongs to root and is read-only for the service user; only the +# cache directory is writable (group 0 as well, so an arbitrary uid in group 0 +# can use it) +RUN apt-get update && \ + apt-get dist-upgrade -y && \ + rm -rf /var/lib/apt/lists/* && \ + python3 -m pip uninstall -y --root-user-action=ignore pip && \ + groupadd --gid 1000 netbox-sync && \ + useradd --uid 1000 --gid netbox-sync --shell /bin/sh --no-create-home --system netbox-sync && \ + mkdir -p /app/cache && chown netbox-sync:0 /app/cache && chmod 0770 /app/cache USER netbox-sync -# Prepare the application -WORKDIR /app -COPY --chown=netbox-sync:netbox-sync . . - # Use virtual env packages and allow timezone setup ENV PATH=$VENV/bin:$PATH ENV TZ=Europe/Berlin diff --git a/LICENSE.txt b/LICENSE.txt index cefeaf2f..974ccad1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 - 2025 Ricardo Bartels +Copyright (c) 2020 - 2026 netbox-sync team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index ce8c6beb..cd832576 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # NetBox-Sync This is a tool to sync data from different sources to a NetBox instance. @@ -30,14 +29,8 @@ This ensures stale objects are removed from NetBox keeping an accurate current s ## Requirements ### Software -* python >= 3.6 -* packaging -* urllib3==2.2.1 -* wheel -* requests==2.31.0 -* pyvmomi==8.0.2.0.1 -* aiodns==3.0.0 -* pyyaml==6.0.1 +* python >= 3.12 +* see [requirements.txt](requirements.txt) ### Environment * NetBox >= 2.9 @@ -49,20 +42,17 @@ This ensures stale objects are removed from NetBox keeping an accurate current s # Installing * here we assume we install in ```/opt``` -## RedHat based OS -* on RedHat/CentOS 7 you need to install python3.6 and pip from EPEL first -* on RedHat/CentOS 8 systems the package name changed to `python3-pip` +## RedHat based distributions ```shell -yum install python36-pip +yum install python3-pip ``` -## Ubuntu 18.04 & 20.04 && 22.04 +## Debian (Ubuntu) based distributions ```shell apt-get update && apt-get install python3-venv ``` ## Clone repo and install dependencies -* If you need to use python 3.6 then you would need `requirements_3.6.txt` to install requirements * download and setup of virtual environment ```shell cd /opt @@ -76,10 +66,10 @@ pip3 install -r requirements.txt || pip install -r requirements.txt ``` ### VMware tag sync (if necessary) -The `vsphere-automation-sdk` must be installed if tags should be synced from vCenter to NetBox +The `vcf-sdk` must be installed if tags should be synced from vCenter to NetBox * assuming we are still in an activated virtual env ```shell -pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git +pip install --upgrade vcf-sdk ``` ## NetBox API token @@ -88,6 +78,8 @@ In order to updated data in NetBox you need a NetBox API token. * auth * secrets * users +* Both v1 (legacy) and v2 tokens (NetBox 4.5+, `nbt_` prefix) are supported. + The correct authorization header (`Token` or `Bearer`) is detected automatically. A short description can be found [here](https://docs.netbox.dev/en/stable/integrations/rest-api/#authentication) @@ -99,7 +91,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.8.1 (2026-03-18) +Version: 1.9.0 (2026-10-02) Project URL: https://github.com/bb-ricardo/netbox-sync options: @@ -215,21 +207,26 @@ In Order to sync all items regularly you can add a cron job like this one ## Docker -Run the application in a docker container. You can build it yourself or use the ones from docker hub. +Run the application in a docker container. You can build it yourself or use the published images. + +Available here: [ghcr.io/bb-ricardo/netbox-sync](https://github.com/bb-Ricardo/netbox-sync/pkgs/container/netbox-sync) -Available here: [bbricardo/netbox-sync](https://hub.docker.com/r/bbricardo/netbox-sync) +Images used to be published on Docker Hub as [bbricardo/netbox-sync](https://hub.docker.com/r/bbricardo/netbox-sync). +v1.9.0 is the last release pushed there; later releases are only published on the GitHub Container Registry, +so switch the pull address to `ghcr.io/bb-ricardo/netbox-sync`. * The application working directory is ```/app``` * Required to mount your ```settings.ini``` +* The NetBox cache is written to ```/app/cache```, mount a volume there to keep it between runs To build it by yourself just run: ```shell -docker build -t bbricardo/netbox-sync:latest . +docker build -t ghcr.io/bb-ricardo/netbox-sync:latest . ``` To start the container just use: ```shell -docker run --rm -it -v $(pwd)/settings.ini:/app/settings.ini bbricardo/netbox-sync:latest +docker run --rm -it -v $(pwd)/settings.ini:/app/settings.ini -v netbox-sync-cache:/app/cache ghcr.io/bb-ricardo/netbox-sync:latest ``` ## Kubernetes @@ -295,6 +292,17 @@ Check out the documentations for the different sources * [vmware](https://github.com/bb-Ricardo/netbox-sync/blob/main/docs/source_vmware.md) * [check_redfish](https://github.com/bb-Ricardo/netbox-sync/blob/main/docs/source_check_redfish.md) +### Filtering +netbox-sync provides various filtering capabilities to control what objects are synced from sources to NetBox: + +1. **General VM filtering**: Use `vm_include_filter` and `vm_exclude_filter` to include or exclude VMs by name. +2. **Tag-based VM filtering**: Use `vm_exclude_by_tag_filter` to exclude VMs with specific vCenter tags. +3. **Partial information filtering**: + - Use `vm_exclude_disk_sync` to exclude disk synchronization for VMs matching specific name patterns. + - Use `vm_exclude_disk_sync_by_tag` to exclude disk synchronization for VMs with specific vCenter tags. + +These filters allow for fine-grained control over what information is synchronized, helping to avoid clutter in the change log from temporary or backup-related disk changes. + If you have multiple vCenter instances or check_redfish folders just add another source with the same type in the **same** file. diff --git a/docs/source_hetzner.md b/docs/source_hetzner.md new file mode 100644 index 00000000..c8625c93 --- /dev/null +++ b/docs/source_hetzner.md @@ -0,0 +1,12 @@ +# Source: hetzner + +## Setup +You need to have a source section in your `settings.ini` file with following type: +```ini +type = hetzner +``` + + +### Hetzner api +You need to create a "Read-only" api_token +https://docs.hetzner.com/cloud/api/getting-started/generating-api-token/ \ No newline at end of file diff --git a/docs/source_vmware.md b/docs/source_vmware.md index 11f1d03e..247083b8 100644 --- a/docs/source_vmware.md +++ b/docs/source_vmware.md @@ -107,3 +107,99 @@ Primary IPv4/6 will be determined by interface that provides the default route f **Note:**
IP address information can only be extracted if guest tools are installed and running. + +#### 6. Sync VMware Tools guest hostname to a custom field + +The vCenter VM name (used as the NetBox VM `name`) and the hostname reported from inside the +guest OS by VMware Tools are not always identical. This can happen if: +* a VM was renamed in vCenter but the OS hostname was not changed (or vice versa) +* an OS administrator changed the hostname without informing the virtualization team +* a VM was cloned and the guest hostname was not adjusted afterwards + +To make this drift visible in NetBox, the option `vm_guest_hostname_custom_field` can be set to +the name of a NetBox custom field. If set, the guest hostname reported by VMware Tools (vCenter +property `guest.hostName`) is synced into that custom field on every synced VM, while the NetBox +VM `name` keeps reflecting the vCenter inventory name unchanged. This allows administrators to +compare the NetBox VM name against the custom field value on the same VM record to spot naming +drift. + +```ini +vm_guest_hostname_custom_field = vmware_guest_hostname +``` + +If the custom field does not exist in NetBox yet it will be created automatically (type `Text`, +assigned to the `Virtual Machine` object type), the same way other custom fields managed by +netbox-sync (e.g. `vcsa_*` custom attributes) are created. An administrator can also pre-create +the field beforehand; netbox-sync will then just use the existing field. + +This option is unset by default, so the feature is disabled and no additional custom field is +created unless explicitly configured. + +VMware Tools may not always report a hostname, for example if Tools are not installed, not +running, outdated, or simply have not reported guest information yet. In that case +netbox-sync leaves the custom field untouched and keeps the last known value, instead of +clearing it. A message is logged at debug level (`-l DEBUG2`) whenever this happens. + +Example result in NetBox for a VM named `APPPRD01` in vCenter whose guest OS reports +`appprd01.corp.example.com` as its hostname: + +```text +Virtual Machine Name: APPPRD01 +Custom Fields: + VMware Guest Hostname: appprd01.corp.example.com +``` + +### Cables to CDP/LLDP neighbors + +An ESXi host reports the switch and the switch port each of its physical interfaces (pNICs) is +connected to, if CDP or LLDP is enabled on the switch. With the option `sync_host_cables` enabled +netbox-sync uses this information to create cables in NetBox between the host interface and the +switch port. + +```ini +sync_host_cables = True +``` + +Cables are objects which are usually maintained by hand, that's why this option is disabled by +default. With the option disabled no cable is read from or written to NetBox at all. NetBox 3.3 or +newer is needed, on older versions the option is ignored. + +netbox-sync only connects things it can find, it never creates the other end of a cable: + +* the device the neighbor reports as its system name must already exist in NetBox. The name is + matched exactly first, a short name is only matched against a FQDN if that match is unambiguous +* the port the neighbor reports must already exist as an interface of that device. Long and short + interface names are matched against each other, so a reported `FastEthernet0/16` also matches an + interface named `Fa0/16` in NetBox. CDP reports the port ID, LLDP additionally reports a port + description and both are tried +* both interfaces must already exist in NetBox. An interface which was just discovered gets its + cable during the next run +* neither of the two interfaces may be connected already. A cable which was created by hand or which + connects to a different port is never changed or deleted, it is reported at log level `DEBUG` + instead + +Cables created by this source are tagged like every other object and are marked as orphaned and +pruned once the host stops reporting that neighbor (see `prune_enabled`). Disabling the option again +leaves all previously created cables untouched in NetBox. + +### Filtering VM Disk Information +VM disks are synchronized between vCenter and NetBox. Since NetBox 3.7.0, virtual disks are tracked as separate objects +linked to VMs. In some scenarios, such as when temporary disks are attached to VMs during backup operations +(e.g., "Independent-nonpersistent" disks from Veeam), you might want to exclude these changes from synchronization +to avoid cluttering your NetBox change log. + +You can use the following filter options to exclude disk synchronization for specific VMs: + +1. **`vm_exclude_disk_sync`**: A regex pattern matching VM names where disk synchronization should be excluded. + ```ini + vm_exclude_disk_sync = backup-.*, veeam-.* + ``` + +2. **`vm_exclude_disk_sync_by_tag`**: A comma-separated list of vCenter tags. VMs with any of these tags will have + their disk information excluded from synchronization. + ```ini + vm_exclude_disk_sync_by_tag = backup-vm, veeam-job + ``` + +When a VM matches these filters, it will still be synchronized to NetBox with all its other information +(CPU, memory, interfaces, IP addresses, etc.), but changes to disk information will be ignored. diff --git a/k8s-netbox-sync-cronjob.yaml b/k8s-netbox-sync-cronjob.yaml index defc7d9e..c2ac6c7d 100644 --- a/k8s-netbox-sync-cronjob.yaml +++ b/k8s-netbox-sync-cronjob.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: netbox-sync - image: bbricardo/netbox-sync:latest + image: ghcr.io/bb-ricardo/netbox-sync:latest imagePullPolicy: IfNotPresent args: - -c diff --git a/module/__init__.py b/module/__init__.py index a099794e..f5e5c9c7 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -7,8 +7,8 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -__version__ = "1.8.1" -__version_date__ = "2026-03-18" +__version__ = "1.9.0" +__version_date__ = "2026-10-02" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/module/common/__init__.py b/module/common/__init__.py index 88ba5d35..1bab456f 100644 --- a/module/common/__init__.py +++ b/module/common/__init__.py @@ -1,11 +1,8 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # # This work is licensed under the terms of the MIT license. # For a copy, see file LICENSE.txt included in this # repository or visit: . - - - diff --git a/module/common/cli_parser.py b/module/common/cli_parser.py index b26e1593..f76e2887 100644 --- a/module/common/cli_parser.py +++ b/module/common/cli_parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/common/config.py b/module/common/config.py index 21522ad1..9c54454a 100644 --- a/module/common/config.py +++ b/module/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/common/logging.py b/module/common/logging.py index b6764fd8..8e2ac9c7 100644 --- a/module/common/logging.py +++ b/module/common/logging.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/common/misc.py b/module/common/misc.py index 50526efa..7b795598 100644 --- a/module/common/misc.py +++ b/module/common/misc.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -159,6 +159,8 @@ def get_string_or_none(text=None): """ Only return stripped content of text if text is not None and not empty + A structured value is not a name and returns None. Scalars, including ints, still stringify. + Parameters ---------- text: str @@ -169,6 +171,9 @@ def get_string_or_none(text=None): (str, None): content of text """ + if isinstance(text, (dict, list, set, tuple)): + return None + if text is not None and len(str(text).strip()) > 0: return str(text).strip() diff --git a/module/common/support.py b/module/common/support.py index 6fea6464..93f52ba6 100644 --- a/module/common/support.py +++ b/module/common/support.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -44,7 +44,7 @@ def normalize_mac_address(mac_address=None): return mac_address -def perform_ptr_lookups(ips, dns_servers=None): +async def perform_ptr_lookups(ips, dns_servers=None): """ Perform DNS reverse lookups for IP addresses to find corresponding DNS name @@ -60,9 +60,7 @@ def perform_ptr_lookups(ips, dns_servers=None): dict: of {"ip": "hostname"} for requested ips, hostname will be None if no hostname returned """ - loop = asyncio.get_event_loop() - - resolver = aiodns.DNSResolver(loop=loop) + resolver = aiodns.DNSResolver() if dns_servers is not None: if isinstance(dns_servers, list): @@ -71,8 +69,12 @@ def perform_ptr_lookups(ips, dns_servers=None): else: log.error(f"List of provided DNS servers invalid: {dns_servers}") - queue = asyncio.gather(*(reverse_lookup(resolver, ip) for ip in ips)) - results = loop.run_until_complete(queue) + # TaskGroup is faster than gather in 3.14 and provides better error safety + async with asyncio.TaskGroup() as tg: + tasks = [tg.create_task(reverse_lookup(resolver, ip)) for ip in ips] + + # After the 'async with' block, all tasks are guaranteed complete + results = [task.result() for task in tasks] # return dictionary instead of a list of dictionaries return {k: v for x in results for k, v in x.items()} diff --git a/module/config/__init__.py b/module/config/__init__.py index b657b980..005e161e 100644 --- a/module/config/__init__.py +++ b/module/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/config/base.py b/module/config/base.py index 7d71b1f4..10614938 100644 --- a/module/config/base.py +++ b/module/config/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -35,6 +35,7 @@ def __getattr__(self, item): return getattr(self, item) return None + class ConfigBase: """ Base class to parse config data diff --git a/module/config/file_output.py b/module/config/file_output.py index 5adfd49a..6a002f15 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -16,6 +16,7 @@ from module.netbox.config import NetBoxConfig from module.sources.vmware.config import VMWareConfig from module.sources.check_redfish.config import CheckRedfishConfig +from module.sources.hetzner.config import HetznerConfig from module.common.logging import get_logger from module.config import default_config_file_path, source_config_section_name from module.config.files import ConfigFile, ConfigFileINI, ConfigFileYAML @@ -33,7 +34,8 @@ class ConfigFileOutput(DescriptionFormatterMixin): source_config_list = [ VMWareConfig, - CheckRedfishConfig + CheckRedfishConfig, + HetznerConfig ] header = f"Welcome to the {__description__} configuration file." diff --git a/module/config/files.py b/module/config/files.py index 6bed2d41..bb6c7002 100644 --- a/module/config/files.py +++ b/module/config/files.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/config/formatter.py b/module/config/formatter.py index 533c9bec..3f4df229 100644 --- a/module/config/formatter.py +++ b/module/config/formatter.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/config/group.py b/module/config/group.py index 14e50b8e..7b333089 100644 --- a/module/config/group.py +++ b/module/config/group.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/config/option.py b/module/config/option.py index b458abde..d2bc1502 100644 --- a/module/config/option.py +++ b/module/config/option.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/config/parser.py b/module/config/parser.py index ddfeaae0..3d06108a 100644 --- a/module/config/parser.py +++ b/module/config/parser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -9,6 +9,7 @@ import os +import re import configparser from typing import Dict import yaml @@ -161,6 +162,11 @@ def _add_config_data(self, config_data: dict, config_file: str = "") -> None: for source_name, source_data in section_data.items(): + if not isinstance(source_data, dict): + self._add_error(f"Parsed config data from file '{config_file}' for " + f"'{section}/{source_name}' is not a dictionary") + continue + current_data = grab(self.content, f"{section}|{source_name}", separator="|") if current_data is None: @@ -300,11 +306,14 @@ def _parse_source_env_vars(self) -> Dict: env_var_list[key.upper()] = value env_var_names[key.upper()] = key + # a source is identified by NBS_SOURCE__NAME. The index carries no + # underscore, so an option that merely ends in _NAME (strip_host_domain_name, + # vlan_sync_exclude_by_name, ...) is not mistaken for a source of its own + source_name_pattern = re.compile(rf"^{re.escape(env_var_source_prefix)}_(?P[^_]+)_NAME$") for env_var in env_var_list.keys(): - - # try to find a var which contains the source name - if env_var.endswith("_NAME"): - source_indexes.add(env_var.replace(f"{env_var_source_prefix}_", "", 1).replace("_NAME", "", 1)) + match = source_name_pattern.match(env_var) + if match is not None: + source_indexes.add(match.group("index")) for source_index in source_indexes: @@ -317,6 +326,11 @@ def _parse_source_env_vars(self) -> Dict: for key, value in env_var_list.items(): + # only this source's own variables; anything else belongs to another + # source or to nobody and must not end up in this config + if not key.startswith(f"{source_prefix}_"): + continue + if key != f"{source_prefix}_NAME": source_env_config[key.replace(f"{source_prefix}_", "", 1).lower()] = value diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index b2cbb2bc..aa6b5094 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -17,6 +17,8 @@ NBTenant, NBSite, NBSiteGroup, + NBRegion, + NBLocation, NBVRF, NBVLAN, NBVLANList, @@ -38,7 +40,11 @@ NBMACAddress, NBFHRPGroupItem, NBInventoryItem, - NBPowerPort + NBPowerPort, + NBCable, + NBModuleType, + NBModuleBay, + NBModule ) primary_tag_name = "NetBox-synced" diff --git a/module/netbox/config.py b/module/netbox/config.py index 7bf00f12..04df50bc 100644 --- a/module/netbox/config.py +++ b/module/netbox/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -26,7 +26,8 @@ def __init__(self): ConfigOption("api_token", str, description="""Requires an NetBox API token with full permissions on all objects except - 'auth', 'secrets' and 'users' + 'auth', 'secrets' and 'users'. + Both v1 (legacy) and v2 (NetBox 4.5+, nbt_ prefix) tokens are supported. """, config_example="XYZ", mandatory=True, @@ -97,6 +98,30 @@ def __init__(self): """, default_value=False), + ConfigOption("orphaned_device_status", + str, + description="""Set the status of orphaned devices to this value. If undefined + the status of a device is never changed by this program. Needs to be a valid + device status in NetBox (i.e: 'decommissioning', 'offline', 'planned') and + requires 'prune_enabled' to be true, as pruning is switched off whenever a + source was unavailable. Once a device is reported by a source again its + status is set back to 'active', but only if it still carries the status + defined here + """, + config_example="decommissioning"), + + ConfigOption("orphaned_vm_status", + str, + description="""Set the status of orphaned virtual machines to this value. If + undefined the status of a virtual machine is never changed by this program. + Needs to be a valid virtual machine status in NetBox (i.e: 'decommissioning', + 'offline', 'planned') and requires 'prune_enabled' to be true, as pruning is + switched off whenever a source was unavailable. Once a virtual machine is + reported by a source again its status is set back to 'active', but only if it + still carries the status defined here + """, + config_example="decommissioning"), + ConfigOption("default_netbox_result_limit", int, description="""The maximum number of objects returned in a single request. diff --git a/module/netbox/connection.py b/module/netbox/connection.py index 02ba53bd..c4d686fb 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -9,6 +9,7 @@ import json import os +import re import pickle import pprint from datetime import datetime @@ -152,8 +153,13 @@ def create_session(self) -> requests.Session: requests.Session: session handler of new NetBox session """ + # the config value is the bare token; a scheme typed in front of it + # ("Bearer nbt_...", "Token abc...") must not be sent twice + token = re.sub(r"^\s*(?:bearer|token)\s+", "", str(self.settings.api_token), flags=re.IGNORECASE).strip() + # NetBox 4.5+ API tokens (nbt_.) use the Bearer scheme + keyword = "Bearer" if token.startswith("nbt_") else "Token" header = { - "Authorization": f"Token {self.settings.api_token}", + "Authorization": f"{keyword} {token}", "User-Agent": f"netbox-sync/{__version__}", "Content-Type": "application/json" } @@ -268,6 +274,11 @@ class definition of the desired NetBox object if "limit" not in params.keys(): params["limit"] = self.settings.default_netbox_result_limit + # rows tied on a non-unique default ordering have no stable position across pages, + # so a walk can repeat one and skip another; the primary key makes the sort total + if nb_id is None and "ordering" not in params: + params["ordering"] = "id" + # always exclude config context params["exclude"] = "config_context" diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 3674c872..4d137406 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -8,6 +8,7 @@ # repository or visit: . import json +import asyncio from module.netbox import * from module.common.misc import grab @@ -308,6 +309,35 @@ def get_all_interfaces(self, this_object: (NBVM, NBDevice)): return interfaces + @staticmethod + def get_orphaned_status(this_object, netbox_handler): + """ + Return the status this program assigns to $this_object while it is orphaned. + + Only devices and virtual machines have such an option and only if the user + configured one. An undefined option also means that this program never touches + the status of these objects. + + Parameters + ---------- + this_object: NetBoxObject + the object to return the configured orphaned status for + netbox_handler: NetBoxHandler + the object instance of a NetBox handler to get the settings from + + Returns + ------- + (str, None): the configured status, None if unconfigured or unsupported object type + """ + + if isinstance(this_object, NBDevice): + return netbox_handler.settings.orphaned_device_status + + if isinstance(this_object, NBVM): + return netbox_handler.settings.orphaned_vm_status + + return None + def tag_all_the_things(self, netbox_handler): """ Tag all items which have been created/updated/inherited by this program @@ -346,6 +376,17 @@ def tag_all_the_things(self, netbox_handler): if netbox_handler.orphaned_tag in this_object_tags: this_object.remove_tags(netbox_handler.orphaned_tag) + # set the status back to active, but only if this program parked this + # object at the configured orphaned status. Any other status was set + # by a user or reported by the source and has to be left alone. + orphaned_status = self.get_orphaned_status(this_object, netbox_handler) + if orphaned_status is not None: + current_status = grab(this_object, "data.status") + if isinstance(current_status, dict): + current_status = current_status.get("value") + if current_status == orphaned_status: + this_object.update(data={"status": "active"}) + # if object was tagged by this program in previous runs but is not present # anymore then add the orphaned tag except it originated from a disabled source else: @@ -389,6 +430,14 @@ def tag_all_the_things(self, netbox_handler): this_object.add_tags(netbox_handler.orphaned_tag) + # set orphaned status on devices/VMs if configured + # and pruning is enabled (if a source was unavailable, + # pruning is disabled to prevent false orphaning) + if netbox_handler.settings.prune_enabled is True: + orphaned_status = self.get_orphaned_status(this_object, netbox_handler) + if orphaned_status is not None: + this_object.update(data={"status": orphaned_status}) + def query_ptr_records_for_all_ips(self): """ Perform a DNS lookup for all IP address of a certain source if desired. @@ -428,7 +477,7 @@ def query_ptr_records_for_all_ips(self): continue # get DNS names for IP addresses: - records = perform_ptr_lookups(data.get("ips"), data.get("servers")) + records = asyncio.run(perform_ptr_lookups(data.get("ips"), data.get("servers"))) for ip in self.get_all_items(NBIPAddress): diff --git a/module/netbox/manufacturer_mapping.py b/module/netbox/manufacturer_mapping.py index cecb163e..a69d2a57 100644 --- a/module/netbox/manufacturer_mapping.py +++ b/module/netbox/manufacturer_mapping.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dae2113a..2c05f1b4 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -13,7 +13,7 @@ # noinspection PyUnresolvedReferences from packaging import version -from module.common.misc import grab +from module.common.misc import grab, get_string_or_none from module.common.logging import get_logger from module.netbox.manufacturer_mapping import sanitize_manufacturer_name @@ -377,7 +377,7 @@ def to_dict(self): if isinstance(data_value, list): new_data_value = list() for possible_option in data_value: - if type(possible_option) == type: + if type(possible_option) is type: new_data_value.append(str(possible_option)) else: new_data_value.append(possible_option) @@ -385,7 +385,7 @@ def to_dict(self): data_value = new_data_value # if value is class name then print class name - if type(data_value) == type: + if type(data_value) is type: data_value = str(data_value) data_model[data_key] = data_value @@ -458,7 +458,7 @@ def format_slug(text=None, max_len=50): # Enforce max length return text[0:max_len] - def get_uniq_slug(self, text=None, max_len=50)-> str: + def get_uniq_slug(self, text=None, max_len=50) -> str: """ return an uniq slug. If the default slug is already used try to append a number until a slug is found which has not been used. @@ -480,7 +480,7 @@ def get_uniq_slug(self, text=None, max_len=50)-> str: if self.inventory.slug_used(self.__class__, slug) is False: return slug - for x in range(1,20): + for x in range(1, 20): new_slug = f"{slug}-{x}" if self.inventory.slug_used(self.__class__, new_slug) is False and len(new_slug) <= max_len: log.info(f"Slug '{slug}' for {self.name} '{text}' has been used. " @@ -552,7 +552,6 @@ def update(self, data=None, read_from_netbox=False, source=None): parsed_data = dict() for key, value in data.items(): - if key not in self.data_model.keys(): log.error(f"Found undefined data model key '{key}' for object '{self.__class__.__name__}'") continue @@ -686,27 +685,47 @@ def update(self, data=None, read_from_netbox=False, source=None): # Fix for object/multi-object custom fields # When patching, we only need the IDs, not the full object representation - new_value_copy = new_value.copy() - for field_name, field_value in new_value_copy.items(): - # Check for custom field type - custom_field = self.inventory.get_by_data(NBCustomField, data={"name": field_name}) - if custom_field is not None: + # returned by the NetBox API. The values of the current AND the new data + # need to be reduced. Reducing only the new values would miss unchanged + # object custom fields which get merged into the update from the current + # data, and comparing reduced to unreduced values would report a change + # on every run. + def reduce_object_custom_fields_to_ids(custom_field_data: dict) -> dict: + + reduced_data = dict(custom_field_data) + for f_name, f_value in custom_field_data.items(): + # Check for custom field type + custom_field = self.inventory.get_by_data(NBCustomField, data={"name": f_name}) + if custom_field is None: + continue + field_type = grab(custom_field, "data.type") + # custom fields read from the NetBox API report the type as + # a dict like {"value": "multiobject", "label": "Multiple objects"} + if isinstance(field_type, dict): + field_type = field_type.get("value") + # Handle object type custom fields - need only ID - if field_type == "object" and isinstance(field_value, dict) and field_value.get('id') is not None: - new_value[field_name] = field_value.get('id') + if field_type == "object" and isinstance(f_value, dict) and \ + f_value.get('id') is not None: + reduced_data[f_name] = f_value.get('id') # Handle multi-object type custom fields - need list of IDs - elif field_type == "multi-object" and isinstance(field_value, list): + # NetBox reports the type of these fields as 'multiobject' + elif field_type in ("multiobject", "multi-object") and isinstance(f_value, list): ids = [] - for item in field_value: + for item in f_value: if isinstance(item, dict) and item.get('id') is not None: ids.append(item.get('id')) if ids: - new_value[field_name] = ids + reduced_data[f_name] = ids - new_value = {**current_value, **new_value} + return reduced_data + + current_value = reduce_object_custom_fields_to_ids(current_value) + current_value_str = str(current_value) + new_value = {**current_value, **reduce_object_custom_fields_to_ids(new_value)} new_value_str = str(new_value) elif isinstance(new_value, (NetBoxObject, NBObjectList)): new_value_str = str(new_value.get_display_name()) @@ -1304,14 +1323,16 @@ def __init__(self, *args, **kwargs): NBPowerPort.object_type, NBClusterGroup.object_type, NBVMInterface.object_type, - NBVM.object_type + NBVM.object_type, + NBModule.object_type ] self.data_model = { "object_types": list, # field name (object_types) for NetBox < 4.0.0 "content_types": list, - "type": ["text", "longtext", "integer", "boolean", "date", "url", "json", "select", "multiselect", "object", "multi-object"], + "type": ["text", "longtext", "integer", "boolean", "date", "url", "json", "select", "multiselect", "object", + "multiobject", "multi-object"], "name": 50, "label": 50, "description": 200, @@ -1424,39 +1445,39 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) -# class NBLocation(NetBoxObject): -# name = "location" -# api_path = "dcim/locations" -# object_type = "dcim.location" -# primary_key = "name" -# prune = False -# read_only = True -# -# def __init__(self, *args, **kwargs): -# self.data_model = { -# "name": 100, -# "slug": 100, -# "site": NBSite, -# "tags": NBTagList -# } -# super().__init__(*args, **kwargs) -# -# -# class NBRegion(NetBoxObject): -# name = "region" -# api_path = "dcim/regions" -# object_type = "dcim.region" -# primary_key = "name" -# prune = False -# read_only = True -# -# def __init__(self, *args, **kwargs): -# self.data_model = { -# "name": 100, -# "slug": 100, -# "tags": NBTagList -# } -# super().__init__(*args, **kwargs) +class NBLocation(NetBoxObject): + name = "location" + api_path = "dcim/locations" + object_type = "dcim.location" + primary_key = "name" + prune = False + read_only = True + + def __init__(self, *args, **kwargs): + self.data_model = { + "name": 100, + "slug": 100, + "site": NBSite, + "tags": NBTagList + } + super().__init__(*args, **kwargs) + + +class NBRegion(NetBoxObject): + name = "region" + api_path = "dcim/regions" + object_type = "dcim.region" + primary_key = "name" + prune = False + read_only = True + + def __init__(self, *args, **kwargs): + self.data_model = { + "name": 100, + "slug": 100, + "tags": NBTagList + } + super().__init__(*args, **kwargs) class NBSite(NetBoxObject): @@ -1699,7 +1720,6 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) - def resolve_relations(self): self.resolve_scoped_relations("scope_id", "scope_type") @@ -1868,14 +1888,14 @@ class NBCluster(NetBoxObject): api_path = "virtualization/clusters" object_type = "virtualization.cluster" primary_key = "name" - secondary_key = "site" + secondary_key = "scope_id" prune = False - # include_secondary_key_if_present = True def __init__(self, *args, **kwargs): self.mapping = NetBoxMappings() + # scope types allowed for clusters self.scopes = [ - NBSite, NBSiteGroup + NBSite, NBSiteGroup, NBLocation, NBRegion ] self.data_model = { "name": 100, @@ -1884,28 +1904,22 @@ def __init__(self, *args, **kwargs): "tenant": NBTenant, "group": NBClusterGroup, "scope_type": self.mapping.scopes_object_types(self.scopes), - # currently only site is supported as a scope - "scope_id": NBSite, + # supports scoped clusters + "scope_id": self.scopes, + # supports pre4.2.0 clusters with site + "site": NBSite, "tags": NBTagList } super().__init__(*args, **kwargs) def update(self, data=None, read_from_netbox=False, source=None): - # Add adaption for change in NetBox 4.2.0 Device model - if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): - if data.get("site") is not None: - data["scope_id"] = data.get("site") - data["scope_type"] = "dcim.site" - del data["site"] - - if data.get("scope_id") is not None: - data["scope_type"] = "dcim.site" - super().update(data=data, read_from_netbox=read_from_netbox, source=source) def resolve_relations(self): - + log.debug2(f"Resolving relations for {self.name} '{self.get_display_name()}'") + # NetBox reports the scope as an id, turn it back into the object it points to, + # otherwise every run sees a change from the id to the object and updates the cluster self.resolve_scoped_relations("scope_id", "scope_type") super().resolve_relations() @@ -2063,7 +2077,9 @@ def __init__(self, *args, **kwargs): "description": 200, "mark_connected": bool, "tags": NBTagList, - "parent": object + "parent": object, + # NetBox cascade-deletes module components, so the module owns its interfaces + "module": NBModule } super().__init__(*args, **kwargs) @@ -2225,6 +2241,26 @@ def get_device_vm(self): elif isinstance(o_interface, NBVMInterface): return o_interface.data.get("virtual_machine") + def get_role(self): + """ + Return the role of this IP address as a plain string. + + NetBox reports the role as a dict ({"value": ..., "label": ...}), + an object this program created itself carries the plain value. + + Returns + ------- + (str, None): the role of this IP address or None if unset + """ + + role = self.data.get("role") + + if isinstance(role, dict): + return role.get("value") + + return role + + def remove_interface_association(self): o_id = self.data.get("assigned_object_id") o_type = self.data.get("assigned_object_type") @@ -2240,6 +2276,7 @@ def remove_interface_association(self): if o_type is not None: self.unset_attribute("assigned_object_type") + class NBMACAddress(NetBoxObject): name = "MAC address" api_path = "dcim/mac-addresses" @@ -2398,7 +2435,9 @@ def __init__(self, *args, **kwargs): "allocated_draw": int, "mark_connected": bool, "tags": NBTagList, - "custom_fields": NBCustomField + "custom_fields": NBCustomField, + # the PSU module owns its power port, NetBox cascade-deletes it with the module + "module": NBModule } super().__init__(*args, **kwargs) @@ -2417,4 +2456,179 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) + +class NBCable(NetBoxObject): + name = "cable" + api_path = "dcim/cables" + object_type = "dcim.cable" + # a cable has no natural name, the label is the only free form text attribute it has + primary_key = "label" + prune = True + # cable terminations are lists of objects since NetBox 3.3 + min_netbox_version = "3.3" + + def __init__(self, *args, **kwargs): + self.data_model = { + "label": 100, + "a_terminations": list, + "b_terminations": list, + "status": ["connected", "planned", "decommissioning"], + "type": [ + "cat3", "cat5", "cat5e", "cat6", "cat6a", "cat7", "cat7a", "cat8", + "dac-active", "dac-passive", + "mmf", "mmf-om1", "mmf-om2", "mmf-om3", "mmf-om4", "mmf-om5", + "smf", "smf-os1", "smf-os2", "aoc", "power", "usb", "coaxial" + ], + "description": 200, + "color": str, + "length": float, + "length_unit": ["km", "m", "cm", "mi", "ft", "in"], + "tags": NBTagList + } + super().__init__(*args, **kwargs) + + def format_termination(self, termination): + """ + format a single cable termination as string + + Parameters + ---------- + termination: dict + a single entry of a cable "a_terminations"/"b_terminations" list + + Returns + ------- + (str, None): the name of the terminated object, None if it can't be determined + """ + + if not isinstance(termination, dict): + return None + + # data read from NetBox contains the terminated object, data compiled by a source only the ID + termination_object = termination.get("object") + if isinstance(termination_object, dict) and termination_object.get("display") is not None: + return f"{termination_object.get('display')}" + + object_id = termination.get("object_id") + if object_id is None: + return None + + # a source only knows the ID of an interface it compiled a cable for + if termination.get("object_type") == NBInterface.object_type and self.inventory is not None: + interface_object = self.inventory.get_by_id(NBInterface, nb_id=object_id) + if interface_object is not None: + return interface_object.get_display_name(including_second_key=True) + + return f"{termination.get('object_type')} {object_id}" + + def get_display_name(self, data=None, including_second_key=False): + """ + A cable label is optional and mostly unset. Fall back to the objects this cable + connects to get a name which actually says something. + """ + + this_data = data if data is not None else self.data + + label = get_string_or_none(this_data.get(self.primary_key)) + if label is not None: + return label + + terminations = list() + for side in ["a_terminations", "b_terminations"]: + side_names = [self.format_termination(x) for x in this_data.get(side) or list()] + side_names = [x for x in side_names if x is not None] + if len(side_names) > 0: + terminations.append(", ".join(side_names)) + + if len(terminations) == 0: + return None + + return " <> ".join(terminations) + + +class NBModuleType(NetBoxObject): + name = "module type" + api_path = "dcim/module-types" + object_type = "dcim.moduletype" + # matched by model only, like NBDeviceType (server part models are effectively unique) + primary_key = "model" + prune = False + # modules replace the deprecated inventory items starting with NetBox 4.3 + min_netbox_version = "4.3" + + def __init__(self, *args, **kwargs): + self.data_model = { + "model": 100, + "manufacturer": NBManufacturer, + "part_number": 50, + "description": 200, + "comments": str, + "tags": NBTagList, + "custom_fields": NBCustomField + } + super().__init__(*args, **kwargs) + + +class NBModuleBay(NetBoxObject): + name = "module bay" + api_path = "dcim/module-bays" + object_type = "dcim.modulebay" + primary_key = "name" + secondary_key = "device" + prune = True + min_netbox_version = "4.3" + + def __init__(self, *args, **kwargs): + self.data_model = { + "device": NBDevice, + "name": 64, + "label": 64, + "position": 30, + "description": 200, + "tags": NBTagList, + "custom_fields": NBCustomField + } + super().__init__(*args, **kwargs) + + +class NBModule(NetBoxObject): + name = "module" + api_path = "dcim/modules" + object_type = "dcim.module" + # a module has no name of its own, it is identified by the bay it is installed in + primary_key = "module_bay" + secondary_key = "device" + prune = True + min_netbox_version = "4.3" + + def __init__(self, *args, **kwargs): + self.data_model = { + "device": NBDevice, + "module_bay": NBModuleBay, + "module_type": NBModuleType, + "status": ["offline", "active", "planned", "staged", "failed", "inventory", "decommissioning"], + "serial": 50, + "asset_tag": 50, + "description": 200, + "tags": NBTagList, + "custom_fields": NBCustomField + } + super().__init__(*args, **kwargs) + + def get_display_name(self, data=None, including_second_key=False): + + # a module has no name on its own, derive its display name from the module bay it lives in + this_data_set = data if data is not None else self.data + + if this_data_set is not None: + module_bay = this_data_set.get("module_bay") + if isinstance(module_bay, NetBoxObject): + return module_bay.get_display_name(including_second_key=including_second_key) + if isinstance(module_bay, dict): + bay_name = module_bay.get("name") or module_bay.get("display") + if bay_name is not None: + return bay_name + + return super().get_display_name(data=data, including_second_key=including_second_key) + # EOF diff --git a/module/sources/__init__.py b/module/sources/__init__.py index e1bf82c0..0666da30 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -9,6 +9,7 @@ # define all available sources here from module.sources.vmware.connection import VMWareHandler +from module.sources.hetzner.connection import HetznerHandler from module.sources.check_redfish.import_inventory import CheckRedfish from module.common.logging import get_logger @@ -18,7 +19,7 @@ from module.config import source_config_section_name # list of valid sources -valid_sources = [VMWareHandler, CheckRedfish] +valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler] def validate_source(source_class_object=None, state="pre"): diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 38d6e8e9..34119bcb 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -46,6 +46,21 @@ def __init__(self): overwrites the device host name in NetBox""", default_value=False), + ConfigOption("dell_serial_from_service_tag", + bool, + description="""for Dell devices, use the Service Tag as the NetBox device + serial number (matching what dmidecode and the OS report) instead of the + system serial number. The original system serial number (the Dell PPID) + is then stored in the 'system_serial' custom field""", + default_value=False), + + ConfigOption("model_components_as_modules", + bool, + description="""model discovered hardware components (CPUs, memory, drives, + controllers, NICs, ...) as NetBox modules instead of the deprecated inventory + items. Requires NetBox >= 4.3, on older versions inventory items are used""", + default_value=False), + ConfigOption("overwrite_power_supply_name", bool, description="""define if the name of the power supply discovered via check_redfish @@ -70,6 +85,13 @@ def __init__(self): via check_redfish if False only data which is not preset in NetBox will be added""", default_value=False), + ConfigOption("skip_fhrp_group_ips", + bool, + description="""define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be + skipped. If True this IP address will be skipped and not synced to NetBox to prevent incorrect + syncing.""", + default_value=False), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), ] diff --git a/module/sources/check_redfish/import_inventory.py b/module/sources/check_redfish/import_inventory.py index ee146732..b79ccf2e 100644 --- a/module/sources/check_redfish/import_inventory.py +++ b/module/sources/check_redfish/import_inventory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -9,6 +9,7 @@ import os import glob +import hashlib import json from packaging import version @@ -21,6 +22,11 @@ from module.netbox.inventory import NetBoxInventory from module.netbox import * +# NetBox stores dcim.modulebay.name at 64 chars. A longer name is shortened to a prefix plus a +# short digest of the full name, so two long slots sharing a prefix stay distinct bays. +MODULE_BAY_NAME_MAX_LENGTH = 64 +MODULE_BAY_NAME_HASH_LENGTH = 8 + log = get_logger() @@ -54,7 +60,14 @@ class CheckRedfish(SourceBase): NBVLANGroup, NBPowerPort, NBInventoryItem, - NBCustomField + NBCustomField, + # modules are always read back so that interfaces and power ports which reference a + # module can resolve that relation, even on a run where the option is off (e.g. after + # a user turns it off again). The option only gates whether we *create* components as + # modules, not whether we can read existing ones. + NBModuleBay, + NBModuleType, + NBModule ] source_type = "check_redfish" @@ -86,6 +99,10 @@ def __init__(self, name=None): self.interface_adapter_type_dict = dict() + # maps a network adapter id to the module bay name of its NIC module, so discovered + # ports can be attached to their parent module + self.nic_module_bay_by_adapter_id = dict() + def apply(self): """ Main source handler method. This method is called for each source from "main" program @@ -107,41 +124,8 @@ def apply(self): if self.read_inventory_file_content(filename) is False: continue - # try to get device by supplied NetBox id - inventory_id = grab(self.inventory_file_content, "meta.inventory_id") - - # parse inventory id to int as all NetBox ids are type integer - try: - inventory_id = int(inventory_id) - except (ValueError, TypeError): - log.warning(f"Value for meta.inventory_id '{inventory_id}' must be an integer. " - f"Cannot use inventory_id to match device in NetBox.") - - self.device_object = self.inventory.get_by_id(NBDevice, inventory_id) - - if self.device_object is not None: - log.debug2("Found a matching %s object '%s' based on inventory id '%d'" % - (self.device_object.name, - self.device_object.get_display_name(including_second_key=True), - inventory_id)) - - else: - # try to find device by serial of first system in inventory - device_serial = grab(self.inventory_file_content, "inventory.system.0.serial") - if self.device_object is None: - self.device_object = self.inventory.get_by_data(NBDevice, data={ - "serial": device_serial - }) - - if self.device_object is None: - log.error(f"Unable to find {NBDevice.name} with id '{inventory_id}' or " - f"serial '{device_serial}' in NetBox inventory from inventory file {filename}") - continue - else: - log.debug2("Found a matching %s object '%s' based on serial '%s'" % - (self.device_object.name, - self.device_object.get_display_name(including_second_key=True), - device_serial)) + if self.find_device_object(filename) is False: + continue # parse all components self.update_device() @@ -156,6 +140,78 @@ def apply(self): self.update_network_adapter() self.update_network_interface() + def find_device_object(self, filename): + """ + Match the current inventory file to a NetBox device and store it in self.device_object. + Tried in order: meta.inventory_id, the system serial, then the Dell Service Tag. + + Parameters + ---------- + filename: str + inventory file name, used for logging only + + Returns + ------- + bool: True if a matching device was found + """ + + supplied_id = grab(self.inventory_file_content, "meta.inventory_id") + inventory_id = None + + # bool is a subclass of int and float truncates, so int() would turn both `true` and + # `1.9` into the id of a real but unrelated device + if isinstance(supplied_id, bool) or not isinstance(supplied_id, (int, str)): + parsed_id = None + else: + try: + parsed_id = int(str(supplied_id).strip()) + except ValueError: + parsed_id = None + + if parsed_id is not None and parsed_id > 0: + inventory_id = parsed_id + elif supplied_id is not None: + # an absent id is the normal case, so only warn about a supplied one which is unusable + log.warning(f"Value for meta.inventory_id '{supplied_id}' must be a positive integer. " + f"Cannot use inventory_id to match device in NetBox.") + + self.device_object = None + if inventory_id is not None: + self.device_object = self.inventory.get_by_id(NBDevice, inventory_id) + + if self.device_object is not None: + log.debug2("Found a matching %s object '%s' based on inventory id '%d'" % + (self.device_object.name, + self.device_object.get_display_name(including_second_key=True), + inventory_id)) + return True + + # normalize as update_device() stores it, and never probe serial=None: get_by_data() + # compares dicts exactly, so it would match a device which has no serial at all + device_serial = get_string_or_none(grab(self.inventory_file_content, "inventory.system.0.serial")) + if device_serial is not None: + self.device_object = self.inventory.get_by_data(NBDevice, data={"serial": device_serial}) + + # unconditional, so disabling the option cannot strand a device already persisted + # with its Service Tag as the serial. get_service_tag() self-gates on the vendor. + if self.device_object is None: + service_tag = self.get_service_tag() + if service_tag is not None: + self.device_object = self.inventory.get_by_data(NBDevice, data={"serial": service_tag}) + if self.device_object is not None: + device_serial = service_tag + + if self.device_object is None: + log.error(f"Unable to find {NBDevice.name} with id '{inventory_id}' or " + f"serial '{device_serial}' in NetBox inventory from inventory file {filename}") + return False + + log.debug2("Found a matching %s object '%s' based on serial '%s'" % + (self.device_object.name, + self.device_object.get_display_name(including_second_key=True), + device_serial)) + return True + def reset_inventory_state(self): """ reset attributes to make sure not using data from a previous inventory file @@ -167,6 +223,7 @@ def reset_inventory_state(self): # reset interface types self.interface_adapter_type_dict = dict() + self.nic_module_bay_by_adapter_id = dict() def read_inventory_file_content(self, filename: str) -> bool: """ @@ -209,6 +266,22 @@ def read_inventory_file_content(self, filename: str) -> bool: return True + def get_service_tag(self): + """ + Return the Dell Service Tag (the chassis SKU), or None when this is not a Dell or no + Service Tag is reported. Shared so matching and updating agree on the value. + + Returns + ------- + (str, None): the Dell Service Tag + """ + + manufacturer = get_string_or_none(grab(self.inventory_file_content, "inventory.system.0.manufacturer")) + if manufacturer is None or "dell" not in manufacturer.lower(): + return None + + return get_string_or_none(grab(self.inventory_file_content, "inventory.chassis.0.sku")) + def update_device(self): system = grab(self.inventory_file_content, "inventory.system.0") @@ -217,7 +290,7 @@ def update_device(self): log.error(f"No system data found for '{self.device_object.get_display_name()}' in inventory file.") return - serial = get_string_or_none(grab(system, "serial")) + system_serial = get_string_or_none(grab(system, "serial")) name = get_string_or_none(grab(system, "host_name")) manufacturer = get_string_or_none(grab(system, "manufacturer")) @@ -234,13 +307,13 @@ def update_device(self): } } - if serial is not None: - device_data["serial"] = serial + serial = system_serial + if name is not None and self.settings.overwrite_host_name is True: device_data["name"] = name if "dell" in str(manufacturer).lower(): - chassis = grab(self.inventory_file_content, "inventory.chassis.0") - if chassis and "sku" in chassis: + service_tag = self.get_service_tag() + if service_tag is not None: # add ServiceTag self.add_update_custom_field({ @@ -253,11 +326,31 @@ def update_device(self): "description": "Dell Service Tag" }) - device_data["custom_fields"]["service_tag"] = chassis.get("sku") + device_data["custom_fields"]["service_tag"] = service_tag + + if grab(self.settings, "dell_serial_from_service_tag", fallback=False) is True: + serial = service_tag + + # custom fields are merged, not None-skipped, so a missing value would clear it + if system_serial is not None: + self.add_update_custom_field({ + "name": "system_serial", + "label": "System Serial Number", + "object_types": [ + "dcim.device" + ], + "type": "text", + "description": "System serial number reported by the BMC (Dell PPID)" + }) + + device_data["custom_fields"]["system_serial"] = system_serial else: log.warning(f"No chassis or sku data found for " f"'{self.device_object.get_display_name()}' in inventory file.") + if serial is not None: + device_data["serial"] = serial + self.device_object.update(data=device_data, source=self) def update_power_supply(self): @@ -272,6 +365,9 @@ def update_power_supply(self): ps_index = 1 ps_items = list() + # each power port with the bay name of its supply, linked after update_all_items creates + # the modules further down + power_port_links = list() for ps in grab(self.inventory_file_content, "inventory.power_supply", fallback=list()): if grab(ps, "operation_status") in ["NotPresent", "Absent"]: @@ -311,10 +407,12 @@ def update_power_supply(self): # compile inventory item data ps_items.append({ - "inventory_type": "Power Supply", "health": health_status, "description": description, + # the slot, not the AC/DC bearing display name, so a swap reuses the bay + "bay_name": ps_name, "full_name": name, + "model": model, "serial": get_string_or_none(grab(ps, "serial")), "manufacturer": get_string_or_none(grab(ps, "vendor")), "part_number": get_string_or_none(grab(ps, "part_number")), @@ -347,18 +445,29 @@ def update_power_supply(self): break if ps_object is None: - self.inventory.add_object(NBPowerPort, data=ps_data, source=self) + ps_object = self.inventory.add_object(NBPowerPort, data=ps_data, source=self) else: if self.settings.overwrite_power_supply_name is False: - del(ps_data["name"]) + del (ps_data["name"]) data_to_update = self.patch_data(ps_object, ps_data, self.settings.overwrite_power_supply_attributes) ps_object.update(data=data_to_update, source=self) current_ps.remove(ps_object) + power_port_links.append((ps_object, ps_name)) + ps_index += 1 - self.update_all_items(ps_items) + self.update_all_items(ps_items, "Power Supply") + + # NetBox cascade-deletes a module's components, so the port must follow its PSU module; + # detach a stale link when modules are off or no module resolves + for power_port, bay_name in power_port_links: + psu_module = self.find_device_module_by_bay_name(bay_name) if self.use_modules() is True else None + if psu_module is not None: + power_port.update(data={"module": psu_module}, source=self) + else: + power_port.unset_attribute("module") def update_fan(self): @@ -372,27 +481,19 @@ def update_fan(self): health_status = get_string_or_none(grab(fan, "health_status")) physical_context = get_string_or_none(grab(fan, "physical_context")) fan_id = get_string_or_none(grab(fan, "id")) - reading = get_string_or_none(grab(fan, "reading")) - reading_unit = get_string_or_none(grab(fan, "reading_unit")) - description = list() - speed = None if physical_context is not None: description.append(f"Context: {physical_context}") - if reading is not None and reading_unit is not None: - reading_unit = "%" if reading_unit.lower() == "percent" else reading_unit - speed = f"{reading}{reading_unit}" - + # a fan's reading is a live measurement, not something the inventory describes. + # Writing it would update this object in NetBox on every single run items.append({ - "inventory_type": "Fan", "description": description, "full_name": f"{fan_name} (ID: {fan_id})", - "health": health_status, - "speed": speed + "health": health_status }) - self.update_all_items(items) + self.update_all_items(items, "Fan") def update_memory(self): @@ -417,6 +518,9 @@ def update_memory(self): memory_size_total += size_in_mb + # the slot label is the stable bay identity, captured before the DIMM type is appended + dimm_bay = name + name_details = list() if dimm_type is not None: name_details.append(f"{dimm_type}") @@ -436,8 +540,8 @@ def update_memory(self): speed = f"{speed}MHz" items.append({ - "inventory_type": "DIMM", "description": description, + "bay_name": dimm_bay or "None", "full_name": name or "None", "serial": get_string_or_none(grab(memory, "serial")), "manufacturer": get_string_or_none(grab(memory, "manufacturer")), @@ -447,7 +551,7 @@ def update_memory(self): "speed": speed, }) - self.update_all_items(items) + self.update_all_items(items, "DIMM") if memory_size_total > 0: memory_size_total = memory_size_total / 1024 @@ -496,17 +600,19 @@ def update_proc(self): description.append(f"Threads: {threads}") items.append({ - "inventory_type": "CPU", "description": description, "manufacturer": get_string_or_none(grab(processor, "manufacturer")), + # the socket is the stable bay identity, independent of the installed model + "bay_name": socket, "full_name": name, + "model": model, "serial": get_string_or_none(grab(processor, "serial")), "health": health_status, "size": size, "speed": current_speed }) - self.update_all_items(items) + self.update_all_items(items, "CPU") if num_cores > 0: custom_fields_data = {"custom_fields": {"host_cpu_cores": f"{num_cores} {cpu_name}"}} @@ -548,6 +654,9 @@ def update_physical_drive(self): name = pd_name + # the drive slot is the stable bay identity, captured before type/model is appended + drive_bay = pd_name + name_details = list() if pd_type is not None: name_details.append(pd_type) @@ -568,9 +677,10 @@ def update_physical_drive(self): speed = f"{speed_in_rpm}RPM" items.append({ - "inventory_type": "Physical Drive", "description": description, "manufacturer": get_string_or_none(grab(pd, "manufacturer")), + "model": model, + "bay_name": drive_bay or "None", "full_name": name or "None", "serial": serial, "part_number": get_string_or_none(grab(pd, "part_number")), @@ -580,7 +690,7 @@ def update_physical_drive(self): "speed": speed }) - self.update_all_items(items) + self.update_all_items(items, "Physical Drive") def update_storage_controller(self): @@ -614,9 +724,9 @@ def update_storage_controller(self): size = f"{cache_size_in_mb}MB" items.append({ - "inventory_type": "Storage Controller", "description": description, "manufacturer": get_string_or_none(grab(sc, "manufacturer")), + "model": model, "full_name": name or "None", "serial": get_string_or_none(grab(sc, "serial")), "firmware": get_string_or_none(grab(sc, "firmware")), @@ -624,7 +734,7 @@ def update_storage_controller(self): "size": size }) - self.update_all_items(items) + self.update_all_items(items, "Storage Controller") def update_storage_enclosure(self): @@ -650,8 +760,8 @@ def update_storage_enclosure(self): size = f"Bays: {num_bays}" items.append({ - "inventory_type": "Storage Enclosure", "manufacturer": get_string_or_none(grab(se, "manufacturer")), + "model": model, "full_name": name or "None", "serial": get_string_or_none(grab(se, "serial")), "firmware": get_string_or_none(grab(se, "firmware")), @@ -659,7 +769,7 @@ def update_storage_enclosure(self): "size": size }) - self.update_all_items(items) + self.update_all_items(items, "Storage Enclosure") def update_network_adapter(self): @@ -706,13 +816,18 @@ def update_network_adapter(self): nic_type = NetBoxInterfaceType(name) + # the adapter id is the stable slot identity; adapter_name embeds a mutable label + stable_bay_name = adapter_id or adapter_name or "None" + if adapter_id is not None: self.interface_adapter_type_dict[adapter_id] = nic_type + self.nic_module_bay_by_adapter_id[adapter_id] = stable_bay_name items.append({ - "inventory_type": "NIC", "manufacturer": manufacturer, + "bay_name": stable_bay_name, "full_name": name, + "model": model, "serial": serial, "part_number": get_string_or_none(grab(adapter, "part_number")), "firmware": firmware, @@ -721,7 +836,39 @@ def update_network_adapter(self): "speed": nic_type.get_speed_human() }) - self.update_all_items(items) + self.update_all_items(items, "NIC") + + def find_device_module_by_bay_name(self, bay_name: str) -> NBModule: + """Return the module installed in the named bay on the current device, or None.""" + + if bay_name is None: + return None + + for module in self.inventory.get_all_items(NBModule): + if grab(module, "data.device") == self.device_object and \ + grab(module, "data.module_bay.data.name") == bay_name: + return module + + return None + + def interface_parent_module(self, adapter_id, mgmt_only: bool) -> NBModule: + """ + Determine the module a discovered interface belongs to: a management interface belongs to + the BMC/manager module, a regular NIC port to its network adapter's module. Returns None + when components are not modeled as modules or no matching module exists. + """ + + if self.use_modules() is not True: + return None + + if mgmt_only is True and self.manager_name is not None: + bay_name = self.manager_name + elif adapter_id is not None: + bay_name = self.nic_module_bay_by_adapter_id.get(adapter_id) + else: + bay_name = None + + return self.find_device_module_by_bay_name(bay_name) def update_network_interface(self): @@ -767,7 +914,18 @@ def update_network_interface(self): if wwn is not None: discovered_int_list.append(wwn) - if port_name is not None: + # a port belonging to a manager is a BMC port + mgmt_only = len(manager_ids) > 0 + + friendly_name = port_name + name_from_stable_id = False + + if self.use_modules() and mgmt_only is False and port_id is not None: + # the redfish id (e.g. NIC.Integrated.1-1) is stable; the long label moves to + # the description + port_name = port_id + name_from_stable_id = True + elif port_name is not None: port_name += f" ({port_id})" else: port_name = port_id @@ -778,14 +936,11 @@ def update_network_interface(self): link_type = NetBoxInterfaceType(link_speed) description = list() + if name_from_stable_id is True and friendly_name is not None and friendly_name != port_name: + description.append(friendly_name) if hostname is not None: description.append(f"Hostname: {hostname}") - mgmt_only = False - # if number of managers belonging to this port is not 0 then it's a BMC port - if len(manager_ids) > 0: - mgmt_only = True - # get enabled state enabled = False @@ -808,6 +963,10 @@ def update_network_interface(self): "health": health_status } + parent_module = self.interface_parent_module(adapter_id, mgmt_only) + if parent_module is not None: + port_data_dict[port_name]["module"] = parent_module + if len(description) > 0: port_data_dict[port_name]["description"] = ", ".join(description) if mgmt_only is True: @@ -841,10 +1000,15 @@ def update_network_interface(self): # get current object for this interface if it exists nic_object = data.get(port_name) + # clear a stale link when no parent module resolves, so a module prune cannot + # cascade-delete a port this source still manages + if nic_object is not None and "module" not in port_data: + nic_object.unset_attribute("module") + # unset "illegal" attributes for attribute in ["inventory_type", "health"]: if attribute in port_data: - del(port_data[attribute]) + del (port_data[attribute]) # del empty mac address attribute if port_data.get("mac_address") is None: @@ -857,7 +1021,7 @@ def update_network_interface(self): # create or update interface with data if nic_object is not None: if self.settings.overwrite_interface_name is False and port_data.get("name") is not None: - del(port_data["name"]) + del (port_data["name"]) this_link_type = port_data.get("type") mgmt_only = port_data.get("mgmt_only") @@ -875,7 +1039,9 @@ def update_network_interface(self): port_data = data_to_update - self.add_update_interface(nic_object, self.device_object, port_data, nic_ips.get(port_name, list())) + # redfish only reliably reports the BMC IP, never the host NIC / bond / bridge IPs + self.add_update_interface(nic_object, self.device_object, port_data, + nic_ips.get(port_name, list()), keep_undiscovered_ips=True) def update_manager(self): @@ -900,17 +1066,35 @@ def update_manager(self): description = f"Licenses: %s" % (", ".join(licenses)) items.append({ - "inventory_type": "Manager", "description": description, "full_name": name, + "model": model, "manufacturer": grab(self.device_object, "data.device_type.data.manufacturer.data.name"), "firmware": get_string_or_none(grab(manager, "firmware")), "health": get_string_or_none(grab(manager, "health_status")) }) - self.update_all_items(items) + self.update_all_items(items, "Manager") - def update_all_items(self, items): + def use_modules(self) -> bool: + """ + Decide if discovered hardware components should be modeled as NetBox modules + instead of the deprecated inventory items. + + Modules are only used if explicitly enabled via config AND the connected NetBox + instance is recent enough to support the modules data model (>= 4.3). + + Returns + ------- + bool: True if components should be modeled as modules + """ + + if grab(self.settings, "model_components_as_modules", fallback=False) is not True: + return False + + return version.parse(self.inventory.netbox_api_version) >= version.parse("4.3") + + def update_all_items(self, items, inventory_type): """ Updates all inventory items of a certain type. Both (current and supplied list of items) will be sorted by name and matched 1:1. @@ -919,6 +1103,9 @@ def update_all_items(self, items): ---------- items: list a list of items to update + inventory_type: str + the component type this batch is about (CPU, DIMM, Fan, ...), stated by the caller + so an empty batch still says which components are gone Returns ------- @@ -928,15 +1115,13 @@ def update_all_items(self, items): if not isinstance(items, list): raise ValueError(f"Value for 'items' must be type 'list' got: {items}") - if len(items) == 0: - return - - # get device - inventory_type = grab(items, "0.inventory_type") + # stamp the type so the lookup value and the stored value cannot drift apart + for item in items: + item["inventory_type"] = inventory_type - if inventory_type is None: - log.error(f"Unable to find inventory type for inventory item {items[0]}") - return + # model components as NetBox modules instead of the deprecated inventory items + if self.use_modules() is True: + return self.update_all_modules(items, inventory_type) # get current inventory items for this device and type current_inventory_items = dict() @@ -977,8 +1162,8 @@ def update_all_items(self, items): if len(unmatched_inventory_items) > 0: matched_inventory[nb_inventory_item] = unmatched_inventory_items.pop(0) - # set item health to absent if item can't be found in redfish inventory anymore - elif grab(nb_inventory_item, "data.custom_fields.health") != "Absent": + # unconditional: an object a run does not touch is tagged orphaned + else: nb_inventory_item.update(data={"custom_fields": {"health": "Absent"}}, source=self) # update items with matching NetBox inventory item @@ -1049,6 +1234,289 @@ def update_item(self, item_data: dict, inventory_object: NBInventoryItem = None) return + def get_current_modules_by_bay_name(self, inventory_type: str) -> dict: + """ + Collect all currently known modules of a certain component type for the current device, + keyed by the name of the module bay they are installed in. + + Parameters + ---------- + inventory_type: str + the component type to filter for (CPU, DIMM, Fan, ...) + + Returns + ------- + dict: module bay name -> NBModule, sorted by module bay name + """ + + current_modules = dict() + for module in self.inventory.get_all_items(NBModule): + if grab(module, "data.device") != self.device_object: + continue + if grab(module, "data.custom_fields.inventory_type") != inventory_type: + continue + + bay_name = grab(module, "data.module_bay.data.name") + if bay_name is not None: + current_modules[bay_name] = module + + return dict(sorted(current_modules.items())) + + def update_all_modules(self, items, inventory_type): + """ + Module based counterpart of 'update_all_items'. Updates all modules of a certain type. + Each component is represented by a module bay (the slot) holding a single module which is + typed by a module type (the catalog entry, e.g. the exact CPU/DIMM/NIC model). + + Both (current and supplied list of items) will be sorted by the module bay name and + matched 1:1, exactly like 'update_all_items' does for inventory items. + + Parameters + ---------- + items: list + a list of items to update + inventory_type: str + the component type this batch describes (CPU, DIMM, Fan, ...) + + Returns + ------- + None + """ + + # get current modules for this device and type, keyed by their module bay name + current_modules = self.get_current_modules_by_bay_name(inventory_type) + + # NB module object -> parsed data matching its module bay name + matched_modules = dict() + unmatched_module_items = list() + + # try to match items to existing modules by their stable module bay identity + for item in items: + + current_module = current_modules.get(self.module_bay_name(item)) + if current_module is not None: + matched_modules[current_module] = item + else: + unmatched_module_items.append(item) + + # sort unmatched items by module bay name for deterministic new-module creation order + unmatched_module_items.sort(key=lambda x: self.module_bay_name(x) or "") + + # strict by bay: update_module never moves a module, so an unmatched current module is a + # removed component, not a target to remap another component onto + for nb_module in current_modules.values(): + + if nb_module in matched_modules: + continue + + # unconditional: an object a run does not touch is tagged orphaned + nb_module.update(data={"custom_fields": {"health": "Absent"}}, source=self) + self.mark_module_bay_seen(nb_module) + + # update modules with matching NetBox module + for module_object, module_data in matched_modules.items(): + self.update_module(module_data, module_object) + + # create new module in NetBox + for unmatched_module_item in unmatched_module_items: + self.update_module(unmatched_module_item) + + def module_bay_name(self, item_data: dict) -> str: + """ + Return the stable module bay identity (the physical slot) for a component. + + The bay represents the slot, so it must be keyed on a stable identifier (CPU socket, + NIC slot, ...) that does not change when the installed part's model changes - otherwise + a model swap would rename the bay and churn it. Parsers provide it via 'bay_name'; we + fall back to the display name for components whose name is already slot based and does + not embed a model. + + The name is shortened to the module bay's max length (NetBox limits dcim.modulebay.name to + 64 chars). NetBox stores the shortened name, so the key used to match an existing bay must + be shortened the same way - otherwise a name longer than the limit never matches its stored + counterpart and the bay + module churn on every sync (the module path matches strictly, with + no alphabetical fallback like the inventory-item path has). Shortening keeps a prefix and + appends a deterministic hash of the full name so two distinct slots that happen to share the + first 64 chars (e.g. long drive/enclosure location strings) do not collapse onto one bay. + """ + + name = item_data.get("bay_name") or item_data.get("full_name") + if name is not None and len(name) > MODULE_BAY_NAME_MAX_LENGTH: + digest = hashlib.blake2s(name.encode("utf-8"), + digest_size=MODULE_BAY_NAME_HASH_LENGTH // 2).hexdigest() + prefix_length = MODULE_BAY_NAME_MAX_LENGTH - MODULE_BAY_NAME_HASH_LENGTH - 1 + name = f"{name[:prefix_length]}-{digest}" + return name + + def device_manufacturer_name(self) -> str: + """ + NetBox requires a manufacturer on every module type. Components like fans, PCIe extenders + or storage enclosures don't report one, so fall back to the device's own manufacturer + (the server vendor), or a generic placeholder when even that is unavailable. + """ + + device_manufacturer = grab(self.device_object, "data.device_type.data.manufacturer") + if isinstance(device_manufacturer, NetBoxObject): + return device_manufacturer.get_display_name() + + return "Unknown" + + def resolve_module_type(self, item_data: dict) -> NBModuleType: + """ + Find or create the module type (catalog entry) describing the installed part, e.g. the + exact CPU/DIMM/NIC model. Shared by create and update so a replaced part re-points to the + correct module type instead of keeping a stale reference. + """ + + part_number = item_data.get("part_number") + + # the module type model is the catalog identifier of the part (e.g. the exact CPU model) + # a type is a catalog entry shared by identical parts. Without a model or part number + # the component class is the closest thing to one; the instance name would create a new + # type for every fan and drive in the fleet + model = item_data.get("model") or part_number or item_data.get("inventory_type") or \ + item_data.get("full_name") + module_type_data = {"model": model} + if part_number is not None: + module_type_data["part_number"] = part_number + + # NetBox requires a manufacturer: redfish, then the existing type's own value, then + # the device vendor + manufacturer = item_data.get("manufacturer") + if manufacturer is None: + existing_module_type = self.inventory.get_by_data(NBModuleType, data={"model": model}) + if existing_module_type is None or grab(existing_module_type, "data.manufacturer") is None: + manufacturer = self.device_manufacturer_name() + + if manufacturer is not None: + module_type_data["manufacturer"] = {"name": manufacturer} + + return self.inventory.add_update_object(NBModuleType, data=module_type_data, source=self) + + def update_module(self, item_data: dict, module_object: NBModule = None): + """ + Updates a single module with the supplied data. If no module is provided a new module bay, + module type and module will be created (see 'create_module'). + + Parameters + ---------- + item_data: dict + a dict with data for the component to update + module_object: NBModule, None + the NetBox module to update. + + Returns + ------- + None + """ + + description = item_data.get("description") + if isinstance(description, list): + description = ", ".join(description) + + # custom fields tracked on the module itself + module_custom_fields = { + "firmware": item_data.get("firmware"), + "health": item_data.get("health"), + "inventory_type": item_data.get("inventory_type"), + "inventory_size": item_data.get("size"), + "inventory_speed": item_data.get("speed") + } + + # create a new module (incl. its module bay and module type) + if module_object is None: + self.create_module(item_data, description, module_custom_fields) + return + + # the bay is the slot the module sits in and is still present, so mark it seen too + self.upsert_module_bay(item_data, description) + + # update an existing module; re-point the module type in case the installed part was + # replaced with a different model in the same bay + module_data = { + "custom_fields": module_custom_fields, + "module_type": self.resolve_module_type(item_data) + } + if item_data.get("serial") is not None: + module_data["serial"] = item_data.get("serial") + if description is not None and len(description) > 0: + module_data["description"] = description + + module_object.update(data=module_data, source=self) + + def upsert_module_bay(self, item_data: dict, description: str) -> NBModuleBay: + """ + Add or update the module bay (the physical slot) of a component and mark it as seen by + this source. + + Both the create and the update path go through here. tag_all_the_things() adds the + orphaned tag to every object carrying the primary tag whose source is unset after a run, + so a bay that a run never touches is tagged orphaned even while the module installed in + it stays healthy. + """ + + module_bay_data = { + "device": self.device_object, + "name": self.module_bay_name(item_data) + } + if item_data.get("label") is not None: + module_bay_data["label"] = item_data.get("label") + if description is not None and len(description) > 0: + module_bay_data["description"] = description + + return self.inventory.add_update_object(NBModuleBay, data=module_bay_data, source=self) + + def mark_module_bay_seen(self, module_object: NBModule) -> None: + """ + Register the bay a module sits in with this source without changing it. The slot outlives + the component installed in it, so it must not be orphan tagged once that component is gone. + """ + + module_bay = grab(module_object, "data.module_bay") + if module_bay is None: + return + + module_bay.update(data={"name": grab(module_bay, "data.name")}, source=self) + + def create_module(self, item_data: dict, description: str, module_custom_fields: dict): + """ + Create a new module for a discovered component. This creates (or reuses) the module type + (catalog entry), the module bay (the physical slot) and the module installed in that bay. + + Parameters + ---------- + item_data: dict + a dict with data for the component to create + description: str + the already compiled description string for this component + module_custom_fields: dict + the custom fields to store on the module + """ + + serial = item_data.get("serial") + has_description = description is not None and len(description) > 0 + + module_type = self.resolve_module_type(item_data) + + # the module bay represents the physical slot the component lives in; it is keyed on a + # stable slot identifier so a later model swap reuses the same bay instead of churning it + module_bay = self.upsert_module_bay(item_data, description) + + # the module is the actual installed component + module_data = { + "device": self.device_object, + "module_bay": module_bay, + "module_type": module_type, + "status": "active", + "custom_fields": module_custom_fields + } + if serial is not None: + module_data["serial"] = serial + if has_description is True: + module_data["description"] = description + + self.inventory.add_object(NBModule, data=module_data, source=self) + def add_necessary_base_objects(self): """ Adds/updates source tag and all custom fields necessary for this source. @@ -1060,6 +1528,10 @@ def add_necessary_base_objects(self): "description": f"Marks objects synced from check_redfish inventory '{self.name}' to this NetBox Instance." }) + # components are stored as modules (NetBox >= 4.3) or as the deprecated inventory items, + # so their custom fields must follow that choice + component_object_type = "dcim.module" if self.use_modules() is True else "dcim.inventoryitem" + self.add_update_custom_field({ "name": "host_cpu_cores", "label": "Physical CPU Cores", @@ -1095,7 +1567,7 @@ def add_necessary_base_objects(self): "name": "firmware", "label": "Firmware", "object_types": [ - "dcim.inventoryitem", + component_object_type, "dcim.powerport" ], "type": "text", @@ -1106,7 +1578,7 @@ def add_necessary_base_objects(self): self.add_update_custom_field({ "name": "inventory_type", "label": "Type", - "object_types": ["dcim.inventoryitem"], + "object_types": [component_object_type], "type": "text", "description": "Describes the type of inventory item" }) @@ -1115,7 +1587,7 @@ def add_necessary_base_objects(self): self.add_update_custom_field({ "name": "inventory_size", "label": "Size", - "object_types": ["dcim.inventoryitem"], + "object_types": [component_object_type], "type": "text", "description": "Describes the size of the inventory item if applicable" }) @@ -1124,7 +1596,7 @@ def add_necessary_base_objects(self): self.add_update_custom_field({ "name": "inventory_speed", "label": "Speed", - "object_types": ["dcim.inventoryitem"], + "object_types": [component_object_type], "type": "text", "description": "Describes the speed of the inventory item if applicable" }) @@ -1134,7 +1606,7 @@ def add_necessary_base_objects(self): "name": "health", "label": "Health", "object_types": [ - "dcim.inventoryitem", + component_object_type, "dcim.powerport", "dcim.device" ], diff --git a/module/sources/common/config.py b/module/sources/common/config.py index 115bc70b..d8d81488 100644 --- a/module/sources/common/config.py +++ b/module/sources/common/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/module/sources/common/handle_vlan.py b/module/sources/common/handle_vlan.py index b93253c9..ff5e1b79 100644 --- a/module/sources/common/handle_vlan.py +++ b/module/sources/common/handle_vlan.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -122,6 +122,8 @@ def __init__(self, vlan, filter_type="exclude"): def matches(self, vlan_id, site=None): + # FIXME: + # * site_name is not defined if self.site_matches(site) is False: log.debug2(f"VLAN {self.filter_type} site name '{site_name}' matches '{self.site}'") return False diff --git a/module/sources/common/permitted_subnets.py b/module/sources/common/permitted_subnets.py index 82cf30dc..e3b5c715 100644 --- a/module/sources/common/permitted_subnets.py +++ b/module/sources/common/permitted_subnets.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -30,8 +30,15 @@ def __init__(self, config_string: str): log.info(f"Config option 'permitted_subnets' is undefined. No IP addresses will be populated to NetBox!") return + # a yaml config may define the subnets as a list instead of a comma separated string + if isinstance(config_string, list): + config_string = ", ".join(str(x) for x in config_string) + if not isinstance(config_string, str): - raise ValueError("permitted subnets need to be of type string") + log.error(f"permitted subnets need to be a comma separated string or a list, " + f"got {type(config_string).__name__}: {config_string}") + self._validation_failed = True + return subnet_list = [x.strip() for x in config_string.split(",") if x.strip() != ""] diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 4d20e9d2..df098e78 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -42,10 +42,59 @@ def implements(cls, source_type): return False + # NetBox exempts addresses with these roles from its uniqueness check, they are + # meant to exist on more than one interface at the same time + non_unique_ip_roles = ("anycast", "vip", "vrrp", "hsrp", "glbp", "carp") + # stub function to implement a finish call for each source def finish(self): pass + def ip_is_primary_ip_of_object(self, ip_object, device_vm_object) -> bool: + """ + Check if a NBIPAddress object is currently set as primary IPv4 or IPv6 + address of a NBDevice or NBVM object. + + Parameters + ---------- + ip_object: NBIPAddress + IP address object to check + device_vm_object: NBDevice | NBVM + device or VM object to compare the primary IPs of + + Returns + ------- + bool: True if 'ip_object' is set as 'primary_ip4' or 'primary_ip6' + """ + + if not isinstance(ip_object, NBIPAddress) or not isinstance(device_vm_object, (NBDevice, NBVM)): + return False + + ip_address = grab(ip_object, "data.address") + + for primary_ip_key in ["primary_ip4", "primary_ip6"]: + + primary_ip = grab(device_vm_object, f"data.{primary_ip_key}") + + if primary_ip is None: + continue + + if primary_ip is ip_object: + return True + + primary_ip_address = None + if isinstance(primary_ip, NBIPAddress): + primary_ip_address = grab(primary_ip, "data.address") + elif isinstance(primary_ip, dict): + primary_ip_address = primary_ip.get("address") + elif isinstance(primary_ip, int): + primary_ip_address = grab(self.inventory.get_by_id(NBIPAddress, nb_id=primary_ip), "data.address") + + if primary_ip_address is not None and primary_ip_address == ip_address: + return True + + return False + def map_object_interfaces_to_current_interfaces(self, device_vm_object, interface_data_dict=None, append_unmatched_interfaces=False): """ @@ -68,6 +117,10 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac ens1 > vNIC 3 ... > ... + Current interfaces whose name matches the source setting 'vm_interface_exclude_filter' + (or 'host_interface_exclude_filter' for devices) are excluded from all matching + attempts and will therefore never be altered. + Parameters ---------- device_vm_object: (NBDevice, NBVM) @@ -96,6 +149,11 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac log.debug2("Trying to match current object interfaces in NetBox with discovered interfaces") + if isinstance(device_vm_object, NBVM): + interface_exclude_filter = self.settings.vm_interface_exclude_filter + else: + interface_exclude_filter = self.settings.host_interface_exclude_filter + current_object_interfaces = { "virtual": dict(), "physical": dict() @@ -109,6 +167,12 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac for interface in self.inventory.get_all_interfaces(device_vm_object): int_mac = grab(interface, "data.mac_address") int_name = grab(interface, "data.name") + + if interface_exclude_filter is not None and int_name is not None and \ + interface_exclude_filter.match(int_name): + log.debug2(f"Current interface '{int_name}' matches interface_exclude_filter. " + f"Excluding it from all interface matching attempts") + continue int_type = "virtual" if "virtual" not in str(grab(interface, "data.type", fallback="virtual")): int_type = "physical" @@ -232,7 +296,7 @@ def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None return current_longest_matching_prefix def add_update_interface(self, interface_object, device_object, interface_data, interface_ips=None, - vmware_object=None): + vmware_object=None, keep_undiscovered_ips=False): """ Adds/Updates an interface to/of a NBVM or NBDevice including IP addresses. Validates/enriches data in following order: @@ -257,6 +321,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, a list of ip addresses which are assigned to this interface vmware_object: vim.HostSystem | vim.VirtualMachine object to add to list of objects to reevaluate + keep_undiscovered_ips: bool + if True, keep the existing IPs of an interface the source discovered no IPs for Returns ------- @@ -347,8 +413,9 @@ def add_update_interface(self, interface_object, device_object, interface_data, # if a new interface or not matching assigned MAC address, try to find an existing unassigned mac address if primary_mac_address_object is None: for mac_address_object in self.inventory.get_all_items(NBMACAddress): + # an object already assigned to this very interface is the one we want, not a duplicate if (grab(mac_address_object, "data.mac_address") == interface_mac_address and - grab(mac_address_object, "data.assigned_object_id") is None): + grab(mac_address_object, "data.assigned_object_id") in (None, interface_object)): primary_mac_address_object = mac_address_object break @@ -439,11 +506,33 @@ def add_update_interface(self, interface_object, device_object, interface_data, log.warning(f"{matching_ip_prefix.name} got wrong format. Unable to add IP address to NetBox") continue + # If skip_fhrp_group_ips is set and this address is already assigned to an FHRP + # group in NetBox, leave it on the FHRP group instead of rebinding it to this + # interface (issue #445). The whole inventory is scanned for a match on this + # address, so the outcome does not depend on inventory order, and only a + # candidate in the same VRF counts, so an unrelated FHRP-group IP does not cause + # this address to be skipped and every regular IP to be unbound (issue #476). + if self.settings.skip_fhrp_group_ips: + skip_fhrp_ip = False + for ip in self.inventory.get_all_items(NBIPAddress): + if grab(ip, "data.assigned_object_type", fallback="") != "ipam.fhrpgroup": + continue + if not grab(ip, "data.address", fallback="").startswith(f"{ip_object.ip.compressed}/"): + continue + if possible_ip_vrf != grab(ip, "data.vrf"): + continue + log.info(f"IP address '{grab(ip, 'data.address')}' is assigned to an FHRP Group and " + f"skip_fhrp_group_ips is set to True, skipping.") + skip_fhrp_ip = True + break + if skip_fhrp_ip is True: + continue + # try to find matching IP address object this_ip_object = None skip_this_ip = False + non_unique_ip_role = None for ip in self.inventory.get_all_items(NBIPAddress): - # check if address matches (without prefix length) ip_address_string = grab(ip, "data.address", fallback="") @@ -487,6 +576,16 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = ip break + # an address whose role marks it as non unique belongs on several interfaces at + # the same time, so this interface gets an object of its own rather than taking + # this one over. Keep looking, this interface may already have its own object + current_ip_role = ip.get_role() + if current_ip_role in self.non_unique_ip_roles: + log.debug(f"{ip.name} '{ip.get_display_name()}' is a '{current_ip_role}' address and " + f"can be assigned to multiple interfaces at the same time.") + non_unique_ip_role = current_ip_role + continue + # get current IP interface status current_nic_enabled = grab(current_ip_nic, "data.enabled", fallback=True) this_nic_enabled = grab(interface_object, "data.enabled", fallback=True) @@ -509,12 +608,6 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = ip - if grab(ip, "data.role.value") == "anycast": - log.debug(f"{ip.name} '{ip.get_display_name()}' is an Anycast address and " - f"can be assigned to multiple interfaces at the same time.") - skip_this_ip = True - break - if current_nic_enabled == this_nic_enabled: this_log_handler = log.warning @@ -580,6 +673,10 @@ def add_update_interface(self, interface_object, device_object, interface_data, nic_ip_data["tenant"] = ip_tenant if not isinstance(this_ip_object, NBIPAddress): + + if non_unique_ip_role is not None: + nic_ip_data["role"] = non_unique_ip_role + log.debug(f"No existing {NBIPAddress.name} object found. Creating a new one.") this_ip_object = self.inventory.add_object(NBIPAddress, data=nic_ip_data, source=self) @@ -593,17 +690,41 @@ def add_update_interface(self, interface_object, device_object, interface_data, ip_address_objects.append(this_ip_object) + # keyed on what the source reported, not on what survived parsing: an unusable address + # is still a statement that the interface was seen + skip_ip_removal = keep_undiscovered_ips is True and len(interface_ips or list()) == 0 + + # guest tools which report as running but hand back no interface at all are broken + # (seen on old TMOS releases), not a statement that every address is gone. Keep what is + # in NetBox instead of tearing it off on every run. A real removal still reports the + # interface, just without an address, so that case is unaffected + reported_interfaces = grab(vmware_object, "guest.net") + if type(device_object) == NBVM and isinstance(reported_interfaces, list) and \ + len(reported_interfaces) == 0: + log.debug(f"VM '{device_object.name}' guest tools report no network interface at all, " + f"keeping the addresses currently assigned in NetBox") + skip_ip_removal = True + for current_ip in interface_object.get_ip_addresses(): - if skip_ip_handling is True: + if skip_ip_handling is True or skip_ip_removal is True: continue - if grab(current_ip, "data.role.value") == "anycast": - log.debug2(f"{current_ip.name} '{current_ip.get_display_name()}' is an Anycast address and will " - f"NOT be deleted from interface") + current_ip_role = current_ip.get_role() + if current_ip_role in self.non_unique_ip_roles: + log.debug2(f"{current_ip.name} '{current_ip.get_display_name()}' is a '{current_ip_role}' address " + f"and will NOT be deleted from interface") continue if current_ip not in ip_address_objects: + + if bool(self.settings.preserve_primary_ips) is True and \ + self.ip_is_primary_ip_of_object(current_ip, device_object): + log.debug(f"{current_ip.name} '{current_ip.get_display_name()}' is the primary IP of " + f"'{device_object.get_display_name()}' and 'preserve_primary_ips' is enabled. " + f"NOT removing it from this interface") + continue + log.info(f"{current_ip.name} is no longer assigned to {interface_object.get_display_name()} and " f"therefore removed from this interface") current_ip.remove_interface_association() @@ -884,7 +1005,8 @@ def get_vlan_object_if_exists(self, vlan_data=None, vlan_site=None, vlan_cluster # try find matching VLAN by group if grab(vlan, "data.group") is not None: vlan_group = grab(vlan, "data.group") - if vlan_group.matches_site_cluster(vlan_site, vlan_cluster): + if isinstance(vlan_group, NetBoxObject) and \ + vlan_group.matches_site_cluster(vlan_site, vlan_cluster): vlan_object_by_group = vlan break diff --git a/module/sources/hetzner/__init__.py b/module/sources/hetzner/__init__.py new file mode 100644 index 00000000..8e9c3cc8 --- /dev/null +++ b/module/sources/hetzner/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from module.sources.hetzner.connection import HetznerHandler \ No newline at end of file diff --git a/module/sources/hetzner/client.py b/module/sources/hetzner/client.py new file mode 100644 index 00000000..4244d2d2 --- /dev/null +++ b/module/sources/hetzner/client.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from hcloud import Client + + +class HetznerClient: + + def __init__(self, token): + self.client = Client(token=token) + + def get_servers(self): + return self.client.servers.get_all() diff --git a/module/sources/hetzner/config.py b/module/sources/hetzner/config.py new file mode 100644 index 00000000..27206df3 --- /dev/null +++ b/module/sources/hetzner/config.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from module.config import source_config_section_name +from module.config.base import ConfigBase +from module.config.option import ConfigOption + + +class HetznerConfig(ConfigBase): + + section_name = source_config_section_name + source_name_example = "my-hetzner-example" + + def __init__(self): + self.options = [ + + ConfigOption( + "enabled", + bool, + default_value=True, + description="Enable or disable the Hetzner Cloud source." + ), + + ConfigOption( + "type", + str, + default_value="hetzner", + description="Source type identifier. Must remain 'hetzner'." + ), + + ConfigOption( + "api_token", + str, + mandatory=True, + description="Hetzner Cloud API token used to authenticate against the Hetzner Cloud API." + ), + ] + + super().__init__() diff --git a/module/sources/hetzner/connection.py b/module/sources/hetzner/connection.py new file mode 100644 index 00000000..cbd23aa0 --- /dev/null +++ b/module/sources/hetzner/connection.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from module.common.logging import get_logger +from module.sources.common.source_base import SourceBase +from module.sources.hetzner.client import HetznerClient +from module.sources.hetzner.config import HetznerConfig +from module.sources.hetzner.network import sync_vm_network +from module.sources.hetzner.disk import sync_vm_disks + +log = get_logger() + + +from module.netbox.inventory import ( + NetBoxInventory, + NBVM, + NBSite, + NBCluster, + NBClusterType, + NBVMInterface, + NBIPAddress, + NBVirtualDisk, +) + + +class HetznerHandler(SourceBase): + + source_type = "hetzner" + source_tag = "hetzner" + + settings = HetznerConfig() + + dependent_netbox_objects = [ + NBVM, + NBCluster, + NBSite, + NBClusterType, + NBIPAddress, + NBVirtualDisk, + NBVMInterface, + ] + + def __init__(self, name=None): + + if name is None: + raise ValueError(f"Invalid value for attribute 'name': '{name}'.") + + self.inventory = NetBoxInventory() + self.name = name + self.log = get_logger() + self.client = None + + settings_handler = HetznerConfig() + settings_handler.source_name = self.name + self.settings = settings_handler.parse() + + self.set_source_tag() + + if self.settings.enabled is False: + log.info(f"Source '{name}' is currently disabled. Skipping") + return + + self.init_successful = True + + @classmethod + def implements(cls, source_type): + return source_type == "hetzner" + + def apply(self): + + token = self.settings.api_token + + if not token: + self.log.error("Hetzner api_token not defined in settings.ini") + return + + self.client = HetznerClient(token=token) + + servers = self.client.get_servers() + + self.log.info(f"Connected to Hetzner, found {len(servers)} servers") + + # --------------------------- + # main object + # --------------------------- + + site = self.inventory.add_update_object( + NBSite, + data={"name": "cloud"}, + source=self, + ) + + cluster_type = self.inventory.add_update_object( + NBClusterType, + data={"name": "cloud"}, + source=self, + ) + + cluster_name = f"Hetzner: {self.name}" + + cluster = self.inventory.add_update_object( + NBCluster, + data={ + "name": cluster_name, + "type": cluster_type, + "scope_type": 17, + "scope_id": site, + }, + source=self, + ) + + # --------------------------- + # servers loop + # --------------------------- + + for server in servers: + + # -------- VM -------- + vm = self.inventory.add_update_object( + NBVM, + data={ + "name": server.name, + "status": "active", + "cluster": cluster, + "site": site, + }, + source=self, + ) + + # -------- interfaces -------- + sync_vm_network(self, vm, server) + + # -------- disks -------- + sync_vm_disks(self, vm, server) diff --git a/module/sources/hetzner/disk.py b/module/sources/hetzner/disk.py new file mode 100644 index 00000000..0b8e7531 --- /dev/null +++ b/module/sources/hetzner/disk.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from module.netbox.inventory import NBVirtualDisk + + +def sync_vm_disks(handler, vm, server): + """ + Sync Hetzner volumes → NetBox virtual disks + """ + + inventory = handler.inventory + + if not server.volumes: + return + + for volume in server.volumes: + + disk_name = f"{server.name}-{volume.name}"[:60] + size_mb = int(volume.size) * 1024 # Hetzner size = GB + + disk_data = { + "name": disk_name, + "virtual_machine": vm, # object, не id + "size": size_mb, + } + + existing_disk = None + + for disk in inventory.get_all_items(NBVirtualDisk): + if ( + disk.data.get("name") == disk_name + and disk.data.get("virtual_machine") == vm + ): + existing_disk = disk + break + + if existing_disk is None: + inventory.add_object( + NBVirtualDisk, + data=disk_data, + source=handler, + ) + else: + existing_disk.update(disk_data, source=handler) diff --git a/module/sources/hetzner/network.py b/module/sources/hetzner/network.py new file mode 100644 index 00000000..a2a93d0c --- /dev/null +++ b/module/sources/hetzner/network.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +from module.netbox.inventory import NBVMInterface, NBIPAddress + + +def sync_vm_network(handler, vm, server): + """ + Create interfaces + assign IPs for Hetzner VM + """ + + inventory = handler.inventory + interfaces = [] + + # ----------------------- + # interfaces + # ----------------------- + + # public → eth0 + if server.public_net and server.public_net.ipv4: + iface = inventory.add_update_object( + NBVMInterface, + data={ + "name": "eth0", + "virtual_machine": vm, + "enabled": True, + }, + source=handler, + ) + interfaces.append(iface) + + # private → ethX + if server.private_net: + start_index = 1 if len(interfaces) > 0 else 0 + + for idx, net in enumerate(server.private_net, start=start_index): + iface = inventory.add_update_object( + NBVMInterface, + data={ + "name": f"eth{idx}", + "virtual_machine": vm, + "enabled": True, + }, + source=handler, + ) + interfaces.append(iface) + + # ----------------------- + # IP assignment + # ----------------------- + + # public ip + if server.public_net and server.public_net.ipv4 and len(interfaces) >= 1: + ip_addr = server.public_net.ipv4.ip + if "/" not in ip_addr: + ip_addr += "/32" + + assign_ip(inventory, handler, ip_addr, interfaces[0]) + + # private ips + if server.private_net: + private_start_index = 1 if (server.public_net and server.public_net.ipv4) else 0 + + for idx, net in enumerate(server.private_net, start=private_start_index): + + if len(interfaces) <= idx: + continue + + ip_addr = net.ip + if "/" not in ip_addr: + ip_addr += "/32" + + assign_ip(inventory, handler, ip_addr, interfaces[idx]) + + +def assign_ip(inventory, handler, ip_addr, interface): + """ + Safe IP assign without duplicates + """ + + ip_data = { + "address": ip_addr, + "assigned_object_type": "virtualization.vminterface", + "assigned_object_id": interface, + } + + existing_ip = next( + (ip for ip in inventory.get_all_items(NBIPAddress) + if ip.data.get("address") == ip_addr), + None + ) + + if existing_ip is None: + inventory.add_object(NBIPAddress, data=ip_data, source=handler) + else: + existing_ip.update(ip_data, source=handler) diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index e4d8e2bc..72e32ac9 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -89,7 +89,7 @@ def __init__(self): If a filter is unset it will be ignored. Filters are all treated as regex expressions! If more then one expression should match, a '|' needs to be used """, - config_example="""Example: (exclude all VMs with "replica" in their name + config_example="""Example: (exclude all VMs with "replica" in their name and all VMs starting with "backup"): vm_exclude_filter = .*replica.*|^backup.*""", options=[ ConfigOption("cluster_exclude_filter", @@ -116,7 +116,22 @@ def __init__(self): """, config_example="tag-a, tag-b" ), - + ConfigOption("vm_exclude_disk_sync", + str, + description="""defines a comma separated list of VM names (regex) where disk synchronization + will be excluded. A VM matching this filter will still be synced to NetBox, + but its disk information won't be updated. + """, + config_example="backup-.*, temp-.*" + ), + ConfigOption("vm_exclude_disk_sync_by_tag", + str, + description="""defines a comma separated list of vCenter tags which (if assigned to a VM) + will exclude this VM from disk synchronization. A VM with this tag will still be synced + to NetBox, but its disk information won't be updated. + """, + config_example="backup-vm, veeam-job" + ), ConfigOptionGroup(title="relations", options=[ ConfigOption("cluster_site_relation", @@ -143,6 +158,27 @@ def __init__(self): description="""Same as cluster site but on host level. If unset it will fall back to cluster_site_relation""", config_example="nyc02.* = New York, ffm01.* = Frankfurt"), + ConfigOption("cluster_scope_type_relation", + str, + description="""This option defines the scope type for a cluster. + The scope type can be 'dcim.site', 'dcim.sitegroup', 'dcim.location' or 'dcim.region'. + This is done with a comma separated key = value list. + Can be set to "" to not assign a scope type. + Note: this does not remove scope types from existing clusters in NetBox. + key: defines a cluster name as regex + value: defines the NetBox scope type name (use quotes if name contains commas) + """, + config_example="Cluster_NYC = dcim.site, Cluster_FFM = dcim.sitegroup, Cluster_BER = dcim.location"), + ConfigOption("cluster_scope_id_relation", + str, + description="""This option defines the scope id for a cluster. + The scope id is the NetBox ID of the scope type. + This is done with a comma separated key = value list. + To be used in combination with the 'cluster_scope_type_relation'. + key: defines a cluster name as regex + value: defines the NetBox scope id (use quotes if name contains commas) + """, + config_example="Cluster_NYC = 1, Cluster_FFM.* = 2, Cluster_BER = 7"), ConfigOption("cluster_tenant_relation", str, description="""\ @@ -202,6 +238,31 @@ def __init__(self): description="""Try to find existing host based on serial number. This can cause issues with blade centers if VMWare does not report the blades serial number properly.""", default_value=True), + + ConfigOption("match_vm_by_serial", + bool, + description="""Fall back to matching VMs by serial number (BIOS UUID) if no name+cluster + match is found. Can misattribute a VM to an unrelated NetBox object if the same UUID is + reported by multiple sources, e.g. a cloned/migrated VM whose stale copy overwrites the + real VM's cluster/site/status.""", + default_value=True), + + ConfigOption("match_vm_by_mac_address", + bool, + description="""Fall back to matching VMs by vNIC MAC address if no name+cluster match is + found. Runs before 'match_vm_by_serial', so disabling that option alone is not enough if + MACs are also shared. Same misattribution risk as match_vm_by_serial, triggered by a + cloned/copied VM with a duplicate MAC.""", + default_value=True), + + ConfigOption("match_vm_by_ip_address", + bool, + description="""Fall back to matching VMs by primary IP if no name/cluster/MAC/serial + match is found. Same misattribution risk, triggered even transiently, e.g. a duplicate VM + in another cluster briefly powered on with the same IP. Not guaranteed to self-correct + afterwards, since vCenter can keep reporting a cached IP after power-off.""", + default_value=True), + ConfigOption("collect_hardware_asset_tag", bool, description="Attempt to collect asset tags from vCenter hosts", @@ -238,6 +299,15 @@ def __init__(self): as "when-undefined" """, default_value="when-undefined"), + ConfigOption("preserve_primary_ips", + bool, + description="""defines if primary IP addresses of devices and VMs are protected from + removal. If enabled, an IP address which is set as primary IPv4/IPv6 of a device or + VM in NetBox will never be removed from its interface by this source, even if the + source does not report this IP address (anymore). This prevents the primary IP from + being unset when i.e. an outdated guest agent does not report all IP addresses. + """, + default_value=False), ConfigOption("skip_vm_comments", bool, description="Do not sync notes from a VM in vCenter to the comments field on a VM in netbox", @@ -258,6 +328,30 @@ def __init__(self): description="""If the VMware Site Recovery Manager is used to can skip syncing placeholder/replicated VMs from fail-over site to NetBox.""", default_value=False), + ConfigOption("skip_fhrp_group_ips", + bool, + description="""If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) + then this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", + default_value=False), + ConfigOption("vm_status_on_create", + str, + description="""defines the status a VM gets assigned in NetBox when netbox-sync + creates it as a new NetBox VM. Updates of already existing NetBox VMs are not + affected by this option. This way new VMs can start their lifecycle in NetBox + as i.e. 'planned' until changed manually in NetBox. + possible values: offline, active, planned, staged, failed, decommissioning + """, + config_example="planned"), + ConfigOption("vm_status_preserve", + str, + description="""defines a comma separated list of NetBox VM statuses which will be + preserved on updates. If the current status of an existing NetBox VM matches one of + these values then netbox-sync will not change the status of this VM. This way VMs + can be kept in i.e. 'planned' or 'staged' until changed manually in NetBox. + Set to an empty value to always update the VM status. + possible values: offline, active, planned, staged, failed, decommissioning + """, + config_example="planned, staged, decommissioning"), ConfigOption("strip_host_domain_name", bool, description="strip domain part from host name before syncing device to NetBox", @@ -269,7 +363,7 @@ def __init__(self): ConfigOptionGroup(title="tag source", description="""\ sync tags assigned to clusters, hosts and VMs in vCenter to NetBox - INFO: this requires the installation of the 'vsphere-automation-sdk', + INFO: this requires the installation of the 'vcf-sdk' package, see docs about installation possible values: * object : the host or VM itself * parent_folder_1 : the direct folder this object is organized in (1 level up) @@ -284,6 +378,17 @@ def __init__(self): ConfigOption("host_tag_source", str), ConfigOption("vm_tag_source", str) ]), + ConfigOption("tag_name_include_category", + bool, + description="""\ + If enabled, vCenter tag names synced to NetBox will include the vCenter category as a + prefix in the format 'CategoryName:TagName'. Useful if TagName and CategoryName is used + as key/value pairs in vCenter. + When changed, existing synced tags are replaced on + the next run. Note: vm_exclude_by_tag_filter entries must use 'CategoryName:TagName' + format when this option is enabled. + """, + default_value=False), ConfigOption("sync_custom_attributes", bool, description="""sync custom attributes defined for hosts and VMs @@ -305,6 +410,19 @@ def __init__(self): str, config_example="config.uuid") ]), + ConfigOption("vm_guest_hostname_custom_field", + str, + description="""defines the name of a NetBox custom field which is used to store the + hostname reported by VMware Tools from inside the guest OS (vCenter property + 'guest.hostName'). This is independent of the vCenter VM inventory name and can be + used to detect naming drift between the vCenter VM name and the actual OS hostname. + The custom field must be of type "Text" and assigned to the "Virtual Machine" object + type. If it does not exist yet, it will be created automatically, the same way other + netbox-sync managed custom fields are created. + If this option is unset (default) the guest hostname is not synced. + If VMware Tools does not report a hostname (not installed, not running or no data yet) + the custom field is left untouched so any previously synced value is preserved.""", + config_example="vmware_guest_hostname"), ConfigOption("set_source_name_as_cluster_group", bool, description="""this will set the sources name as cluster group name instead of the datacenter. @@ -357,8 +475,8 @@ def __init__(self): ConfigOption("track_vm_host", bool, - description="""enabling this option will add the ESXi host - this VM is running on to the VM details""", + description="""fills the 'Host Device' field of a VM in NetBox with + the ESXi host it currently runs on. Needs NetBox 3.3 or newer""", default_value=False), ConfigOption("overwrite_device_interface_name", bool, @@ -400,6 +518,27 @@ def __init__(self): """, config_example="AA:BB:CC:11:22:33, 66:77:88:AA:BB:CC" ), + ConfigOption("vm_interface_exclude_filter", + str, + description="""defines a regex expression to exclude VM interfaces from sync by name. + VM interfaces in NetBox whose name matches this filter are completely ignored by this + source: they are excluded from interface matching and will never be updated or altered. + Discovered VM interfaces with a matching name will be excluded from sync as well. + Useful to protect interfaces which are managed by other tools inside the guest + (i.e. 'tailscale0' or 'docker0') from being overwritten with data of a different + interface. The filter is treated as a regex expression which is only anchored at the + beginning of the name ('$' can be used to anchor the end) and is case sensitive. + If more then one expression should match, a '|' needs to be used + """, + config_example="(tailscale|docker)\\d+$" + ), + ConfigOption("host_interface_exclude_filter", + str, + description="""defines a regex expression to exclude host interfaces from sync by name. + Same behavior as 'vm_interface_exclude_filter' but applies to host (device) interfaces. + """, + config_example="(?i)^ipmi" + ), ConfigOption("custom_attribute_exclude", str, description="""defines a comma separated list of custom attribute which should be excluded @@ -415,6 +554,22 @@ def __init__(self): The same behavior also applies for VM disk sizes.""", default_value=True ), + ConfigOption("skip_host_nics", + bool, + description="""Skip creating or updating host physical nics in NetBox. Normal operation + will maintain all physical nics in netbox. This option will skip this part.""" , + default_value=False + ), + ConfigOption("sync_host_cables", + bool, + description="""Create cables in NetBox between the physical interfaces (pNICs) of an + ESXi host and the switch ports which are reported as CDP/LLDP neighbors by this host. + A cable is only created if the reported switch and the reported switch port both + already exist in NetBox and if neither of the two interfaces is cabled yet. Cables + are visible objects which are usually maintained by hand, that's why this is + disabled by default.""", + default_value=False + ), # removed settings ConfigOption("netbox_host_device_role", @@ -469,6 +624,25 @@ def validate_options(self): continue + if option.key == "vm_exclude_disk_sync_by_tag": + + option.set_value(quoted_split(option.value)) + + continue + + if option.key == "vm_exclude_disk_sync": + + re_compiled = None + try: + re_compiled = re.compile(option.value) + except Exception as e: + log.error(f"Problem parsing regular expression for '{self.source_name}.{option.key}': {e}") + self.set_validation_failed() + + option.set_value(re_compiled) + + continue + if "relation" in option.key and "vlan_group_relation" not in option.key: relation_data = list() @@ -524,6 +698,28 @@ def validate_options(self): log.error(f"Primary IP option '{option.key}' value '{option.value}' invalid.") self.set_validation_failed() + # keep in sync with NBVM data_model status values in module/netbox/object_classes.py + valid_vm_statuses = ["offline", "active", "planned", "staged", "failed", "decommissioning"] + + if option.key == "vm_status_on_create": + option.set_value(option.value.lower()) + if option.value not in valid_vm_statuses: + log.error(f"Config option '{option.key}' value '{option.value}' invalid. " + f"Possible values: {', '.join(valid_vm_statuses)}") + self.set_validation_failed() + + continue + + if option.key == "vm_status_preserve": + option.set_value([x.lower() for x in quoted_split(option.value) or list()]) + for status_value in option.value: + if status_value not in valid_vm_statuses: + log.error(f"Config option '{option.key}' value '{status_value}' invalid. " + f"Possible values: {', '.join(valid_vm_statuses)}") + self.set_validation_failed() + + continue + if option.key == "custom_dns_servers": dns_name_lookup = self.get_option_by_name("dns_name_lookup") diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index e63763c9..27ea23c3 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -62,6 +62,8 @@ class VMWareHandler(SourceBase): NBDeviceRole, NBSite, NBSiteGroup, + NBLocation, + NBRegion, NBCluster, NBDevice, NBVM, @@ -78,6 +80,22 @@ class VMWareHandler(SourceBase): NBMACAddress ] + # maps the long interface name a CDP/LLDP neighbor can report to the short forms which are + # commonly used as interface name in NetBox and the other way around: Fa0/16 <> FastEthernet0/16 + # the first entry which matches a reported name wins, longer prefixes need to be listed first + interface_name_prefixes = [ + ("HundredGigabitEthernet", ["HundredGigE", "Hu"]), + ("FiftyGigabitEthernet", ["FiftyGigE", "Fi"]), + ("FortyGigabitEthernet", ["FortyGigE", "Fo"]), + ("TwentyFiveGigabitEthernet", ["TwentyFiveGigE", "25GigE", "Twe", "TF"]), + ("TenGigabitEthernet", ["TenGigE", "Te"]), + # Huawei style 10G + ("XGigabitEthernet", ["XGE", "XGi"]), + ("GigabitEthernet", ["GigE", "Gi", "GE"]), + ("FastEthernet", ["Fa"]), + ("Ethernet", ["Eth", "Et"]) + ] + source_type = "vmware" recursion_level = 0 @@ -88,6 +106,11 @@ class VMWareHandler(SourceBase): site_name = None + # values a BIOS reports when the vendor left the SMBIOS fields unset, plus the dummy vendor/model + # used for hosts where nothing better is known. None of these identify real hardware. + unknown_hardware_identifiers = ["Default string", "NA", "N/A", "None", "Null", "oem", "o.e.m", + "to be filled by o.e.m.", "Unknown", "Generic Vendor", "Generic Model"] + def __init__(self, name=None): if name is None: @@ -104,6 +127,18 @@ def __init__(self, name=None): self.set_source_tag() self.site_name = f"vCenter: {name}" + # index of NetBox interface id to the cable terminated on it, compiled on demand + self.cable_index = None + + # cables are only read from and written to NetBox if this source is meant to maintain them + if self.settings.sync_host_cables is True: + if version.parse(self.inventory.netbox_api_version) < version.parse(NBCable.min_netbox_version): + log.warning(f"Option 'sync_host_cables' needs NetBox version {NBCable.min_netbox_version} " + f"or newer. Disabling it for source '{name}'.") + self.settings.sync_host_cables = False + else: + self.dependent_netbox_objects = self.dependent_netbox_objects + [NBCable] + if self.settings.enabled is False: log.info(f"Source '{name}' is currently disabled. Skipping") return @@ -135,6 +170,350 @@ def __init__(self, name=None): self.objects_to_reevaluate = list() self.parsing_objects_to_reevaluate = False + @classmethod + def get_interface_name_variants(cls, name): + """ + return all spellings of an interface name a CDP/LLDP neighbor reported + + A neighbor can report the long name of a port ("FastEthernet0/16") while the very same + interface is named with a short form in NetBox ("Fa0/16") or the other way around. + Comparing names is done case-insensitive, that's why only one spelling per variant + is returned. + + Parameters + ---------- + name: str + interface name as reported by the neighbor + + Returns + ------- + list: of all name variants, empty if no name was reported + """ + + name = get_string_or_none(name) + if name is None: + return list() + + variants = [name] + name_lower = name.lower() + + for long_prefix, short_prefixes in cls.interface_name_prefixes: + + remainder = None + if name_lower.startswith(long_prefix.lower()): + remainder = name[len(long_prefix):] + else: + for short_prefix in short_prefixes: + if not name_lower.startswith(short_prefix.lower()): + continue + short_remainder = name[len(short_prefix):] + # "Te0/1" uses the short form, "TenGigE0/1" just starts with the same letters + if len(short_remainder) > 0 and (short_remainder[0].isdigit() or short_remainder[0] in "/-"): + remainder = short_remainder + break + + if remainder is None: + continue + + variants.append(f"{long_prefix}{remainder}") + variants.extend([f"{x}{remainder}" for x in short_prefixes]) + break + + return list(dict.fromkeys(variants)) + + @staticmethod + def get_pnic_neighbor(hint): + """ + extract the neighbor a physical host interface reported via CDP or LLDP + + CDP is preferred as it reports the name of the connected switch directly. LLDP + reports the same information in a list of key/value parameters. + + Parameters + ---------- + hint: vim.host.PhysicalNic.NetworkHint + network hint of a single physical interface as returned by QueryNetworkHint() + + Returns + ------- + (dict, None): "system_name", "port_id", "port_description" and "protocol" of the + reported neighbor, None if this interface reported no usable neighbor + """ + + if hint is None: + return None + + connected_switch_port = grab(hint, "connectedSwitchPort") + if connected_switch_port is not None: + system_name = get_string_or_none(grab(connected_switch_port, "systemName")) or \ + get_string_or_none(grab(connected_switch_port, "devId")) + + if system_name is not None: + return { + "system_name": system_name, + "port_id": get_string_or_none(grab(connected_switch_port, "portId")), + "port_description": None, + "protocol": "CDP" + } + + lldp_info = grab(hint, "lldpInfo") + if lldp_info is not None: + + parameters = dict() + for parameter in grab(lldp_info, "parameter", fallback=list()): + key = get_string_or_none(grab(parameter, "key")) + value = get_string_or_none(grab(parameter, "value")) + if key is not None and value is not None: + parameters[key.lower()] = value + + system_name = parameters.get("system name") or parameters.get("systemname") + + # the port id is the interface name of the neighbor (i.e.: "XGigabitEthernet0/0/14") + port_id = parameters.get("port id") or parameters.get("portid") + if port_id is None: + port_id = get_string_or_none(grab(lldp_info, "portId")) + + # the port description is maintained by the switch admin (i.e.: "MAIN-DETAIL12/Eth1") + port_description = parameters.get("port description") or parameters.get("portdescription") + + if system_name is not None: + return { + "system_name": system_name, + "port_id": port_id, + "port_description": port_description, + "protocol": "LLDP" + } + + return None + + @staticmethod + def get_cable_interface_ids(cable): + """ + return the NetBox IDs of all interfaces a cable is terminated on + + Parameters + ---------- + cable: NBCable + the cable object to read the terminations from + + Returns + ------- + list: of NetBox interface IDs + """ + + interface_ids = list() + for side in ["a_terminations", "b_terminations"]: + for termination in grab(cable, f"data.{side}", fallback=list()): + + if not isinstance(termination, dict): + continue + if termination.get("object_type") != NBInterface.object_type: + continue + if isinstance(termination.get("object_id"), int): + interface_ids.append(termination.get("object_id")) + + return interface_ids + + def get_cable_for_interface_id(self, interface_id): + """ + return the cable which is terminated on a NetBox interface + + All cables are looked at only once, cables added afterwards are added to the index + by add_cable_to_neighbor(). + + Parameters + ---------- + interface_id: int + NetBox ID of the interface to find the cable for + + Returns + ------- + (NBCable, None): the cable terminated on this interface, None if there is none + """ + + if self.cable_index is None: + self.cable_index = dict() + for cable in self.inventory.get_all_items(NBCable): + for cable_interface_id in self.get_cable_interface_ids(cable): + self.cable_index.setdefault(cable_interface_id, cable) + + return self.cable_index.get(interface_id) + + def get_device_by_neighbor_name(self, name): + """ + find the NetBox device a CDP/LLDP neighbor reported as its system name + + An exact match always wins. A neighbor can report a FQDN while the device is named + with its short name in NetBox (or the other way around), that's why short names are + compared as well. A short name match is only accepted if it is unambiguous and if it + does not compare two different domains with each other. + + Parameters + ---------- + name: str + system name the neighbor reported + + Returns + ------- + (NBDevice, None): the matching device, None if there was no or no unique match + """ + + name = get_string_or_none(name) + if name is None: + return None + + name = name.lower() + short_name = name.split(".")[0] + + short_name_matches = list() + for device in self.inventory.get_all_items(NBDevice): + + device_name = get_string_or_none(grab(device, "data.name")) + if device_name is None: + continue + + device_name = device_name.lower() + if device_name == name: + return device + + # "sw01.dc1.example.com" and "sw01.dc2.example.com" are not the same device + if "." in name and "." in device_name: + continue + + if device_name.split(".")[0] == short_name: + short_name_matches.append(device) + + if len(short_name_matches) == 1: + return short_name_matches[0] + + if len(short_name_matches) > 1: + log.debug(f"Neighbor '{name}' matches more than one {NBDevice.name} in NetBox: " + f"{[grab(x, 'data.name') for x in short_name_matches]}") + + return None + + def get_interface_by_neighbor_port(self, device, port_names): + """ + find the interface of a device which matches one of the port names a neighbor reported + + Parameters + ---------- + device: NBDevice + the device to look for the interface on + port_names: list + port names reported by the neighbor, in the order they should be tried + + Returns + ------- + (NBInterface, None): the matching interface, None if none of the names matched + """ + + if device is None: + return None + + wanted_names = list() + for port_name in port_names: + wanted_names.extend([x.lower() for x in self.get_interface_name_variants(port_name)]) + + if len(wanted_names) == 0: + return None + + device_interfaces = dict() + for interface in self.inventory.get_all_interfaces(device): + interface_name = get_string_or_none(grab(interface, "data.name")) + if interface_name is not None: + device_interfaces.setdefault(interface_name.lower(), interface) + + for wanted_name in dict.fromkeys(wanted_names): + if device_interfaces.get(wanted_name) is not None: + return device_interfaces.get(wanted_name) + + return None + + def add_cable_to_neighbor(self, host_interface, neighbor, host_name, pnic_name): + """ + add a cable between a physical host interface and the switch port its CDP/LLDP neighbor reported + + A cable is only added if the reported switch and switch port were both found in NetBox and + if neither of the two interfaces is connected with a cable already. Cables which were created + by this source before are claimed again so they don't end up being marked as orphaned. + + Parameters + ---------- + host_interface: NBInterface + interface object of the physical host interface + neighbor: dict + neighbor data as returned by get_pnic_neighbor() + host_name: str + name of the host this interface belongs to, used for logging + pnic_name: str + name of the physical interface, used for logging + """ + + if host_interface is None or neighbor is None: + return + + log_name = f"Neighbor of interface '{pnic_name}' on host '{host_name}'" + + switch_object = self.get_device_by_neighbor_name(neighbor.get("system_name")) + if switch_object is None: + log.debug2(f"{log_name}: {NBDevice.name} '{neighbor.get('system_name')}' not found in NetBox. " + f"Not adding a cable.") + return + + port_names = [neighbor.get("port_id"), neighbor.get("port_description")] + switch_interface = self.get_interface_by_neighbor_port(switch_object, port_names) + if switch_interface is None: + log.debug2(f"{log_name}: no interface matching {[x for x in port_names if x is not None]} found on " + f"{NBDevice.name} '{grab(switch_object, 'data.name')}'. Not adding a cable.") + return + + host_interface_id = getattr(host_interface, "nb_id", 0) + switch_interface_id = getattr(switch_interface, "nb_id", 0) + + # a cable can only reference interfaces which exist in NetBox. + # an interface which was just discovered gets its cable during the next run + if host_interface_id == 0 or switch_interface_id == 0: + log.debug2(f"{log_name}: {NBInterface.name} '{host_interface.get_display_name()}' or " + f"'{switch_interface.get_display_name()}' does not exist in NetBox yet. " + f"A cable can be added during the next run.") + return + + existing_cable = self.get_cable_for_interface_id(host_interface_id) or \ + self.get_cable_for_interface_id(switch_interface_id) + + if existing_cable is not None: + + existing_interface_ids = self.get_cable_interface_ids(existing_cable) + + if host_interface_id in existing_interface_ids and switch_interface_id in existing_interface_ids: + log.debug2(f"{log_name}: cable '{existing_cable.get_display_name()}' already exists") + + # a cable this source added before is still valid and must not be marked as orphaned. + # a cable which somebody else created stays untouched and unmanaged + if self.source_tag in existing_cable.get_tags(): + existing_cable.set_source(self) + else: + log.debug(f"{log_name}: {NBInterface.name} '{host_interface.get_display_name()}' or " + f"'{switch_interface.get_display_name()}' is already connected with cable " + f"'{existing_cable.get_display_name()}'. Not adding a cable.") + + return + + log.debug2(f"{log_name}: reported via {neighbor.get('protocol')} as " + f"'{neighbor.get('system_name')}' port '{neighbor.get('port_id')}'") + + cable_object = self.inventory.add_object(NBCable, source=self, data={ + # a label is not mandatory in NetBox and stays empty, the terminations name this cable + "label": "", + "a_terminations": [{"object_type": NBInterface.object_type, "object_id": host_interface_id}], + "b_terminations": [{"object_type": NBInterface.object_type, "object_id": switch_interface_id}], + "status": "connected" + }) + + for interface_id in [host_interface_id, switch_interface_id]: + self.cable_index[interface_id] = cable_object + def create_sdk_session(self): """ Initialize SDK session with vCenter @@ -226,7 +605,9 @@ def create_api_session(self): return False if vsphere_automation_sdk_available is False: - log.warning(f"Unable to import Python 'vsphere-automation-sdk'. Tag syncing will be disabled.") + log.warning("Unable to import the Python 'vcf-sdk' package (successor of the archived " + "'vsphere-automation-sdk', which no longer imports on Python 3.12+ with " + "setuptools >= 82), run 'pip install --upgrade vcf-sdk'. Tag syncing will be disabled.") return False log.debug(f"Starting vCenter API connection to '{self.settings.host_fqdn}'") @@ -445,6 +826,27 @@ def passes_filter(name, include_filter, exclude_filter): return True + @staticmethod + def hardware_identifier_is_unknown(value): + """ + checks if a hardware identifier (vendor, model, asset tag) reported for a host + is a placeholder rather than a real value. + + Parameters + ---------- + value: str + identifier to check + + Returns + ------- + bool: True if value is unset or one of the known placeholders, otherwise False + """ + + if value is None: + return True + + return value.lower() in [x.lower() for x in VMWareHandler.unknown_hardware_identifiers] + def get_site_name(self, object_type, object_name, cluster_name=""): """ Return a site name for a NBCluster or NBDevice depending on config options @@ -474,13 +876,20 @@ def get_site_name(self, object_type, object_name, cluster_name=""): site_name = self.get_object_relation(object_name, relation_name) - if object_type == NBDevice and site_name is None: + # check if cluster is in a different site than the host and override the site name if so + if object_type == NBDevice: site_name = self.get_site_name(NBCluster, cluster_name) if site_name is not None: - log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") - - # set default site name - if site_name is None: + log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'. Overriding host site relation '{relation_name}'") + else: + site_name = self.get_object_relation(object_name, relation_name) + # set deault site name if no relation was found + if site_name is None: + site_name = self.site_name + log.debug2(f"No site relation for {type(object_name)}: '{object_name}' found, using default site '{site_name}'") + + # set default site name for devices + if site_name is None and object_type == NBDevice: site_name = self.site_name log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") @@ -489,8 +898,95 @@ def get_site_name(self, object_type, object_name, cluster_name=""): site_name = None log.debug2(f"Site relation for '{object_name}' set to None") + log.debug2(f"Returning site name '{site_name}' for {object_type.name} '{object_name}'.") + return site_name + def get_scope_type(self, object_type, object_name): + """ + Retrieve the scope_type for a NBCluster instance by object name or from the config option + cluster_scope_type_relation + + Note: Only NBCluster is supported as the object_type. + + Parameters + ---------- + object_type: object type + The NetBox object type (must be NBCluster). + object_name: str + The name of the object to look up. + + Returns + ------- + str or None: scope type if one is found, otherwise None + """ + + # Validate object type + if object_type != NBCluster: + raise ValueError(f"Object type must be '{NBCluster.name}'.") + + # get scope type from relation config + relation_name = "cluster_scope_type_relation" + scope_type = self.get_object_relation(object_name, relation_name) + log.debug(f"Retrieved scope type '{scope_type}' for {object_type.name} '{object_name}' from relation '{relation_name}'.") + + # if the scope_type is a list, use the first element + if scope_type is not None and type(scope_type) is list: + scope_type_list = scope_type + scope_type = scope_type_list[0] if len(scope_type_list) > 0 else None + log.debug(f"Scope type for {object_type.name} '{object_name}' is a list, using first element: '{scope_type}'") + + # if scope_type is not a str, return None + if type(scope_type) is not str: + log.debug(f"scope_type is type: {type(scope_type)}, not str") + return None + + # set scope_type to None if it is configured as "" + if scope_type == "": + log.debug(f"Scope type for {object_type.name} '{object_name}' is set to None") + return None + + log.debug2(f"Returning scope type '{scope_type}' for {object_type.name} '{object_name}'.") + return scope_type + + def get_scope_id(self, object_type, object_name): + """ + Retrieve the scope_id for a NBCluster instance by object name or from the config option + cluster_scope_id_relation + + Note: Only NBCluster is supported as the object_type. + + Parameters + ---------- + object_type: type + The NetBox object type (must be NBCluster). + object_name: str + The name of the object to look up. + + Returns + ------- + str or None: scope id if one is found, otherwise None + """ + # Validate object type + if object_type != NBCluster: + raise ValueError(f"Object type must be '{NBCluster.name}'.") + + # get scope id from relation config + relation_name = "cluster_scope_id_relation" + scope_id = self.get_object_relation(object_name, relation_name) + + # return None if scope_id is None or not a string + if scope_id is None: + log.debug(f"No scope id found for {object_name}.") + return None + if type(scope_id) is not str: + log.debug(f"scope_id is type: {type(scope_id)}, not str") + return None + + log.debug2(f"Retrieved scope id '{scope_id}' for {object_type.name} '{object_name}' from relation '{relation_name}'. End of method.") + + return scope_id + def get_object_based_on_macs(self, object_type, mac_list=None): """ Try to find a NetBox object based on list of MAC addresses. @@ -613,7 +1109,10 @@ def _matches_device_primary_ip(device_primary_ip, ip_needle): ip = None if device_primary_ip is not None and ip_needle is not None: - if isinstance(device_primary_ip, dict): + if isinstance(device_primary_ip, NBIPAddress): + ip = grab(device_primary_ip, "data.address") + + elif isinstance(device_primary_ip, dict): ip = grab(device_primary_ip, "address") elif isinstance(device_primary_ip, int): @@ -681,19 +1180,30 @@ def get_vmware_object_tags(self, obj): # noinspection PyBroadException try: - tag_name = self.tag_session.tagging.Tag.get(tag_id).name - tag_description = self.tag_session.tagging.Tag.get(tag_id).description + tag = self.tag_session.tagging.Tag.get(tag_id) # store the object + tag_name = tag.name + tag_description = tag.description except Exception as e: log.error(f"Unable to retrieve vCenter tag '{tag_id}' for '{obj.name}': {e}") - continue + continue # skip tag entirely if basic fetch fails - if tag_name is not None: + category_name = None + if bool(self.settings.tag_name_include_category) is True: + # noinspection PyBroadException + try: + category_name = self.tag_session.tagging.Category.get(tag.category_id).name + except Exception as e: + log.debug(f"Unable to retrieve category of vCenter tag '{tag_name}': {e}") + if tag_name is not None: if tag_description is not None and len(f"{tag_description}") > 0: tag_description = f"{primary_tag_name}: {tag_description}" else: tag_description = primary_tag_name + if category_name is not None: + tag_name = f"{category_name}:{tag_name}" + tag_list.append(self.inventory.add_update_object(NBTag, data={ "name": tag_name, "description": tag_description @@ -825,6 +1335,28 @@ def get_object_custom_fields(self, obj): return_custom_fields[grab(custom_field, "data.name")] = f"{memory_size} {memory_unit}" + # add VMware Tools guest hostname to VM + if object_type == "virtualization.virtualmachine" and self.settings.vm_guest_hostname_custom_field: + + guest_hostname_field_name = self.settings.vm_guest_hostname_custom_field + guest_hostname = get_string_or_none(grab(obj, "guest.hostName")) + + if guest_hostname is not None: + custom_field = self.add_update_custom_field({ + "name": guest_hostname_field_name, + "label": "VMware Guest Hostname", + "object_types": [object_type], + "type": "text", + "description": "Hostname reported by VMware Tools from inside the guest OS" + }) + + return_custom_fields[grab(custom_field, "data.name")] = guest_hostname + + else: + log.debug2(f"VM '{grab(obj, 'name')}' guest hostname not reported by VMware Tools " + "(not installed, not running or no data available yet). Keeping current " + f"value of custom field '{guest_hostname_field_name}' untouched.") + field_definition = {grab(k, "key"): grab(k, "name") for k in grab(obj, "availableField", fallback=list())} for obj_custom_field in custom_value: @@ -1012,6 +1544,10 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v disk_data: list data of discs which belong to a VM + Returns + ------- + tuple: the added/updated (NBDevice, NBVM) object and a dict of all interface objects + which were added/updated for it, discovered interface name as key """ if object_type not in [NBDevice, NBVM]: @@ -1039,6 +1575,10 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v (object_type.name, device_vm_object.get_display_name(including_second_key=True))) # keep searching if no exact match was found + elif object_type == NBVM and self.settings.match_vm_by_mac_address is False: + + log.debug2("Matching VMs by MAC address is disabled via 'match_vm_by_mac_address'. Skipping.") + else: log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") @@ -1066,7 +1606,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v data={"asset_tag": object_data.get("asset_tag")}) # look for VMs with same serial - if object_type == NBVM and device_vm_object is None and object_data.get("serial") is not None: + if object_type == NBVM and device_vm_object is None and object_data.get("serial") is not None and \ + self.settings.match_vm_by_serial is True: log.debug2(f"No match found. Trying to find {object_type.name} based on serial number") device_vm_object = self.inventory.get_by_data(object_type, data={"serial": object_data.get("serial")}) @@ -1075,6 +1616,10 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v (object_type.name, device_vm_object.get_display_name(including_second_key=True))) # keep looking for devices with the same primary IP + elif object_type == NBVM and self.settings.match_vm_by_ip_address is False: + + log.debug2("Matching VMs by primary IP address is disabled via 'match_vm_by_ip_address'. Skipping.") + else: log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") @@ -1084,6 +1629,11 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v if device_vm_object is None: object_name = object_data.get(object_type.primary_key) log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") + + if object_type == NBVM and self.settings.vm_status_on_create is not None and \ + object_data.get("status") is not None: + object_data["status"] = self.settings.vm_status_on_create + device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) else: @@ -1095,6 +1645,22 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v object_data.get("platform") is not None: del object_data["platform"] + # a device type made up from a BIOS placeholder carries no information. Keep the one + # already set in NetBox instead of replacing it on every run (issue #460) + if object_type == NBDevice and object_data.get("device_type") is not None and \ + self.hardware_identifier_is_unknown(grab(object_data, "device_type.model")): + del object_data["device_type"] + + if object_type == NBVM and object_data.get("status") is not None: + current_status = grab(device_vm_object, "data.status") + if isinstance(current_status, dict): + current_status = current_status.get("value") + if current_status in (self.settings.vm_status_preserve or list()): + log.debug2(f"Current status '{current_status}' of " + f"'{device_vm_object.get_display_name()}' is in 'vm_status_preserve' list. " + f"Not updating VM status.") + del object_data["status"] + device_vm_object.update(data=object_data, source=self) # add object to cache @@ -1128,30 +1694,64 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v if version.parse(self.inventory.netbox_api_version) >= version.parse("3.7.0") and \ object_type == NBVM and disk_data is not None and len(disk_data) > 0: - # create pairs of existing and discovered disks. - # currently these disks are only used within the VM model. that's why we use this simple approach and - # just rewrite disk as they appear in order. - # otherwise we would need to implement a matching function like matching interfaces. - disk_zip_list = zip_longest( - sorted(device_vm_object.get_virtual_disks(), key=lambda x: grab(x, "data.name")), - sorted(disk_data, key=lambda x: x.get("name")), - fillvalue="X") - - for existing, discovered in disk_zip_list: - if existing == "X": - self.inventory.add_object(NBVirtualDisk, source=self, - data={**discovered, **{"virtual_machine": device_vm_object}}, ) - elif discovered == "X": - log.info(f"{existing.name} '{existing.get_display_name(including_second_key=True)}' has been deleted") - existing.deleted = True - else: - existing.update(data=discovered, source=self) + # Skip disk updates for VMs that match exclusion filters + skip_disk_sync = False + + # Check if VM name matches vm_exclude_disk_sync filter + if hasattr(self.settings, 'vm_exclude_disk_sync') and self.settings.vm_exclude_disk_sync is not None: + if self.settings.vm_exclude_disk_sync.match(object_data.get("name")): + log.debug(f"VM '{object_data.get('name')}' matches vm_exclude_disk_sync filter. " + f"Skipping disk synchronization.") + skip_disk_sync = True + + # Check if VM has any tags that match vm_exclude_disk_sync_by_tag filter + if not skip_disk_sync and hasattr(self.settings, 'vm_exclude_disk_sync_by_tag') and \ + self.settings.vm_exclude_disk_sync_by_tag is not None: + vm_tags = [NetBoxObject.extract_tag_name(tag) for tag in device_vm_object.data.get("tags", list())] + for exclude_tag in self.settings.vm_exclude_disk_sync_by_tag: + if exclude_tag in vm_tags: + log.debug(f"VM '{object_data.get('name')}' has tag '{exclude_tag}' which matches " + f"vm_exclude_disk_sync_by_tag filter. Skipping disk synchronization.") + skip_disk_sync = True + break + + if not skip_disk_sync: + # create pairs of existing and discovered disks. + # currently these disks are only used within the VM model. that's why we use this simple approach and + # just rewrite disk as they appear in order. + # otherwise we would need to implement a matching function like matching interfaces. + disk_zip_list = zip_longest( + sorted(device_vm_object.get_virtual_disks(), key=lambda x: grab(x, "data.name")), + sorted(disk_data, key=lambda x: x.get("name")), + fillvalue="X") + + for existing, discovered in disk_zip_list: + if existing == "X": + self.inventory.add_object(NBVirtualDisk, source=self, + data={**discovered, **{"virtual_machine": device_vm_object}}, ) + elif discovered == "X": + log.info(f"{existing.name} '{existing.get_display_name(including_second_key=True)}' has been deleted") + existing.deleted = True + else: + existing.update(data=discovered, source=self) # compile all nic data into one dictionary if object_type == NBVM: nic_data = vnic_data + interface_exclude_filter = self.settings.vm_interface_exclude_filter else: nic_data = {**pnic_data, **vnic_data} + interface_exclude_filter = self.settings.host_interface_exclude_filter + + # exclude discovered interfaces which match the exclude filter + if interface_exclude_filter is not None: + for int_name in list(nic_data.keys()): + if interface_exclude_filter.match(int_name): + log.debug(f"Discovered interface '{int_name}' matches interface_exclude_filter. " + f"Excluding it from sync") + del nic_data[int_name] + if nic_ips is not None: + nic_ips.pop(int_name, None) # map interfaces of existing object with discovered interfaces nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) @@ -1174,6 +1774,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v except ValueError: log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") + interface_objects = dict() + for int_name, int_data in nic_data.items(): if nic_object_dict.get(int_name) is not None: @@ -1187,6 +1789,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v int_data, nic_ips.get(int_name, list()), vmware_object=vmware_object) + interface_objects[int_name] = nic_object + # add all interface IPs for ip_object in ip_address_objects: @@ -1235,7 +1839,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v f"'{device_vm_object.get_display_name()}'") device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) - return + return device_vm_object, interface_objects def get_parent_object_by_class(self, obj, object_class_to_find): @@ -1340,6 +1944,29 @@ def add_datacenter(self, obj): self.add_object_to_cache(obj, self.inventory.add_update_object(NBClusterGroup, data=object_data, source=self)) + def object_synced_by_other_source(self, nb_object): + """ + Check if a NetBox object is maintained by a different configured source. During + this run that is the source which touched the object, for objects read from NetBox + the source tags decide. + + Parameters + ---------- + nb_object: NetBoxObject + object to check + + Returns + ------- + bool: True if another configured source synced this object + """ + + if nb_object.source is not None: + return nb_object.source is not self + + other_source_tags = [x.source_tag for x in self.inventory.source_list if x is not self] + + return len(set(nb_object.get_tags()).intersection(other_source_tags)) > 0 + def add_cluster(self, obj): """ Add a vCenter cluster as a NBCluster to NetBox. Cluster name is checked against @@ -1378,9 +2005,20 @@ def add_cluster(self, obj): self.settings.cluster_include_filter, self.settings.cluster_exclude_filter) is False: return + log.debug2(f"Cluster '{name}' passes include and exclude filters. Continuing.") + + # get scope type and id, or site name + scope_type = self.get_scope_type(NBCluster, full_cluster_name) + if scope_type is None: + scope_type = self.get_scope_type(NBCluster, name) site_name = self.get_site_name(NBCluster, full_cluster_name) + scope_id = self.get_scope_id(NBCluster, full_cluster_name) + if scope_id is None: + scope_id = self.get_scope_id(NBCluster, name) + log.debug(f"Cluster '{full_cluster_name}' has scope id '{scope_id}' of type {type(scope_id)}.") + data = { "name": name, "type": {"name": "VMware ESXi"}, @@ -1388,11 +2026,26 @@ def add_cluster(self, obj): } if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): - if site_name is not None: - data["scope_id"] = {"name": site_name} + # set the scope type and id if they are defined + if scope_type is not None: + data["scope_type"] = scope_type + data["scope_id"] = scope_id + log.debug(f"Cluster '{full_cluster_name}' (or {name}) has scope type '{scope_type}' " + f"and scope id '{scope_id}'.") + elif site_name is not None: + # NetBox wants the id of the scoped object, so the site has to be a real + # object here. A plain dict is sent as is and rejected with + # "scope_id: A valid integer is required." data["scope_type"] = "dcim.site" + data["scope_id"] = self.inventory.add_update_object(NBSite, data={"name": site_name}) + else: + log.debug(f"Cluster '{full_cluster_name}' has no scope type or scope id.") else: - data["site"] = {"name": site_name} + # set site_name in the pre-4.2.0 NetBox versions if one is found + if site_name is not None: + data["site"] = {"name": site_name} + + log.debug(f"Cluster '{full_cluster_name}' (or {name}) has data items '{data.items()}'.") tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation") if tenant_name is not None: @@ -1411,11 +2064,21 @@ def add_cluster(self, obj): if grab(cluster_candidate, "data.name") != name: continue - # try to find a cluster with matching site - if cluster_candidate.get_site_name() == site_name: - cluster_object = cluster_candidate - log.debug2("Found an existing cluster where 'name' and 'site' are matching") - break + # a cluster which a different source keeps in a different site is not this cluster. + # NetBox refuses to move a cluster away from the site of its hosts, so adopting it + # would fail on every run and attach this vCenter's hosts to the other cluster. + if site_name is not None and cluster_candidate.get_site_name() not in [None, site_name] and \ + self.object_synced_by_other_source(cluster_candidate) is True: + log.debug2(f"Skipping cluster '{name}' in site '{cluster_candidate.get_site_name()}' " + f"as it is synced by a different source") + continue + + if site_name is not None: + # try to find a cluster with matching site + if cluster_candidate.get_site_name() == site_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'site' are matching") + break if grab(cluster_candidate, "data.group") is not None and \ grab(cluster_candidate, "data.group.data.name") == group_name: @@ -1626,11 +2289,11 @@ def add_host(self, obj): platform = f"{product_name} {product_version}" platform = self.get_object_relation(platform, "host_platform_relation", fallback=platform) - # if the device vendor/model cannot be retrieved (due to problem on the host), - # set a dummy value so the host still gets synced - if manufacturer is None: + # if the device vendor/model cannot be retrieved (due to problem on the host) or the BIOS + # only reports a placeholder, set a dummy value so the host still gets synced + if self.hardware_identifier_is_unknown(manufacturer): manufacturer = "Generic Vendor" - if model is None: + if self.hardware_identifier_is_unknown(model): model = "Generic Model" # get status @@ -1660,12 +2323,9 @@ def add_host(self, obj): if self.settings.collect_hardware_asset_tag is True and "AssetTag" in identifier_dict.keys(): - banned_tags = ["Default string", "NA", "N/A", "None", "Null", "oem", "o.e.m", - "to be filled by o.e.m.", "Unknown"] - this_asset_tag = identifier_dict.get("AssetTag") - if this_asset_tag.lower() not in [x.lower() for x in banned_tags]: + if not self.hardware_identifier_is_unknown(this_asset_tag): asset_tag = this_asset_tag # get host_tenant_relation @@ -1769,6 +2429,7 @@ def add_host(self, obj): # now iterate over all physical interfaces and collect data pnic_data_dict = dict() + pnic_neighbors = dict() pnic_hints = dict() # noinspection PyBroadException try: @@ -1777,7 +2438,11 @@ def add_host(self, obj): except Exception: pass - for pnic in grab(obj, "config.network.pnic", fallback=list()): + pnic_list = grab(obj, "config.network.pnic", fallback=list()) + if self.settings.skip_host_nics is True: + log.debug(f"Skipping physical interfaces of host '{name}' (skip_host_nics)") + pnic_list = list() + for pnic in pnic_list: pnic_name = grab(pnic, "device") pnic_key = grab(pnic, "key") @@ -1853,6 +2518,18 @@ def add_host(self, obj): log.debug2(f"Host NIC with MAC '{pnic_mac_address}' excluded from sync. Skipping") continue + # collect the reported neighbor to add a cable for this interface later on + if self.settings.sync_host_cables is True: + pnic_neighbor = self.get_pnic_neighbor(pnic_hints.get(pnic_name)) + + if pnic_neighbor is not None: + pnic_neighbors[pnic_name] = pnic_neighbor + + # a CDP neighbor is already part of the description + if pnic_neighbor.get("protocol") == "LLDP": + neighbor_port = pnic_neighbor.get("port_id") or pnic_neighbor.get("port_description") + pnic_description += f" (conn: {pnic_neighbor.get('system_name')} - {neighbor_port})" + pnic_data = { "name": unquote(pnic_name), "device": None, # will be set once we found the correct device @@ -2053,9 +2730,15 @@ def add_host(self, obj): host_primary_ip6 = int_v6 # add host to inventory - self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, - vnic_data=vnic_data_dict, nic_ips=vnic_ips, - p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6, vmware_object=obj) + device_object, interface_objects = \ + self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, + vnic_data=vnic_data_dict, nic_ips=vnic_ips, + p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6, vmware_object=obj) + + # add cables to the switch ports which were reported via CDP/LLDP + if device_object is not None: + for pnic_name, pnic_neighbor in pnic_neighbors.items(): + self.add_cable_to_neighbor(interface_objects.get(pnic_name), pnic_neighbor, name, pnic_name) return @@ -2098,7 +2781,8 @@ def add_virtual_machine(self, obj): # get VM UUID vm_uuid = grab(obj, "config.instanceUuid") - if vm_uuid is None or vm_uuid in self.processed_vm_uuid and obj not in self.objects_to_reevaluate: + if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and \ + not (self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate): return log.debug(f"Parsing vCenter VM: {name}") @@ -2230,8 +2914,9 @@ def add_virtual_machine(self, obj): vcenter_tags = self.collect_object_tags(obj) # check if VM tag excludes VM from being synced to NetBox + vcenter_tag_names = [NetBoxObject.extract_tag_name(t) for t in vcenter_tags] for sync_exclude_tag in self.settings.vm_exclude_by_tag_filter or list(): - if sync_exclude_tag in vcenter_tags: + if sync_exclude_tag in vcenter_tag_names: log.debug(f"Virtual machine vCenter tag '{sync_exclude_tag}' in matches 'vm_exclude_by_tag_filter'. " f"Skipping") return @@ -2573,6 +3258,25 @@ def add_virtual_machine(self, obj): nic_data[int_full_name] = vm_nic_data + # if VM has only one IPv4 on all interfaces, use it as primary IPv4 address + if vm_primary_ip4 is None: + potential_primary_ipv4_list = list() + + for ip in [y for xs in nic_ips.values() for y in xs]: + # noinspection PyBroadException + try: + ip_address_object = ip_interface(ip) + except Exception: + continue + + if ip_address_object.version == 4: + potential_primary_ipv4_list.append(ip_address_object) + + if len(potential_primary_ipv4_list) == 1: + log.debug(f"Found one IPv4 '{potential_primary_ipv4_list[0]}' address on all interfaces of " + f"VM '{name}', using it as primary IPv4.") + vm_primary_ip4 = potential_primary_ipv4_list[0] + # if VM has only one IPv6 on all interfaces, use it as primary IPv6 address if vm_primary_ip6 is None or True: all_ips = [y for xs in nic_ips.values() for y in xs] diff --git a/netbox-sync.py b/netbox-sync.py index c85de915..47191f1c 100755 --- a/netbox-sync.py +++ b/netbox-sync.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/pyproject.toml b/pyproject.toml index 1617676b..0dfaa3f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "netbox-sync" -version = "1.8.1" +version = "1.9.0" description = "Sync objects from various sources to NetBox" authors = [ { name = "Ricardo Bartels", email = "ricardo.bartels@telekom.de" } @@ -8,18 +8,39 @@ authors = [ readme = "README.md" license = "MIT" license-files = ["LICENSE.txt"] -requires-python = ">=3.13" +requires-python = ">=3.12" dependencies = [ - "aiodns>=4.0.0", - "packaging>=26.0", - "pyvmomi==8.0.3.0.1", - "pyyaml>=6.0.3", - "requests>=2.32.5", - "setuptools==81.0.0", - "urllib3>=2.6.3", - "wheel>=0.46.3", + "aiodns==4.0.4", + "certifi==2026.7.22", + "cffi==2.1.1", + "charset-normalizer==3.5.2", + "hcloud==2.25.1", + "idna==3.20", + "packaging==26.3", + "pycares==5.0.1", + "pycparser==3.0", + "python-dateutil==2.9.0.post0", + "pyvmomi==9.1.1.0", + "pyyaml==6.0.3", + "requests==2.34.2", + "setuptools==84.0.0", + "six==1.17.0", + "urllib3==2.8.0", + "wheel==0.48.0", +] + +[project.optional-dependencies] +test = [ + "pytest>=9.1.1", ] [project.urls] Repository = "https://github.com/bb-ricardo/netbox-sync.git" Issues = "https://github.com/bb-ricardo/netbox-sync/issues" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" +# the modules under test live in the repository root, which plain `pytest` does +# not put on sys.path (unlike `python -m pytest`) +pythonpath = ["."] diff --git a/requirements.txt b/requirements.txt index 42a9d7dc..4edc950e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,17 @@ -aiodns==4.0.0 -certifi==2026.1.4 -cffi==2.0.0 -charset-normalizer==3.4.4 -idna==3.11 -packaging==26.0 +aiodns==4.0.4 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.2 +hcloud==2.25.1 +idna==3.20 +packaging==26.3 pycares==5.0.1 pycparser==3.0 -pyvmomi==8.0.3.0.1 +python-dateutil==2.9.0.post0 +pyvmomi==9.1.1.0 pyyaml==6.0.3 -requests==2.32.5 -setuptools==81.0.0 +requests==2.34.2 +setuptools==84.0.0 six==1.17.0 -urllib3==2.6.3 -wheel==0.46.3 +urllib3==2.8.0 +wheel==0.48.0 \ No newline at end of file diff --git a/requirements_3.6.txt b/requirements_3.6.txt deleted file mode 100644 index 77ef4a4b..00000000 --- a/requirements_3.6.txt +++ /dev/null @@ -1,7 +0,0 @@ -packaging -urllib3==1.26.12 -wheel -requests==2.27.1 -pyvmomi==7.0.3 -aiodns==2.0.0 -setuptools==59.6.0 diff --git a/scripts/publi.sh b/scripts/publi.sh index 928b6c9d..dd5924e6 100755 --- a/scripts/publi.sh +++ b/scripts/publi.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # diff --git a/scripts/set_version.sh b/scripts/set_version.sh index ae7cc9a0..1facf7c5 100755 --- a/scripts/set_version.sh +++ b/scripts/set_version.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -13,7 +13,7 @@ VERSION_DATA_FILE="module/__init__.py" README_FILE="README.md" PYPROJECT_TOML="pyproject.toml" VERSION_TO_SET="$1" -COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2026 Ricardo Bartels. All rights reserved." +COPYRIGHT_PATTERN="# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved." BASE_PATH="$(realpath "$(dirname "${0}")/..")" # shellcheck disable=SC2181 diff --git a/settings-example.ini b/settings-example.ini index ec2d5f1f..b1c71fa2 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.8.1 (2026-03-18) +;;; Version: 1.9.0 (2026-10-02) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config @@ -44,7 +44,8 @@ [netbox] ; Requires an NetBox API token with full permissions on all objects except 'auth', -; 'secrets' and 'users' +; 'secrets' and 'users'. Both v1 (legacy) and v2 (NetBox 4.5+, nbt_ prefix) tokens are +; supported. api_token = XYZ ; Requires a hostname or IP which points to your NetBox instance @@ -84,6 +85,22 @@ host_fqdn = netbox.example.com ; Ricardo/netbox-sync/issues/176) ;ignore_unknown_source_object_pruning = False +; Set the status of orphaned devices to this value. If undefined the status of a device is +; never changed by this program. Needs to be a valid device status in NetBox (i.e: +; 'decommissioning', 'offline', 'planned') and requires 'prune_enabled' to be true, as +; pruning is switched off whenever a source was unavailable. Once a device is reported by +; a source again its status is set back to 'active', but only if it still carries the +; status defined here +;orphaned_device_status = decommissioning + +; Set the status of orphaned virtual machines to this value. If undefined the status of a +; virtual machine is never changed by this program. Needs to be a valid virtual machine +; status in NetBox (i.e: 'decommissioning', 'offline', 'planned') and requires +; 'prune_enabled' to be true, as pruning is switched off whenever a source was +; unavailable. Once a virtual machine is reported by a source again its status is set back +; to 'active', but only if it still carries the status defined here +;orphaned_vm_status = decommissioning + ; The maximum number of objects returned in a single request. If a NetBox instance is very ; quick responding the value should be raised ;default_netbox_result_limit = 200 @@ -177,6 +194,16 @@ password = super-secret ; tags are collected for VMs. ;vm_exclude_by_tag_filter = tag-a, tag-b +; defines a comma separated list of VM names (regex) where disk synchronization will be +; excluded. A VM matching this filter will still be synced to NetBox, but its disk +; information won't be updated. +;vm_exclude_disk_sync = backup-.*, temp-.* + +; defines a comma separated list of vCenter tags which (if assigned to a VM) will exclude +; this VM from disk synchronization. A VM with this tag will still be synced to NetBox, +; but its disk information won't be updated. +;vm_exclude_disk_sync_by_tag = backup-vm, veeam-job + ; relations options ; This option defines which vCenter cluster is part of a NetBox site. @@ -199,6 +226,19 @@ password = super-secret ; cluster_site_relation ;host_site_relation = nyc02.* = New York, ffm01.* = Frankfurt +; This option defines the scope type for a cluster. The scope type can be 'dcim.site', +; 'dcim.sitegroup', 'dcim.location' or 'dcim.region'. This is done with a comma separated +; key = value list. Can be set to "" to not assign a scope type. Note: this does not +; remove scope types from existing clusters in NetBox. key: defines a cluster name as +; regex value: defines the NetBox scope type name (use quotes if name contains commas) +;cluster_scope_type_relation = Cluster_NYC = dcim.site, Cluster_FFM = dcim.sitegroup, Cluster_BER = dcim.location + +; This option defines the scope id for a cluster. The scope id is the NetBox ID of the +; scope type. This is done with a comma separated key = value list. To be used in +; combination with the 'cluster_scope_type_relation'. key: defines a cluster name as regex +; value: defines the NetBox scope id (use quotes if name contains commas) +;cluster_scope_id_relation = Cluster_NYC = 1, Cluster_FFM.* = 2, Cluster_BER = 7 + ; This option defines which cluster/host/VM belongs to which tenant. ; This is done with a comma separated key = value list. ; key: defines a hosts/VM name as regex @@ -243,6 +283,24 @@ password = super-secret ; centers if VMWare does not report the blades serial number properly. ;match_host_by_serial = True +; Fall back to matching VMs by serial number (BIOS UUID) if no name+cluster match is +; found. Can misattribute a VM to an unrelated NetBox object if the same UUID is reported +; by multiple sources, e.g. a cloned/migrated VM whose stale copy overwrites the real VM's +; cluster/site/status. +;match_vm_by_serial = True + +; Fall back to matching VMs by vNIC MAC address if no name+cluster match is found. Runs +; before 'match_vm_by_serial', so disabling that option alone is not enough if MACs are +; also shared. Same misattribution risk as match_vm_by_serial, triggered by a +; cloned/copied VM with a duplicate MAC. +;match_vm_by_mac_address = True + +; Fall back to matching VMs by primary IP if no name/cluster/MAC/serial match is found. +; Same misattribution risk, triggered even transiently, e.g. a duplicate VM in another +; cluster briefly powered on with the same IP. Not guaranteed to self-correct afterwards, +; since vCenter can keep reporting a cached IP after power-off. +;match_vm_by_ip_address = True + ; Attempt to collect asset tags from vCenter hosts ;collect_hardware_asset_tag = True @@ -271,6 +329,13 @@ password = super-secret ; as "when-undefined" ;set_primary_ip = when-undefined +; defines if primary IP addresses of devices and VMs are protected from removal. If +; enabled, an IP address which is set as primary IPv4/IPv6 of a device or VM in NetBox +; will never be removed from its interface by this source, even if the source does not +; report this IP address (anymore). This prevents the primary IP from being unset when +; i.e. an outdated guest agent does not report all IP addresses. +;preserve_primary_ips = False + ; Do not sync notes from a VM in vCenter to the comments field on a VM in netbox ;skip_vm_comments = False @@ -285,6 +350,23 @@ password = super-secret ; VMs from fail-over site to NetBox. ;skip_srm_placeholder_vms = False +; If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) then this IP +; address will be skipped and not synced to NetBox to prevent incorrect syncing. +;skip_fhrp_group_ips = False + +; defines the status a VM gets assigned in NetBox when netbox-sync creates it as a new +; NetBox VM. Updates of already existing NetBox VMs are not affected by this option. This +; way new VMs can start their lifecycle in NetBox as i.e. 'planned' until changed manually +; in NetBox. possible values: offline, active, planned, staged, failed, decommissioning +;vm_status_on_create = planned + +; defines a comma separated list of NetBox VM statuses which will be preserved on updates. +; If the current status of an existing NetBox VM matches one of these values then netbox- +; sync will not change the status of this VM. This way VMs can be kept in i.e. 'planned' +; or 'staged' until changed manually in NetBox. Set to an empty value to always update the +; VM status. possible values: offline, active, planned, staged, failed, decommissioning +;vm_status_preserve = planned, staged, decommissioning + ; strip domain part from host name before syncing device to NetBox ;strip_host_domain_name = False @@ -294,7 +376,7 @@ password = super-secret ; tag source options ; sync tags assigned to clusters, hosts and VMs in vCenter to NetBox -; INFO: this requires the installation of the 'vsphere-automation-sdk', +; INFO: this requires the installation of the 'vcf-sdk' package, ; see docs about installation possible values: ; * object : the host or VM itself ; * parent_folder_1 : the direct folder this object is organized in (1 level up) @@ -308,6 +390,14 @@ password = super-secret ;host_tag_source = ;vm_tag_source = +; If enabled, vCenter tag names synced to NetBox will include the vCenter category as a +; prefix in the format 'CategoryName:TagName'. Useful if TagName and CategoryName is used +; as key/value pairs in vCenter. +; When changed, existing synced tags are replaced on +; the next run. Note: vm_exclude_by_tag_filter entries must use 'CategoryName:TagName' +; format when this option is enabled. +;tag_name_include_category = False + ; sync custom attributes defined for hosts and VMs in vCenter to NetBox as custom fields ;sync_custom_attributes = False @@ -321,6 +411,17 @@ password = super-secret ;host_custom_object_attributes = summary.runtime.bootTime ;vm_custom_object_attributes = config.uuid +; defines the name of a NetBox custom field which is used to store the hostname reported +; by VMware Tools from inside the guest OS (vCenter property 'guest.hostName'). This is +; independent of the vCenter VM inventory name and can be used to detect naming drift +; between the vCenter VM name and the actual OS hostname. The custom field must be of type +; "Text" and assigned to the "Virtual Machine" object type. If it does not exist yet, it +; will be created automatically, the same way other netbox-sync managed custom fields are +; created. If this option is unset (default) the guest hostname is not synced. If VMware +; Tools does not report a hostname (not installed, not running or no data yet) the custom +; field is left untouched so any previously synced value is preserved. +;vm_guest_hostname_custom_field = vmware_guest_hostname + ; this will set the sources name as cluster group name instead of the datacenter. This ; works if the vCenter has ONLY ONE datacenter configured. Otherwise it will rename all ; datacenters to the source name! @@ -355,7 +456,8 @@ password = super-secret ; which are not present in NetBox will be assigned a VLAN group. ;vlan_group_relation_by_id = 1023-1042 = VLAN Group 1, Tokio/2342 = VLAN Group 2 -; enabling this option will add the ESXi host this VM is running on to the VM details +; fills the 'Host Device' field of a VM in NetBox with the ESXi host it currently runs on. +; Needs NetBox 3.3 or newer ;track_vm_host = False ; define if the name of the device interface discovered overwrites the interface name in @@ -395,6 +497,21 @@ password = super-secret ; host NIC with a matching MAC address will be excluded from sync. ;host_nic_exclude_by_mac_list = AA:BB:CC:11:22:33, 66:77:88:AA:BB:CC +; defines a regex expression to exclude VM interfaces from sync by name. VM interfaces in +; NetBox whose name matches this filter are completely ignored by this source: they are +; excluded from interface matching and will never be updated or altered. Discovered VM +; interfaces with a matching name will be excluded from sync as well. Useful to protect +; interfaces which are managed by other tools inside the guest (i.e. 'tailscale0' or +; 'docker0') from being overwritten with data of a different interface. The filter is +; treated as a regex expression which is only anchored at the beginning of the name ('$' +; can be used to anchor the end) and is case sensitive. If more then one expression should +; match, a '|' needs to be used +;vm_interface_exclude_filter = (tailscale|docker)\d+$ + +; defines a regex expression to exclude host interfaces from sync by name. Same behavior +; as 'vm_interface_exclude_filter' but applies to host (device) interfaces. +;host_interface_exclude_filter = (?i)^ipmi + ; defines a comma separated list of custom attribute which should be excluded from sync. ; Any custom attribute with a matching attribute key will be excluded from sync. ;custom_attribute_exclude = VB_LAST_BACKUP, VB_LAST_BACKUP2 @@ -405,6 +522,17 @@ password = super-secret ; behavior also applies for VM disk sizes. ;vm_disk_and_ram_in_decimal = True +; Skip creating or updating host physical nics in NetBox. Normal operation will maintain +; all physical nics in netbox. This option will skip this part. +;skip_host_nics = False + +; Create cables in NetBox between the physical interfaces (pNICs) of an ESXi host and the +; switch ports which are reported as CDP/LLDP neighbors by this host. A cable is only +; created if the reported switch and the reported switch port both already exist in NetBox +; and if neither of the two interfaces is cabled yet. Cables are visible objects which are +; usually maintained by hand, that's why this is disabled by default. +;sync_host_cables = False + [source/my-redfish-example] ; Defines if this source is enabled or not @@ -425,6 +553,16 @@ inventory_file_path = /full/path/to/inventory/files ; NetBox ;overwrite_host_name = False +; for Dell devices, use the Service Tag as the NetBox device serial number (matching what +; dmidecode and the OS report) instead of the system serial number. The original system +; serial number (the Dell PPID) is then stored in the 'system_serial' custom field +;dell_serial_from_service_tag = False + +; model discovered hardware components (CPUs, memory, drives, controllers, NICs, ...) as +; NetBox modules instead of the deprecated inventory items. Requires NetBox >= 4.3, on +; older versions inventory items are used +;model_components_as_modules = False + ; define if the name of the power supply discovered via check_redfish overwrites the power ; supply name in NetBox ;overwrite_power_supply_name = False @@ -441,6 +579,11 @@ inventory_file_path = /full/path/to/inventory/files ; check_redfish if False only data which is not preset in NetBox will be added ;overwrite_interface_attributes = False +; define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be +; skipped. If True this IP address will be skipped and not synced to NetBox to prevent +; incorrect syncing. +;skip_fhrp_group_ips = False + ; define in which order the IP address tenant will be assigned if tenant is undefined. ; possible values: ; * device : host or VM tenant will be assigned to the IP address @@ -450,4 +593,15 @@ inventory_file_path = /full/path/to/inventory/files ; If the device has a tenant then this one will be used. If not, the prefix tenant will be used if defined ;ip_tenant_inheritance_order = device, prefix +[source/my-hetzner-example] + +; Enable or disable the Hetzner Cloud source. +;enabled = True + +; Source type identifier. Must remain 'hetzner'. +;type = hetzner + +; Hetzner Cloud API token used to authenticate against the Hetzner Cloud API. +api_token = + ;EOF diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..b1335bf9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Shared fixtures for the netbox-sync test suite. + +The integration tests run the real source handlers against vcsim, the vCenter +simulator from the govmomi project, loaded with inventories that were captured +from real vCenters with ``govc object.save`` (see tests/fixtures/vcsim/README.md). +The NetBox side is netbox-sync's own in-memory NetBoxInventory, so no NetBox +instance is needed either. + +vcsim is looked up in ``$VCSIM_BIN`` and then on ``$PATH``. Tests that need it are +skipped when it is not installed; unit tests are unaffected. +""" +import os +import shutil +import socket +import ssl +import subprocess +import tarfile +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from module.config.base import ConfigOptions +from module.config.group import ConfigOptionGroup +from module.config.option import ConfigOption +from module.config.parser import ConfigParser +from module.netbox.connection import NetBoxHandler +from module.netbox.inventory import NetBoxInventory +from module.netbox.object_classes import NBDevice, NBTag +from module.sources import instantiate_sources +from module.sources.check_redfish.config import CheckRedfishConfig +from module.sources.check_redfish.import_inventory import CheckRedfish + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "vcsim" + +# every *.tar.gz in the fixture directory is a vcsim inventory; drop a new capture +# there and the vcsim-backed tests pick it up +VCSIM_DUMPS = sorted(p.name[: -len(".tar.gz")] for p in FIXTURE_DIR.glob("*.tar.gz")) + +# NetBox version the in-memory inventory pretends to be. 4.2 introduced MAC +# address objects, which is the code path current NetBox releases use. +NETBOX_API_VERSION = "4.3.0" + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _wait_for_port(host: str, port: int, process: subprocess.Popen, timeout: float = 30.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + stderr = process.stderr.read().decode(errors="replace") if process.stderr else "" + raise RuntimeError(f"vcsim exited with code {process.returncode}: {stderr.strip()}") + try: + with socket.create_connection((host, port), timeout=1): + return + except OSError: + time.sleep(0.2) + raise RuntimeError(f"vcsim did not start listening on {host}:{port} within {timeout}s") + + +@pytest.fixture(scope="session") +def vcsim_binary() -> str: + binary = os.environ.get("VCSIM_BIN") or shutil.which("vcsim") + if binary is None: + pytest.skip("vcsim not found; set VCSIM_BIN or install it from https://github.com/vmware/govmomi/releases") + return binary + + +@pytest.fixture(scope="session", params=VCSIM_DUMPS, ids=VCSIM_DUMPS) +def vcsim(request, vcsim_binary, tmp_path_factory): + """ + A vcsim process serving one captured inventory. Session scoped, so each dump + is started once per test run; parametrized, so every vcsim-backed test runs + against every dump. + """ + name = request.param + extract_dir = tmp_path_factory.mktemp("vcsim") + with tarfile.open(FIXTURE_DIR / f"{name}.tar.gz") as archive: + archive.extractall(extract_dir, filter="data") + # govc object.save writes into a directory named after the vCenter + load_dir = next(p for p in extract_dir.iterdir() if p.is_dir()) + + host, port = "127.0.0.1", _free_port() + process = subprocess.Popen( + [vcsim_binary, "-load", str(load_dir), "-l", f"{host}:{port}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + try: + _wait_for_port(host, port, process) + # vcsim accepts any credentials + yield SimpleNamespace(name=name, host=host, port=port, username="user", password="pass") + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + + +@pytest.fixture +def inventory(): + """ + A fresh in-memory NetBoxInventory. The class is a singleton with class-level + state, so it is reset before and after each test. + """ + def _reset(): + inv = NetBoxInventory() + inv.base_structure = {} + inv.source_list = [] + inv.init() + inv.netbox_api_version = NETBOX_API_VERSION + return inv + + inv = _reset() + yield inv + _reset() + + +@pytest.fixture +def load_config(tmp_path): + """ + Returns a function that feeds a config file text to netbox-sync's ConfigParser + singleton, replacing whatever a previous test loaded. The file name decides the + format, settings.ini by default. + """ + def _load(text: str, filename: str = "settings.ini") -> ConfigParser: + config_file = tmp_path / filename + config_file.write_text(text) + parser = ConfigParser() + parser.file_list.clear() + parser.content.clear() + parser.config_errors.clear() + parser.config_warnings.clear() + parser.parsing_finished = False + parser.add_config_file(str(config_file)) + parser.read_config() + return parser + + return _load + + +@pytest.fixture +def vmware_settings(vcsim) -> str: + """settings.ini pointing the VMware source at the running vcsim.""" + return f""" +[netbox] +api_token = not-used-by-these-tests +host_fqdn = 127.0.0.1 + +[source/{vcsim.name}] +type = vmware +host_fqdn = {vcsim.host} +port = {vcsim.port} +username = {vcsim.username} +password = {vcsim.password} +validate_tls_certs = False +permitted_subnets = 0.0.0.0/0, ::/0 +dns_name_lookup = False +vm_disk_and_ram_in_decimal = False +""" + + +@pytest.fixture +def vmware_source(vcsim, inventory, load_config, vmware_settings): + """The instantiated VMware source handler for the running vcsim, not yet applied.""" + load_config(vmware_settings) + sources = instantiate_sources() + assert len(sources) == 1 and sources[0].init_successful, "VMware source failed to initialise" + inventory.resolve_relations() + return sources[0] + + +@pytest.fixture +def vmware_sync(inventory, vmware_source): + """The inventory after one full VMware source run, plus the source that produced it.""" + vmware_source.apply() + return SimpleNamespace(inventory=inventory, source=vmware_source) + + +@pytest.fixture +def sdk(vcsim): + """ + A pyVmomi ServiceContent for the running vcsim, independent of netbox-sync, so + tests can compare what was synced with what the SDK reports. + """ + from pyVim import connect + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + instance = connect.SmartConnect( + host=vcsim.host, port=vcsim.port, user=vcsim.username, pwd=vcsim.password, sslContext=context, + ) + try: + yield instance.RetrieveContent() + finally: + connect.Disconnect(instance) + + +@pytest.fixture +def check_redfish_source(inventory): + """ + Returns a function building a minimally initialized CheckRedfish source on the fresh + inventory, with a device to hang components off. The real add_necessary_base_objects() + runs, so the source tag and every custom field are registered as they are in production. + + Settings start from the declared defaults of every CheckRedfishConfig option and are + overridden by keyword arguments, so a test states only what it cares about and an option + added to the config later reaches the tests with its real default. + """ + def _make(**overrides: object) -> SimpleNamespace: + source = object.__new__(CheckRedfish) + source.inventory = inventory + source.name = "test" + source.source_tag = "Source: test" + source.settings = check_redfish_settings(**overrides) + + source.add_necessary_base_objects() + # the primary tag is normally registered by the NetBox handler, not by the source + inventory.add_update_object(NBTag, data={"name": NetBoxHandler.primary_tag}) + + device = inventory.add_object(NBDevice, data={"name": "server01"}, source=source) + source.device_object = device + + return SimpleNamespace(source=source, inventory=inventory, device=device) + + return _make + + +def check_redfish_settings(**overrides) -> ConfigOptions: + """ + The settings a parsed check_redfish config produces: every declared option at its default, + with the given overrides applied. ConfigOptions is what ConfigBase.parse() returns, so an + option this source does not declare reads as None here exactly as it does in production. + """ + values = {} + for entry in CheckRedfishConfig().options: + declared = entry.options if isinstance(entry, ConfigOptionGroup) else [entry] + for option in declared: + if isinstance(option, ConfigOption) and option.removed is not True: + values[option.key] = option.default_value + + unknown = set(overrides) - set(values) + assert not unknown, f"not declared by CheckRedfishConfig: {sorted(unknown)}" + + values.update(overrides) + return ConfigOptions(**values) diff --git a/tests/fixtures/vcsim/README.md b/tests/fixtures/vcsim/README.md new file mode 100644 index 00000000..424f2454 --- /dev/null +++ b/tests/fixtures/vcsim/README.md @@ -0,0 +1,35 @@ +# vcsim inventory fixtures + +Each `*.tar.gz` here is a vCenter inventory captured with `govc object.save` and +replayed by [vcsim](https://github.com/vmware/govmomi/tree/main/vcsim), the +simulator from the govmomi project. The integration tests start one vcsim per +archive and run the real VMware source handler against it, so they exercise the +actual pyVmomi code path without a live vCenter. + +| archive | vCenter | hosts | VMs | notes | +| --- | --- | --- | --- | --- | +| `vchvr.tar.gz` | 6.7.0 | 2 | 28 | most VMs report running guest tools, so IP handling is covered | +| `vc001.tar.gz` | 8.0.3 | 3 | 34 | current API version, a cluster with vCLS VMs | + +Both were contributed in [#474](https://github.com/bb-Ricardo/netbox-sync/issues/474). + +## Adding a capture + +Point `govc` at a vCenter and save its inventory: + +```bash +export GOVC_URL="https://vcenter/sdk" +export GOVC_USERNAME="administrator@vsphere.local" +export GOVC_PASSWORD="..." +export GOVC_INSECURE=true + +govc object.save -d my-vcenter +tar -czf my-vcenter.tar.gz my-vcenter +``` + +Drop the archive in this directory and the vcsim-backed tests pick it up: they +are parametrized over every archive found here, so a new capture is covered +without touching the test code. + +The dumps contain host names, VM names, MAC addresses and guest IP addresses of +the source environment. Only capture inventories you are allowed to publish. diff --git a/tests/fixtures/vcsim/vc001.tar.gz b/tests/fixtures/vcsim/vc001.tar.gz new file mode 100644 index 00000000..5656d1f3 Binary files /dev/null and b/tests/fixtures/vcsim/vc001.tar.gz differ diff --git a/tests/fixtures/vcsim/vchvr.tar.gz b/tests/fixtures/vcsim/vchvr.tar.gz new file mode 100644 index 00000000..e87247cb Binary files /dev/null and b/tests/fixtures/vcsim/vchvr.tar.gz differ diff --git a/tests/test_check_redfish_device_serial.py b/tests/test_check_redfish_device_serial.py new file mode 100644 index 00000000..9b61c789 --- /dev/null +++ b/tests/test_check_redfish_device_serial.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +"""The Dell Service Tag can be used as the NetBox device serial instead of the system serial. + +Drives the real update_device() and find_device_object() against real NBDevice objects. +""" + +from module.common.misc import grab +from module.netbox.object_classes import NBDevice + +SYSTEM_SERIAL = "CNEXAMPLE00001" +SERVICE_TAG = "ABC1234" + + +def dell_system(system_serial=SYSTEM_SERIAL, service_tag=SERVICE_TAG, with_chassis=True): + """A Dell system as check_redfish reports it: system.serial is the board PPID, + the Service Tag is chassis.sku.""" + + content = {"inventory": {"system": [ + {"id": "1", "name": "System", "manufacturer": "Dell Inc.", "model": "PowerEdge R650", + "serial": system_serial, "host_name": "server01", + "health_status": "OK", "power_state": "On"}]}} + if with_chassis: + content["inventory"]["chassis"] = [{"id": "1", "sku": service_tag}] + return content + + +def run_update_device(source, content, dell_serial_from_service_tag): + source.settings.dell_serial_from_service_tag = dell_serial_from_service_tag + source.inventory_file_content = content + source.update_device() + return source.device_object + + +def match_content(system_serial=SYSTEM_SERIAL, service_tag=SERVICE_TAG): + """An inventory file without meta.inventory_id, so matching falls back to the serial.""" + + return {"inventory": { + "system": [{"manufacturer": "Dell Inc.", "serial": system_serial}], + "chassis": [{"sku": service_tag}], + }} + + +def test_serial_defaults_to_the_system_serial(check_redfish_source): + """Existing behaviour with the option off, which must not change.""" + + context = check_redfish_source() + device = run_update_device(context.source, dell_system(), dell_serial_from_service_tag=False) + + assert device.data["serial"] == SYSTEM_SERIAL + assert grab(device, "data.custom_fields.service_tag") == SERVICE_TAG + assert grab(device, "data.custom_fields.system_serial") is None + + +def test_option_makes_the_service_tag_the_serial(check_redfish_source): + context = check_redfish_source() + device = run_update_device(context.source, dell_system(), dell_serial_from_service_tag=True) + + assert device.data["serial"] == SERVICE_TAG + assert grab(device, "data.custom_fields.system_serial") == SYSTEM_SERIAL + assert grab(device, "data.custom_fields.service_tag") == SERVICE_TAG + + +def test_option_falls_back_when_no_service_tag_is_reported(check_redfish_source): + """No Service Tag means no swap, so the serial is not lost.""" + + context = check_redfish_source() + device = run_update_device(context.source, dell_system(with_chassis=False), + dell_serial_from_service_tag=True) + + assert device.data["serial"] == SYSTEM_SERIAL + assert grab(device, "data.custom_fields.system_serial") is None + + +def test_blank_service_tag_is_not_a_service_tag(check_redfish_source): + context = check_redfish_source() + device = run_update_device(context.source, dell_system(service_tag=" "), + dell_serial_from_service_tag=True) + + assert grab(device, "data.custom_fields.service_tag") is None + assert device.data["serial"] == SYSTEM_SERIAL + assert grab(device, "data.custom_fields.system_serial") is None + + +def test_system_serial_custom_field_is_not_overwritten_with_none(check_redfish_source): + """A transient missing system serial must not clear the field on a later sync.""" + + context = check_redfish_source() + device = run_update_device(context.source, dell_system(), dell_serial_from_service_tag=True) + assert grab(device, "data.custom_fields.system_serial") == SYSTEM_SERIAL + + run_update_device(context.source, dell_system(system_serial=None), dell_serial_from_service_tag=True) + + assert grab(device, "data.custom_fields.system_serial") == SYSTEM_SERIAL + + +def test_device_is_matched_by_system_serial(check_redfish_source): + """Existing fallback matching, which must not change.""" + + context = check_redfish_source() + context.source.settings.dell_serial_from_service_tag = False + existing = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = match_content() + + assert context.source.find_device_object("dell-host.json") is True + assert context.source.device_object is existing + + +def test_device_not_yet_migrated_is_matched_by_system_serial_with_the_option_on(check_redfish_source): + context = check_redfish_source() + context.source.settings.dell_serial_from_service_tag = True + existing = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = match_content() + + assert context.source.find_device_object("dell-host.json") is True + assert context.source.device_object is existing + + +def test_device_persisted_with_the_service_tag_is_matched_by_it(check_redfish_source): + """Without the Service Tag fallback such a device is skipped and stops being updated.""" + + context = check_redfish_source() + context.source.settings.dell_serial_from_service_tag = True + existing = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SERVICE_TAG}, source=context.source) + + context.source.inventory_file_content = match_content() + + assert context.source.find_device_object("dell-host.json") is True + assert context.source.device_object is existing + + +def test_service_tag_matching_does_not_depend_on_the_option(check_redfish_source): + """Disabling the option must not strand a device already persisted with the Service Tag.""" + + context = check_redfish_source() + context.source.settings.dell_serial_from_service_tag = False + existing = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SERVICE_TAG}, source=context.source) + + context.source.inventory_file_content = match_content() + + assert context.source.find_device_object("dell-host.json") is True + assert context.source.device_object is existing + + +def test_padded_system_serial_still_matches(check_redfish_source): + """update_device() stores the serial stripped, so the lookup must strip it too.""" + + context = check_redfish_source() + existing = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = match_content(system_serial=f" {SYSTEM_SERIAL} ") + + assert context.source.find_device_object("dell-host.json") is True + assert context.source.device_object is existing + + +def test_missing_serial_does_not_match_a_serial_less_device(check_redfish_source): + """get_by_data() matches on exact dict equality, so probing serial=None would match wrongly.""" + + context = check_redfish_source() + context.inventory.add_object(NBDevice, data={"name": "serial-less"}, source=context.source) + + context.source.inventory_file_content = {"inventory": { + "system": [{"manufacturer": "Dell Inc."}], + }} + + assert context.source.find_device_object("dell-host.json") is False + + +def seed_device_with_nb_id(source, inventory, nb_id, name="wrong-device"): + device = inventory.add_object(NBDevice, data={"name": name}, source=source) + device.nb_id = nb_id + return device + + +def id_content(inventory_id): + content = match_content() + content["meta"] = {"inventory_id": inventory_id} + return content + + +def test_integer_inventory_id_is_used(check_redfish_source): + """The normal path, which must keep working.""" + + context = check_redfish_source() + wanted = seed_device_with_nb_id(context.source, context.inventory, 1, name="by-id") + + context.source.inventory_file_content = id_content(1) + + assert context.source.find_device_object("host.json") is True + assert context.source.device_object is wanted + + +def test_digit_string_inventory_id_is_used(check_redfish_source): + """meta.inventory_id arrives from JSON, where it may be quoted.""" + + context = check_redfish_source() + wanted = seed_device_with_nb_id(context.source, context.inventory, 1, name="by-id") + + context.source.inventory_file_content = id_content("1") + + assert context.source.find_device_object("host.json") is True + assert context.source.device_object is wanted + + +def test_boolean_inventory_id_does_not_match_device_one(check_redfish_source): + """int(True) is 1, so a JSON `true` would silently claim the device with id 1.""" + + context = check_redfish_source() + seed_device_with_nb_id(context.source, context.inventory, 1) + by_serial = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = id_content(True) + + assert context.source.find_device_object("host.json") is True + assert context.source.device_object is by_serial + + +def test_float_inventory_id_does_not_match_the_truncated_device(check_redfish_source): + """int(1.9) is 1, so a JSON float would silently claim the device with id 1.""" + + context = check_redfish_source() + seed_device_with_nb_id(context.source, context.inventory, 1) + by_serial = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = id_content(1.9) + + assert context.source.find_device_object("host.json") is True + assert context.source.device_object is by_serial + + +def test_non_positive_inventory_id_is_rejected(check_redfish_source): + """NetBox ids start at 1, so zero and negatives are not usable ids.""" + + context = check_redfish_source() + seed_device_with_nb_id(context.source, context.inventory, 0) + by_serial = context.inventory.add_object( + NBDevice, data={"name": "dell-host", "serial": SYSTEM_SERIAL}, source=context.source) + + context.source.inventory_file_content = id_content(0) + + assert context.source.find_device_object("host.json") is True + assert context.source.device_object is by_serial diff --git a/tests/test_check_redfish_fan_speed.py b/tests/test_check_redfish_fan_speed.py new file mode 100644 index 00000000..be02beb2 --- /dev/null +++ b/tests/test_check_redfish_fan_speed.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +A fan's reading is a live measurement. Syncing it means NetBox records a change on +every run, for every fan of every server (reported by @marcinpsk on #473). +""" +from module.sources.check_redfish.import_inventory import CheckRedfish + + +def _fan(reading): + return {"inventory": {"fan": [{ + "id": "Fan.Embedded.1", "name": "Fan1A", "health_status": "OK", + "physical_context": "SystemBoard", "reading": reading, "reading_unit": "RPM", + }]}} + + +def _collect(inventory, reading): + source = object.__new__(CheckRedfish) + source.inventory = inventory + source.name = "redfish" + source.source_tag = "Source: redfish" + source.inventory_file_content = _fan(reading) + collected = [] + source.update_all_items = lambda items, inventory_type: collected.extend(items) + source.update_fan() + return collected + + +def test_a_spinning_fan_produces_the_same_item(inventory): + slow = _collect(inventory, 7015) + fast = _collect(inventory, 9120) + + assert slow == fast, "the fan item changes with its rpm, so NetBox is written on every run" + + +def test_the_fan_is_still_described(inventory): + # control: dropping the reading must not empty the item + item = _collect(inventory, 8280)[0] + + assert item["full_name"] == "Fan1A (ID: Fan.Embedded.1)" + assert item["health"] == "OK" + assert "Context: SystemBoard" in item["description"] diff --git a/tests/test_check_redfish_interface_ips.py b/tests/test_check_redfish_interface_ips.py new file mode 100644 index 00000000..182d5de6 --- /dev/null +++ b/tests/test_check_redfish_interface_ips.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +"""An interface the source discovered no IPs for must keep the IPs it already has. + +Drives the real add_update_interface() IP removal loop against real NBInterface and +NBIPAddress objects. +""" + +from module.netbox.object_classes import NBInterface, NBIPAddress + + +def seed_interface_with_ip(context, name="pnet0", address="172.10.10.12/24"): + interface = context.inventory.add_object( + NBInterface, data={"name": name, "device": context.device}, source=context.source) + ip = context.inventory.add_object( + NBIPAddress, data={"address": address, "assigned_object_id": interface}, source=context.source) + assert ip in interface.get_ip_addresses() + return interface, ip + + +def test_ip_is_kept_when_the_source_discovered_no_ips(check_redfish_source): + """The management IP on a bond or bridge matched only by a shared MAC must survive a sync.""" + + context = check_redfish_source() + interface, ip = seed_interface_with_ip(context) + + context.source.add_update_interface(interface, context.device, {"name": "pnet0"}, [], keep_undiscovered_ips=True) + + # unset_attribute() queues the de-assignment in unset_items, it does not mutate data + assert "assigned_object_id" not in ip.unset_items + + +def test_ip_is_still_removed_by_default(check_redfish_source): + """Other sources are unchanged: an IP no longer reported is still removed.""" + + context = check_redfish_source() + interface, ip = seed_interface_with_ip(context) + + context.source.add_update_interface(interface, context.device, {"name": "pnet0"}, []) + + assert "assigned_object_id" in ip.unset_items + + +def test_ip_is_still_removed_when_other_ips_are_discovered(check_redfish_source): + """The guard covers an empty discovery only. An IP dropped from a non-empty set still goes.""" + + context = check_redfish_source() + interface, ip = seed_interface_with_ip(context) + + context.source.add_update_interface(interface, context.device, {"name": "pnet0"}, ["198.51.100.7/24"], + keep_undiscovered_ips=True) + + assert "assigned_object_id" in ip.unset_items + + +def test_ip_is_still_removed_when_the_discovered_ips_are_unusable(check_redfish_source): + """A non-empty discovery is a statement about the interface even when none of the addresses + survive parsing, so the guard must not treat it as "discovered nothing".""" + + context = check_redfish_source() + interface, ip = seed_interface_with_ip(context) + + context.source.add_update_interface(interface, context.device, {"name": "pnet0"}, ["not-an-ip"], + keep_undiscovered_ips=True) + + assert "assigned_object_id" in ip.unset_items diff --git a/tests/test_check_redfish_inventory_items.py b/tests/test_check_redfish_inventory_items.py new file mode 100644 index 00000000..9335bdc9 --- /dev/null +++ b/tests/test_check_redfish_inventory_items.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +"""Integration tests for the check_redfish source and the inventory items it maintains. + +Drives the real CheckRedfish methods against the real NetBoxInventory and NetBoxObject +classes. Only the NetBox REST API itself is out of scope. +""" + +from module.netbox.object_classes import NBInventoryItem + +# a Dell `location` as check_redfish can hand it back: a nested Oem object, not a string +DELL_LOCATION = { + "Oem": { + "Dell": { + "@odata.type": "#DellLocation.v1_2_0.DellLocation", + "Locator": "BP_PSV 0:1", + } + } +} + + +def enclosure(name, location, serial="ENC-AAA"): + return {"inventory": {"storage_enclosure": [ + {"name": name, "model": "BP14G+EXP", "location": location, + "manufacturer": "DELL", "serial": serial, "part_number": "PN-ENC", + "firmware": "1.0", "health_status": "OK", "num_bays": 24, + "operation_status": "Enabled"}]}} + + +def test_structured_location_is_not_stringified_into_the_item_name(check_redfish_source): + """A structured location must not reach the name as its Python repr.""" + + context = check_redfish_source() + + context.source.inventory_file_content = enclosure("BP_PSV 0:1", DELL_LOCATION) + context.source.update_storage_enclosure() + + items = context.inventory.get_all_items(NBInventoryItem) + assert len(items) == 1 + + name = items[0].data["name"] + assert "Oem" not in name + assert "@odata.type" not in name + assert "{" not in name + assert len(name) <= 64 + assert name == "BP_PSV 0:1" + + +def test_plain_string_location_is_kept_in_the_item_name(check_redfish_source): + """A location that really is a string is still used.""" + + context = check_redfish_source() + + context.source.inventory_file_content = enclosure("BP_PSV 0:1", "Slot 3") + context.source.update_storage_enclosure() + + items = context.inventory.get_all_items(NBInventoryItem) + assert len(items) == 1 + assert items[0].data["name"] == "BP_PSV 0:1 Slot 3" + + +def test_two_enclosures_with_structured_locations_stay_distinct(check_redfish_source): + """Dropping the unusable location must not merge two enclosures onto one name.""" + + context = check_redfish_source() + + context.source.inventory_file_content = {"inventory": {"storage_enclosure": [ + enclosure("BP_PSV 0:1", DELL_LOCATION, "ENC-AAA")["inventory"]["storage_enclosure"][0], + enclosure("BP_PSV 0:2", DELL_LOCATION, "ENC-BBB")["inventory"]["storage_enclosure"][0], + ]}} + context.source.update_storage_enclosure() + + names = sorted(item.data["name"] for item in context.inventory.get_all_items(NBInventoryItem)) + assert names == ["BP_PSV 0:1", "BP_PSV 0:2"] diff --git a/tests/test_check_redfish_module_catalog.py b/tests/test_check_redfish_module_catalog.py new file mode 100644 index 00000000..fef76e3e --- /dev/null +++ b/tests/test_check_redfish_module_catalog.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Modelling components as modules has to read the existing modules back from NetBox, +otherwise every run tries to create them again, and a module type has to be the +hardware model so the catalog is shared instead of holding one entry per component. + +Modules are read back on every run regardless of the option, so interfaces and power +ports that reference a module can always resolve that relation - even on a run where the +option is off again. +""" +from types import SimpleNamespace + +import pytest + +from module.netbox.object_classes import NBModule, NBModuleBay, NBModuleType +from module.sources.check_redfish.import_inventory import CheckRedfish + + +def _source(inventory, use_modules): + source = object.__new__(CheckRedfish) + source.inventory = inventory + source.name = "redfish" + source.source_tag = "Source: redfish" + source.settings = SimpleNamespace(model_components_as_modules=use_modules) + source.device_object = None + return source + + +def test_module_objects_are_always_requested_so_relations_resolve(): + # Modules must be read back on every run, even with the option off. Interfaces and power + # ports created by an earlier modules-on run reference a module, and that relation only + # resolves when the module objects were loaded. Against a live NetBox an option-off run + # otherwise logs "Problems resolving relation 'module'" for every such object. + for module_class in (NBModuleBay, NBModuleType, NBModule): + assert module_class in CheckRedfish.dependent_netbox_objects, \ + f"{module_class.name} must always be read from NetBox so module relations resolve" + + +@pytest.mark.parametrize("item_data, expected", [ + ({"model": "ST2000NX0273", "full_name": "Disk.Bay.0 (HDD ST2000NX0273)", + "inventory_type": "Physical Drive"}, "ST2000NX0273"), + ({"part_number": "MTA36ASF8G72PZ", "full_name": "DIMM.A1 (DDR4)", + "inventory_type": "DIMM"}, "MTA36ASF8G72PZ"), + ({"full_name": "Fan1A (ID: Fan.Embedded.1)", "inventory_type": "Fan"}, "Fan"), +]) +def test_module_type_is_the_hardware_model_not_the_instance_name(inventory, item_data, expected): + source = _source(inventory, True) + source.device_manufacturer_name = lambda: "Dell" + + module_type = source.resolve_module_type(item_data) + + assert module_type.data.get("model") == expected diff --git a/tests/test_check_redfish_modules.py b/tests/test_check_redfish_modules.py new file mode 100644 index 00000000..83e10089 --- /dev/null +++ b/tests/test_check_redfish_modules.py @@ -0,0 +1,950 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +"""Modeling check_redfish hardware components as NetBox modules. + +Drives the real CheckRedfish methods against the real NetBoxInventory and NetBoxObject +classes. Only the NetBox REST API itself is out of scope. +""" + +import pytest + +from module.common.misc import grab +from module.netbox.object_classes import ( + NBDevice, + NBDeviceType, + NBInterface, + NBInventoryItem, + NBManufacturer, + NBModule, + NBModuleBay, + NBModuleType, + NBPowerPort, +) + + +@pytest.fixture +def modules_source(check_redfish_source): + """The shared check_redfish fixture, with the modules option and a NetBox version to test.""" + def _make(model_components_as_modules: bool, netbox_api_version: str, **extra: object): + context = check_redfish_source( + model_components_as_modules=model_components_as_modules, **extra) + context.inventory.netbox_api_version = netbox_api_version + return context.source, context.inventory, context.device + return _make + + +def cpu_item(bay_name="Socket 1", + model="Intel Xeon Gold 6248R", + serial="CPU-AAA", + manufacturer="Intel", + health="OK", + full_name=None): + """Build a normalized CPU component item as produced by CheckRedfish.update_proc(). + + bay_name is the stable physical slot (the module bay identity); full_name is the + display name and by default embeds the model, exactly like the real parser does. + """ + return { + "description": ["x86-64", "Cores: 24", "Threads: 48"], + "manufacturer": manufacturer, + "bay_name": bay_name, + "full_name": full_name if full_name is not None else f"{bay_name} ({model})", + "model": model, + "serial": serial, + "health": health, + "size": "24/48", + "speed": "3.0GHz", + } + + +@pytest.mark.parametrize("flag, api_version, expected", [ + (True, "4.3.0", True), + (True, "4.3.1", True), + (True, "5.0.0", True), + (True, "4.2.9", False), # NetBox too old -> fall back to inventory items + (True, "4.0.0", False), + (False, "4.3.0", False), # feature disabled -> inventory items + (False, "5.0.0", False), +]) +def test_use_modules_decision_matrix(modules_source, flag, api_version, expected): + source, _, _ = modules_source(flag, api_version) + assert source.use_modules() is expected + + +def test_creates_full_module_graph_for_cpu(modules_source): + source, inventory, device = modules_source(True, "4.3.0") + + source.update_all_items([cpu_item()], "CPU") + + modules = inventory.get_all_items(NBModule) + bays = inventory.get_all_items(NBModuleBay) + module_types = inventory.get_all_items(NBModuleType) + + # exactly one of each object is created and no deprecated inventory item is touched + assert len(modules) == 1 + assert len(bays) == 1 + assert len(module_types) == 1 + assert len(inventory.get_all_items(NBInventoryItem)) == 0 + + module = modules[0] + bay = bays[0] + module_type = module_types[0] + + # the module is wired to the device, its bay and its module type (same object instances) + assert module.data["device"] is device + assert module.data["module_bay"] is bay + assert module.data["module_type"] is module_type + assert module.data["status"] == "active" + assert module.data["serial"] == "CPU-AAA" + + # descriptive data lives in custom fields on the module + assert grab(module, "data.custom_fields.inventory_type") == "CPU" + assert grab(module, "data.custom_fields.inventory_size") == "24/48" + assert grab(module, "data.custom_fields.inventory_speed") == "3.0GHz" + assert grab(module, "data.custom_fields.health") == "OK" + + # the bay is the stable physical slot (model lives in the module type, not the bay name) + assert bay.data["name"] == "Socket 1" + assert bay.data["device"] is device + + # the module type is the catalog entry carrying the real CPU model + manufacturer + assert module_type.data["model"] == "Intel Xeon Gold 6248R" + assert grab(module_type, "data.manufacturer.data.name") == "Intel" + + # the module derives its display name from the bay (it has no name of its own) + assert module.get_display_name(including_second_key=True) == "Socket 1 (server01)" + + +def test_module_sync_is_idempotent(modules_source): + source, inventory, _ = modules_source(True, "4.3.0") + + source.update_all_items([cpu_item()], "CPU") + source.update_all_items([cpu_item()], "CPU") + + # a second run with identical data must not create duplicates + assert len(inventory.get_all_items(NBModule)) == 1 + assert len(inventory.get_all_items(NBModuleBay)) == 1 + assert len(inventory.get_all_items(NBModuleType)) == 1 + + +def test_same_model_reuses_module_type_across_devices(modules_source): + source, inventory, _ = modules_source(True, "4.3.0") + + # first device gets a CPU + source.update_all_items([cpu_item(serial="CPU-AAA")], "CPU") + + # a second device with the exact same CPU model + device2 = inventory.add_object(NBDevice, data={"name": "server02"}, source=source) + source.device_object = device2 + source.update_all_items([cpu_item(serial="CPU-BBB")], "CPU") + + # the module type (catalog entry) is shared, but each device gets its own bay + module + assert len(inventory.get_all_items(NBModuleType)) == 1 + assert len(inventory.get_all_items(NBModule)) == 2 + assert len(inventory.get_all_items(NBModuleBay)) == 2 + assert len(inventory.get_all_items(NBManufacturer)) == 1 + + +def test_different_model_creates_distinct_module_type(modules_source): + """This is the 'one server type, different CPUs' use case.""" + source, inventory, _ = modules_source(True, "4.3.0") + + source.update_all_items([cpu_item(model="Intel Xeon Gold 6248R", serial="CPU-AAA")], "CPU") + + device2 = inventory.add_object(NBDevice, data={"name": "server02"}, source=source) + source.device_object = device2 + source.update_all_items([cpu_item(model="Intel Xeon Gold 5318Y", serial="CPU-BBB")], "CPU") + + models = sorted(grab(mt, "data.model") for mt in inventory.get_all_items(NBModuleType)) + assert models == ["Intel Xeon Gold 5318Y", "Intel Xeon Gold 6248R"] + assert len(inventory.get_all_items(NBModule)) == 2 + + +def test_same_bay_new_model_updates_module_type(modules_source): + """Same device + same physical bay + a replaced CPU model across runs. + + Regression for CodeRabbit PR #1: the bay must be keyed on a stable slot (not the + model-bearing display name), so a model swap reuses the same bay and the module + re-points to the new module type instead of churning the bay or keeping a stale type. + """ + source, inventory, _ = modules_source(True, "4.3.0") + + # first run: CPU model A installed in socket "Socket 1" + source.update_all_items([cpu_item(bay_name="Socket 1", + model="Intel Xeon Gold 6248R", serial="CPU-AAA")], "CPU") + + # second run: same device, same socket, a different CPU model is now installed + source.update_all_items([cpu_item(bay_name="Socket 1", + model="Intel Xeon Gold 5318Y", serial="CPU-BBB")], "CPU") + + bays = inventory.get_all_items(NBModuleBay) + modules = inventory.get_all_items(NBModule) + + # the physical bay is stable: still a single bay holding a single module on the device + assert len(bays) == 1 + assert len(modules) == 1 + assert bays[0].data["name"] == "Socket 1" + + module = modules[0] + # the module re-points to the replaced part's module type (no stale catalog reference) + assert grab(module, "data.module_type.data.model") == "Intel Xeon Gold 5318Y" + assert module.data["serial"] == "CPU-BBB" + + +def test_missing_component_marks_module_health_absent(modules_source): + source, inventory, _ = modules_source(True, "4.3.0") + + cpu1 = cpu_item(bay_name="Socket 1", serial="CPU-AAA") + cpu2 = cpu_item(bay_name="Socket 2", serial="CPU-BBB") + source.update_all_items([cpu1, cpu2], "CPU") + + assert len(inventory.get_all_items(NBModule)) == 2 + + # second CPU disappears from the inventory file + source.update_all_items([cpu1], "CPU") + + modules_by_bay = { + grab(m, "data.module_bay.data.name"): m for m in inventory.get_all_items(NBModule) + } + assert grab(modules_by_bay["Socket 1"], "data.custom_fields.health") == "OK" + assert grab(modules_by_bay["Socket 2"], "data.custom_fields.health") == "Absent" + + +def test_mixed_bay_transition_does_not_remap_modules(modules_source): + """One bay disappears while a different new bay appears in the same sync. Because the module + bay is the authoritative physical slot (and update_module never moves a module between bays), + the removed bay must go Absent and the new bay must get its own module - the new component + must NOT be silently remapped onto the removed slot.""" + source, inventory, _ = modules_source(True, "4.3.0") + + source.update_all_items([ + cpu_item(bay_name="Socket 1", serial="CPU-AAA"), + cpu_item(bay_name="Socket 2", serial="CPU-BBB"), + ], "CPU") + assert len(inventory.get_all_items(NBModule)) == 2 + + # Socket 2 is removed and a brand new Socket 3 appears in the same run + source.update_all_items([ + cpu_item(bay_name="Socket 1", serial="CPU-AAA"), + cpu_item(bay_name="Socket 3", serial="CPU-CCC"), + ], "CPU") + + modules_by_bay = { + grab(m, "data.module_bay.data.name"): m for m in inventory.get_all_items(NBModule) + } + # three distinct bays now exist: the kept one, the removed one (Absent), and the new one + assert set(modules_by_bay) == {"Socket 1", "Socket 2", "Socket 3"} + assert grab(modules_by_bay["Socket 1"], "data.custom_fields.health") == "OK" + # the removed slot is marked Absent and keeps its own component data (not overwritten) + assert grab(modules_by_bay["Socket 2"], "data.custom_fields.health") == "Absent" + assert grab(modules_by_bay["Socket 2"], "data.serial") == "CPU-BBB" + # the new slot is its own active module carrying the new component's data + assert grab(modules_by_bay["Socket 3"], "data.serial") == "CPU-CCC" + assert grab(modules_by_bay["Socket 3"], "data.custom_fields.health") == "OK" + + +def fan_item(bay_name="System Board Fan1 (ID: 0.56)", health="OK"): + """A component that reports no manufacturer (fans, enclosures, PCIe extenders, ...).""" + return { + "description": ["Context: SystemBoard"], + "full_name": bay_name, + "health": health, + "speed": "9240RPM", + } + + +def test_component_without_manufacturer_uses_device_manufacturer(modules_source): + """NetBox requires a manufacturer on a module type. Components that report none (fans, + storage enclosures, PCIe extenders) must still get one, otherwise the module-type POST + fails with 'manufacturer required' and the whole module create cascade fails.""" + source, inventory, device = modules_source(True, "4.3.0") + + # give the device a manufacturer via its device type, like a real synced device has + manufacturer = inventory.add_object(NBManufacturer, data={"name": "Acme"}, source=source) + device_type = inventory.add_object( + NBDeviceType, data={"model": "PowerEdge R650", "manufacturer": manufacturer}, source=source) + device.update(data={"device_type": device_type}, source=source) + + source.update_all_items([fan_item()], "Fan") + + module_types = inventory.get_all_items(NBModuleType) + assert len(module_types) == 1 + assert len(inventory.get_all_items(NBModule)) == 1 + + # the (required) manufacturer is populated from the device's vendor + device_manufacturer = grab(device, "data.device_type.data.manufacturer.data.name") + assert grab(module_types[0], "data.manufacturer.data.name") == device_manufacturer + + +def test_component_without_manufacturer_falls_back_to_unknown(modules_source): + """When neither the component nor the device exposes a manufacturer, fall back to a + placeholder so the required module type field is always populated.""" + source, inventory, _ = modules_source(True, "4.3.0") # device has no device type / manufacturer + + source.update_all_items([fan_item()], "Fan") + + module_types = inventory.get_all_items(NBModuleType) + assert len(module_types) == 1 + assert grab(module_types[0], "data.manufacturer.data.name") == "Unknown" + assert len(inventory.get_all_items(NBModule)) == 1 + + +def test_existing_module_type_manufacturer_is_preserved(modules_source): + """If a module type for this model already exists in NetBox with a manufacturer (set by a + previous sync or curated by hand), a later sync of a component that reports no manufacturer + must reuse it, not overwrite it with the device-vendor / 'Unknown' fallback.""" + source, inventory, _ = modules_source(True, "4.3.0") + + # a module type for this model already exists in NetBox, manufacturer "Globex" + globex = inventory.add_object(NBManufacturer, data={"name": "Globex"}, source=source) + inventory.add_object( + NBModuleType, data={"model": "PCIe Extender", "manufacturer": globex}, source=source) + + # the PCIe extender reports no manufacturer + source.update_all_items([{ + "full_name": "PCIe Extender", + "model": "PCIe Extender", + "health": "OK", + "description": ["LDs: 1, PDs: 1"], + }], "Storage Controller") + + module_types = [mt for mt in inventory.get_all_items(NBModuleType) + if grab(mt, "data.model") == "PCIe Extender"] + assert len(module_types) == 1 + # the pre-existing manufacturer is preserved, not clobbered by the fallback + assert grab(module_types[0], "data.manufacturer.data.name") == "Globex" + + +def test_nic_and_bmc_interfaces_are_attached_to_their_modules(modules_source): + """NIC port interfaces are attached to their adapter's module and the BMC interface to the + manager module, so NetBox cascade-deletes them when the module is removed (module FK).""" + source, inventory, _ = modules_source(True, "4.3.0") + source.interface_adapter_type_dict = {} + source.nic_module_bay_by_adapter_id = {} + source.manager_name = None + source.settings.overwrite_interface_name = False + source.settings.overwrite_interface_attributes = False + source.settings.permitted_subnets = None # ports carry no IPs, so this is never dereferenced + source.settings.ip_tenant_inheritance_order = [] + + source.inventory_file_content = { + "inventory": { + "manager": [ + {"name": "iDRAC 9", "model": None, "licenses": [], "firmware": "7.0", + "health_status": "OK"} + ], + "network_adapter": [ + {"id": "NIC.Slot.1", "name": "NIC.Slot.1", "model": "BCM57414", + "manufacturer": "Broadcom", "operation_status": "Enabled", "num_ports": "2", + "serial": "NIC-AAA", "firmware": "1.0"} + ], + "network_port": [ + {"id": "NIC.Slot.1-1", "name": "Slot 1 Port 1", "adapter_id": "NIC.Slot.1", + "operation_status": "Enabled", "link_status": "Up", "addresses": [], + "capable_speed": 10000, "manager_ids": []}, + {"id": "NIC.1", "name": "iDRAC", "adapter_id": None, + "operation_status": "Enabled", "link_status": "Up", "addresses": [], + "capable_speed": 1000, "manager_ids": ["iDRAC.Embedded.1"]}, + ], + } + } + + source.update_manager() + source.update_network_adapter() + source.update_network_interface() + + interfaces = {grab(i, "data.name"): i for i in inventory.get_all_items(NBInterface)} + nic_interface = interfaces["NIC.Slot.1-1"] + bmc_interface = interfaces["iDRAC 9 (NIC.1)"] + + # the NIC port belongs to its adapter's module, the BMC port to the manager module + assert grab(nic_interface, "data.module.data.module_bay.data.name") == "NIC.Slot.1" + assert grab(bmc_interface, "data.module.data.module_bay.data.name") == "iDRAC 9" + + # and it really is the same module object created for this device + assert grab(nic_interface, "data.module") is source.find_device_module_by_bay_name("NIC.Slot.1") + + +def test_nic_port_interface_named_by_stable_redfish_id(modules_source): + """With modules on, a NIC port is named by its stable redfish id (e.g. NIC.Slot.1-1) rather + than the long human label prepended to it; the descriptive label moves to the description.""" + source, inventory, _ = modules_source(True, "4.3.0") + source.interface_adapter_type_dict = {} + source.nic_module_bay_by_adapter_id = {} + source.manager_name = None + source.settings.overwrite_interface_name = False + source.settings.overwrite_interface_attributes = False + source.settings.permitted_subnets = None + source.settings.ip_tenant_inheritance_order = [] + + source.inventory_file_content = { + "inventory": { + "network_adapter": [ + {"id": "NIC.Integrated.1", "name": "NIC.Integrated.1", "model": "BCM57412", + "manufacturer": "Broadcom", "operation_status": "Enabled", "num_ports": "1"} + ], + "network_port": [ + {"id": "NIC.Integrated.1-1", "name": "Integrated NIC 1 Port 1 Partition 1", + "adapter_id": "NIC.Integrated.1", "operation_status": "Enabled", + "link_status": "Up", "addresses": [], "capable_speed": 10000, + "manager_ids": []}, + ], + } + } + + source.update_network_adapter() + source.update_network_interface() + + interfaces = {grab(i, "data.name"): i for i in inventory.get_all_items(NBInterface)} + + # the stable id is the name; the description carries the human label, not the name + assert "NIC.Integrated.1-1" in interfaces + assert "Integrated NIC 1 Port 1 Partition 1 (NIC.Integrated.1-1)" not in interfaces + assert grab(interfaces["NIC.Integrated.1-1"], "data.description") == \ + "Integrated NIC 1 Port 1 Partition 1" + + +def test_nic_module_bay_stable_when_adapter_label_changes(modules_source): + """The NIC module bay is keyed on the stable adapter id (e.g. NIC.Slot.1), not the mutable + human label - so a relabeled adapter in the same physical slot reuses the bay instead of + churning a new one (which strict bay matching would otherwise mark the old one Absent for).""" + source, inventory, _ = modules_source(True, "4.3.0") + source.interface_adapter_type_dict = {} + source.nic_module_bay_by_adapter_id = {} + + def adapter(label): + return {"inventory": {"network_adapter": [ + {"id": "NIC.Slot.1", "name": label, "model": "BCM57414", "manufacturer": "Broadcom", + "operation_status": "Enabled", "num_ports": "2", "serial": "NIC-AAA"}]}} + + source.inventory_file_content = adapter("Broadcom Adapter") + source.update_network_adapter() + source.inventory_file_content = adapter("Broadcom Adapter rev2") + source.update_network_adapter() + + bays = inventory.get_all_items(NBModuleBay) + assert len(bays) == 1 + assert bays[0].data["name"] == "NIC.Slot.1" + assert len(inventory.get_all_items(NBModule)) == 1 + # and the in-memory adapter->bay map used for interface linking is the stable id too + assert source.nic_module_bay_by_adapter_id["NIC.Slot.1"] == "NIC.Slot.1" + + +def test_dimm_module_bay_stable_when_dimm_type_changes(modules_source): + """A DIMM's module bay is the stable slot (e.g. "DIMM A1"); the memory type appended to the + display name must not be part of the bay identity, so swapping the DIMM reuses the bay and + only re-points its module type instead of churning a new bay. Drives the real update_memory().""" + source, inventory, _ = modules_source(True, "4.3.0") + + def dimm(dimm_type, part): + return {"inventory": {"memory": [ + {"name": "DIMM A1", "type": dimm_type, "manufacturer": "Samsung", "part_number": part, + "serial": "DIMM-AAA", "size_in_mb": 32768, "speed": 3200, + "health_status": "OK", "operation_status": "GoodInUse"}]}} + + source.inventory_file_content = dimm("DDR4", "PN-DDR4") + source.update_memory() + source.inventory_file_content = dimm("DDR5", "PN-DDR5") + source.update_memory() + + bays = inventory.get_all_items(NBModuleBay) + assert len(bays) == 1 + assert bays[0].data["name"] == "DIMM A1" + assert len(inventory.get_all_items(NBModule)) == 1 + # the swap re-points the module type to the new part instead of creating a second bay + assert grab(inventory.get_all_items(NBModule)[0], "data.module_type.data.model") == "PN-DDR5" + + +def test_physical_drive_module_bay_stable_when_model_changes(modules_source): + """A physical drive's module bay is the stable slot; the type/model appended to the display + name must not churn the bay, so replacing the drive in a slot reuses the bay (a real swap also + brings a new serial). Drives the real update_physical_drive().""" + source, inventory, _ = modules_source(True, "4.3.0") + + def drive(model, serial): + return {"inventory": {"physical_drive": [ + {"name": "Solid State Disk", "id": "Disk.Bay.0", "location": "Slot 5", "type": "SSD", + "model": model, "manufacturer": "Samsung", "serial": serial, "part_number": "PN-DRV", + "size_in_byte": 512000000000, "health_status": "OK", "operation_status": "GoodInUse"}]}} + + source.inventory_file_content = drive("MZ-A", "DRV-AAA") + source.update_physical_drive() + source.inventory_file_content = drive("MZ-B", "DRV-BBB") + source.update_physical_drive() + + bays = inventory.get_all_items(NBModuleBay) + assert len(bays) == 1 + assert bays[0].data["name"] == "Solid State Disk Slot 5" + assert len(inventory.get_all_items(NBModule)) == 1 + assert grab(inventory.get_all_items(NBModule)[0], "data.serial") == "DRV-BBB" + + +def test_interfaces_not_attached_to_modules_when_feature_disabled(modules_source): + """With the modules feature off, interfaces must not get a module reference.""" + source, inventory, _ = modules_source(False, "4.3.0") + source.interface_adapter_type_dict = {} + source.nic_module_bay_by_adapter_id = {} + source.manager_name = None + source.settings.overwrite_interface_name = False + source.settings.overwrite_interface_attributes = False + source.settings.permitted_subnets = None + source.settings.ip_tenant_inheritance_order = [] + + source.inventory_file_content = { + "inventory": { + "network_adapter": [ + {"id": "NIC.Slot.1", "name": "NIC.Slot.1", "model": "BCM57414", + "manufacturer": "Broadcom", "operation_status": "Enabled", "num_ports": "2"} + ], + "network_port": [ + {"id": "NIC.Slot.1-1", "name": "Slot 1 Port 1", "adapter_id": "NIC.Slot.1", + "operation_status": "Enabled", "link_status": "Up", "addresses": [], + "capable_speed": 10000, "manager_ids": []}, + ], + } + } + + source.update_network_adapter() + source.update_network_interface() + + interfaces = inventory.get_all_items(NBInterface) + assert len(interfaces) == 1 + assert grab(interfaces[0], "data.module") is None + # with the feature off, the legacy "