diff --git a/core/change_detector.py b/core/change_detector.py index 24fbd214..57ca4d3f 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -6,13 +6,13 @@ import os from dataclasses import dataclass, field -from functools import lru_cache -from typing import Any, List, Optional from enum import Enum +from functools import lru_cache +from typing import Any -from core.component_registry import COMPONENT_TYPES -from core.normalization import normalize_values +from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES from core.formatting import log_property_diffs +from core.normalization import is_explicit_list, normalize_values from core.schema_reader import load_properties_for_type @@ -26,6 +26,114 @@ class ChangeType(Enum): COMPONENT_REMOVED = "component_removed" +def _manufacturer_slug_of(yaml_data): + """Return the owning manufacturer's slug from a parsed definition. + + ``core.repo`` rewrites every YAML manufacturer to ``{"slug": ...}`` before the importer + sees it, so that is the usual shape here. + """ + manufacturer = (yaml_data or {}).get("manufacturer") + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or "" + return manufacturer or "" + + +def _relation_properties(comp_type): + """Return the relation field names the registry declares for *comp_type*.""" + component = BY_YAML_KEY.get(comp_type) + return component.relations if component else () + + +def _is_relation_list(value): + """Return True when *value* is a list of non-empty strings, the only shape a reference takes. + + Blank is checked after stripping, matching the catalog: accepting " " here only defers + the rejection to the write path, where it skips the whole component's update. + """ + return is_explicit_list(value) and all(isinstance(item, str) and item.strip() for item in value) + + +def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): + """Return a PropertyChange when a relation differs, or None when there is nothing to do. + + A relation is an unordered set of related objects. Where the catalog is available the + comparison is between *identities*, ``(manufacturer slug, slug)``, because two + manufacturers may define the same class name and a bay holding the wrong one reads as + equal on names alone. Without a catalog it falls back to comparing names, which is + what a caller that has not wired one can still do. + + An omitted key leaves the relation unmanaged and an empty list clears it, but a bare + ``module_bay_types:`` parses as None and a malformed entry is not a name: those are left + unmanaged, because reading them as empty would clear a restriction nobody removed. A + field the query did not return is skipped for the same reason. + """ + if prop not in yaml_comp: + return None + declared = yaml_comp.get(prop) + if not _is_relation_list(declared): + if handle is not None: + handle.log( + f"Ignored {prop} on {yaml_comp.get('name', 'Unknown')!r}: expected a list of names, got {declared!r}" + ) + return None + netbox_value = getattr(netbox_comp, prop, _MISSING) + if netbox_value is _MISSING: + return None + + if catalog is not None and manufacturer: + from core.module_bay_types import ModuleBayTypeError + + try: + wanted = catalog.identities_for(manufacturer, declared) + except ModuleBayTypeError: + # Reported as a change so the write path sees it; "no change" is silent. + return PropertyChange( + property_name=prop, + old_value=sorted(_relation_names(netbox_value)), + new_value=sorted(set(declared)), + ) + current = _relation_identities(netbox_value) + if current is not None and wanted == current: + return None + if current is not None: + return PropertyChange( + property_name=prop, old_value=sorted(_relation_names(netbox_value)), new_value=sorted(set(declared)) + ) + + yaml_names = frozenset(declared) + netbox_names = frozenset(_relation_names(netbox_value)) + if yaml_names == netbox_names: + return None + return PropertyChange(property_name=prop, old_value=sorted(netbox_names), new_value=sorted(yaml_names)) + + +def _relation_identities(value): + """Return {(manufacturer slug, slug)} for a relation, or None if the shape lacks them. + + A read path that returns only ``id`` and ``name`` cannot answer the scope question, so + the caller falls back to comparing names rather than inventing an identity. + """ + identities = set() + for item in value or []: + slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) + manufacturer = item.get("manufacturer") if isinstance(item, dict) else getattr(item, "manufacturer", None) + owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) + if not slug or not owner: + return None + identities.add((owner, slug)) + return frozenset(identities) + + +def _relation_names(value): + """Return the ``name`` of each related object NetBox returned for a relation.""" + names = [] + for item in value or []: + name = item.get("name") if isinstance(item, dict) else getattr(item, "name", None) + if name is not None: + names.append(name) + return names + + @dataclass class PropertyChange: """Represents a single property change.""" @@ -42,7 +150,7 @@ class ComponentChange: component_type: str # e.g., "interfaces", "power-ports" component_name: str change_type: ChangeType - property_changes: List[PropertyChange] = field(default_factory=list) + property_changes: list[PropertyChange] = field(default_factory=list) @dataclass @@ -53,9 +161,9 @@ class DeviceTypeChange: model: str slug: str is_new: bool = False - property_changes: List[PropertyChange] = field(default_factory=list) - component_changes: List[ComponentChange] = field(default_factory=list) - netbox_id: Optional[int] = None + property_changes: list[PropertyChange] = field(default_factory=list) + component_changes: list[ComponentChange] = field(default_factory=list) + netbox_id: int | None = None @property def has_changes(self) -> bool: @@ -72,8 +180,8 @@ def has_updates(self) -> bool: class ChangeReport: """Aggregated change report for all device types.""" - new_device_types: List[DeviceTypeChange] = field(default_factory=list) - modified_device_types: List[DeviceTypeChange] = field(default_factory=list) + new_device_types: list[DeviceTypeChange] = field(default_factory=list) + modified_device_types: list[DeviceTypeChange] = field(default_factory=list) unchanged_count: int = 0 @@ -149,7 +257,7 @@ def __init__(self, device_types_instance, handle, remove_unmanaged_types: bool = self.verbose = verbose self.remove_unmanaged_types = remove_unmanaged_types - def detect_changes(self, device_types: List[dict], progress=None) -> ChangeReport: + def detect_changes(self, device_types: list[dict], progress=None) -> ChangeReport: """Analyze all device types and generate a change report. Args: @@ -200,7 +308,7 @@ def detect_changes(self, device_types: List[dict], progress=None) -> ChangeRepor return report - def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> List[PropertyChange]: + def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> list[PropertyChange]: """Compare YAML device type properties against NetBox device type. Args: @@ -241,7 +349,7 @@ def _compare_device_type_properties(self, yaml_data: dict, netbox_dt) -> List[Pr return changes @staticmethod - def _compare_image_properties(yaml_data: dict, netbox_dt) -> List[PropertyChange]: + def _compare_image_properties(yaml_data: dict, netbox_dt) -> list[PropertyChange]: """Compare image properties between YAML and NetBox device type. YAML uses boolean flags (front_image: true) meaning "an image should exist", @@ -280,7 +388,7 @@ def _compare_components( yaml_data: dict, device_type_id: int, parent_type: str = "device", - ) -> List[ComponentChange]: + ) -> list[ComponentChange]: """Compare all components between YAML and cached NetBox data. Args: @@ -310,9 +418,9 @@ def _compare_components( # same as an empty list so chassis YAMLs that omit (e.g.) interfaces can # still drive cleanup of stale templates in NetBox. if yaml_key in yaml_data or self.remove_unmanaged_types: - for existing_name in existing_components.keys(): + for existing_name in existing_components: if existing_name not in yaml_component_names: - changes.append( + changes.append( # noqa: PERF401 ComponentChange( component_type=yaml_key, component_name=existing_name, @@ -339,7 +447,11 @@ def _compare_components( # Check for property changes on existing component existing = existing_components[comp_name] prop_changes = self._compare_component_properties( - yaml_comp, existing, component.compare_properties, comp_type=yaml_key + yaml_comp, + existing, + component.compare_properties, + comp_type=yaml_key, + manufacturer=_manufacturer_slug_of(yaml_data), ) if prop_changes: changes.append( @@ -353,14 +465,31 @@ def _compare_components( return changes + def _relation_catalog(self): + """Return the module bay type catalog, or None when this detector has no real one. + + Tests build the detector around a stand-in for DeviceTypes, and a stand-in cannot + answer what a reference means. Returning None there keeps the name comparison, + which is what those tests are asserting on. + """ + from core.module_bay_types import ModuleBayTypeCatalog + + catalog = getattr(self.device_types, "module_bay_type_catalog", None) + return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None + def _compare_component_properties( self, yaml_comp: dict, netbox_comp, - properties: List[str], + properties: list[str], comp_type: str = "", - ) -> List[PropertyChange]: - """Compare properties between YAML and NetBox component.""" + manufacturer: str = "", + ) -> list[PropertyChange]: + """Compare properties between YAML and NetBox component. + + *manufacturer* is the owning manufacturer's slug, used to resolve a relation + reference to the object it means rather than the name it is written as. + """ changes = [] for prop in properties: @@ -389,7 +518,11 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - has_names = any(m.get("rear_port_name") is not None for m in canonical) + # Compare by name whenever one is available: an empty M2M list has no name to + # infer from but still needs the named path, and the pre-4.5 query returns one. + has_names = bool(getattr(netbox_comp, "_mappings_m2m", False)) or any( + m.get("rear_port_name") is not None for m in canonical + ) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( @@ -426,6 +559,14 @@ def _compare_component_properties( ) continue + if prop in _relation_properties(comp_type): + relation_change = _relation_change( + prop, yaml_comp, netbox_comp, self._relation_catalog(), manufacturer, self.handle + ) + if relation_change is not None: + changes.append(relation_change) + continue + # Only compare properties explicitly present in the YAML component; # an omitted property means the YAML doesn't manage it (absent key != removal). if prop not in yaml_comp: @@ -490,7 +631,7 @@ def _log_modified_summary(self, report: ChangeReport) -> None: if parts: self.handle.log(f" Breakdown: {', '.join(parts)}") - def _log_property_diffs(self, prop_changes: List[PropertyChange], indent: str) -> None: + def _log_property_diffs(self, prop_changes: list[PropertyChange], indent: str) -> None: """Emit diff-u style lines for *prop_changes* at the given *indent*.""" log_property_diffs( [(pc.property_name, pc.old_value, pc.new_value) for pc in prop_changes], diff --git a/core/compat.py b/core/compat.py index 62c33bf1..2b35eb0a 100644 --- a/core/compat.py +++ b/core/compat.py @@ -1,61 +1,27 @@ """NetBox version-compatibility helpers. -Centralises filter parameter names that changed between NetBox releases so -that every caller uses the same logic and drift is impossible. - -NetBox 4.1 renamed several filter keys on the DCIM endpoints: - - devicetype_id → device_type_id - moduletype_id → module_type_id - -Any code that constructs endpoint filter kwargs should call the helpers -here rather than inlining ``"device_type_id" if new_filters else "devicetype_id"``. +Centralises the version line each feature sits behind so that every caller uses the +same logic and drift is impossible. """ from __future__ import annotations +import re -def device_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for device-type component queries. +# Selecting or sending the relation below this release fails the whole query. +MODULE_BAY_TYPE_MINIMUM_VERSION = (4, 7) - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"device_type_id"``); - ``False`` for older releases (returns ``"devicetype_id"``). - """ - return "device_type_id" if new_filters else "devicetype_id" +def parse_netbox_version(version) -> tuple[int, int]: + """Return ``(major, minor)`` from a NetBox version string. -def module_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for module-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"module_type_id"``); - ``False`` for older releases (returns ``"moduletype_id"``). + Padded to two parts so a single-component string cannot raise, and tolerant of the + suffixes NetBox ships ("4.7.0-beta2"). """ - return "module_type_id" if new_filters else "moduletype_id" - - -def device_type_filter_kwargs(device_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a device type. + raw = [int(re.sub(r"\D.*", "", part.strip()) or "0") for part in str(version).split(".")] + return tuple(([*raw, 0, 0])[:2]) # type: ignore[return-value] - Args: - device_type_id: NetBox ID of the device type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {device_type_filter_key(new_filters): device_type_id} - - -def module_type_filter_kwargs(module_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a module type. - - Args: - module_type_id: NetBox ID of the module type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {module_type_filter_key(new_filters): module_type_id} +def supports_module_bay_types(version) -> bool: + """Return True when this NetBox release exposes the module bay type relation.""" + return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION diff --git a/core/component_cache.py b/core/component_cache.py index d8dc4a22..61ddbe83 100644 --- a/core/component_cache.py +++ b/core/component_cache.py @@ -14,12 +14,6 @@ import threading from typing import Any -from core.compat import ( - device_type_filter_key, - device_type_filter_kwargs, - module_type_filter_key, - module_type_filter_kwargs, -) from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError @@ -118,14 +112,13 @@ class ComponentCache: filter, which happens for types created during this run and after an invalidation. """ - def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_record=None): + def __init__(self, netbox, graphql, handle, max_threads, wrap_record=None): """Build a cache over the *netbox* REST client and the *graphql* client. Args: netbox: pynetbox API object, used for REST fallbacks and count checks. graphql: GraphQL client used for the bulk fetch. handle: Log handler. - new_filters (bool): Whether this NetBox takes the newer filter parameter names. max_threads (int): Upper bound on concurrent endpoint fetches. wrap_record (callable | None): Applied to every front-port record, so change detection sees one mappings shape across NetBox versions. @@ -133,7 +126,6 @@ def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_recor self.netbox = netbox self.graphql = graphql self.handle = handle - self.new_filters = new_filters self.max_threads = max_threads self._wrap_record = wrap_record or (lambda record: record) @@ -312,10 +304,7 @@ def get(self, endpoint_name, parent_type, parent_id, endpoint): if key in cached: return cached[key] - if parent_type == "device": - filter_kwargs = device_type_filter_kwargs(parent_id, new_filters=self.new_filters) - else: - filter_kwargs = module_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"device_type_id": parent_id} if parent_type == "device" else {"module_type_id": parent_id} result = {item.name: item for item in endpoint.filter(**filter_kwargs)} self.record(endpoint_name, parent_type, parent_id, result) return result @@ -386,7 +375,7 @@ def _finish_endpoint(self, endpoint_name, future): """Mark *endpoint_name* complete on the display, sizing the bar to the result.""" try: total = max(len(future.result()), 1) - except Exception: + except Exception: # a worker error must not break the progress display # noqa: BLE001 total = 1 self._job["display"].finish(endpoint_name, total) self._job["done"].add(endpoint_name) @@ -479,8 +468,6 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc Raises: GraphQLCountMismatchError: When an endpoint holds fewer records than REST reports. """ - dt_filter_key = device_type_filter_key(self.new_filters) - mt_filter_key = module_type_filter_key(self.new_filters) dt_ids = list(device_type_ids) mt_ids = list(module_type_ids) @@ -495,9 +482,9 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc rest_endpoint = getattr(self.netbox.dcim, endpoint_name) rest_count = 0 if dt_ids: - rest_count += self._rest_count(rest_endpoint, dt_filter_key, dt_ids) + rest_count += self._rest_count(rest_endpoint, "device_type_id", dt_ids) if mt_ids and component.module_types: - rest_count += self._rest_count(rest_endpoint, mt_filter_key, mt_ids) + rest_count += self._rest_count(rest_endpoint, "module_type_id", mt_ids) if cached_count != rest_count: if vendor_scope_valid: diff --git a/core/component_registry.py b/core/component_registry.py index ed991052..10609fb6 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -10,13 +10,23 @@ """ from dataclasses import dataclass, field -from typing import Optional # What a create call must resolve from a name to a NetBox id before it can POST. LINK_BRIDGE = "bridge" LINK_POWER_PORT = "power_port" LINK_REAR_PORTS = "rear_ports" +# Fields holding names that must become NetBox ids before a POST. +RELATION_MODULE_BAY_TYPES = "module_bay_types" + +# Relation fields a module type carries itself, rather than through one of its components. +MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) + + +def relation_selection(name): + """Return the GraphQL fields that identify a related object.""" + return f"{name} {{ id name slug manufacturer {{ slug }} }}" + @dataclass(frozen=True) class ComponentType: @@ -27,19 +37,29 @@ class ComponentType: label: str fields: tuple[str, ...] module_types: bool = True + relations: tuple[str, ...] = field(default_factory=tuple) graphql_extra: tuple[str, ...] = field(default_factory=tuple) compare_extra: tuple[str, ...] = field(default_factory=tuple) - link: Optional[str] = None + link: str | None = None @property def graphql_fields(self): - """Fields to select in a GraphQL query, including the id every consumer needs.""" - return ["id", *self.fields, *self.graphql_extra] + """Fields to select in a GraphQL query, including the id every consumer needs. + + A relation is a list of related objects, so it is selected by name rather than + read as a scalar. + """ + return ["id", *self.fields, *self.graphql_extra, *self.graphql_relation_fields] + + @property + def graphql_relation_fields(self): + """GraphQL selections for this row's relations, one nested block per relation.""" + return [relation_selection(name) for name in self.relations] @property def compare_properties(self): """Properties change detection compares between YAML and NetBox.""" - return [*self.fields, *self.compare_extra] + return [*self.fields, *self.compare_extra, *self.relations] @property def list_key(self): @@ -102,7 +122,8 @@ def create_label(self, parent_type): yaml_key="front-ports", endpoint="front_port_templates", label="Front Port", - fields=("name", "type", "label", "description", "color"), + # positions arrived with the 4.5 mapping model; the query drops it on older servers. + fields=("name", "type", "label", "description", "color", "positions"), graphql_extra=("mappings { id front_port_position rear_port_position rear_port { id name } }",), compare_extra=("_mappings",), link=LINK_REAR_PORTS, @@ -121,6 +142,7 @@ def create_label(self, parent_type): endpoint="module_bay_templates", label="Module Bay", fields=("name", "position", "label", "description"), + relations=(RELATION_MODULE_BAY_TYPES,), ), ) diff --git a/core/config.py b/core/config.py index d37778e6..fd4dd6c8 100644 --- a/core/config.py +++ b/core/config.py @@ -1,9 +1,11 @@ """Resolution of one run's configuration from the command line and the environment.""" +import ipaddress import os import re from argparse import ArgumentParser from dataclasses import dataclass, field +from urllib.parse import urlparse from dotenv import load_dotenv @@ -22,6 +24,27 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def _sends_token_in_cleartext(url): + """Return True when *url* would send the API token over plain HTTP off this host.""" + # requests treats a backslash in the authority as a delimiter and urlparse does not, so + # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. + try: + parsed = urlparse(str(url or "").strip().replace("\\", "/")) + except ValueError: + # urlparse rejects some authorities outright. An authority this tool cannot read is + # also one it cannot clear as loopback, and a notice must never abort the run. + return True + if parsed.scheme != "http": + return False + host = (parsed.hostname or "").casefold() + if host in {"", "localhost"}: + return False + try: + return not ipaddress.ip_address(host).is_loopback + except ValueError: + return True + + def is_local_repo_url(url): """Return True when *url* is the sentinel that turns off every git operation.""" return str(url or "").strip().casefold() == LOCAL_REPO_URL @@ -271,6 +294,12 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + netbox_url = _text(env, "NETBOX_URL") + if _sends_token_in_cleartext(netbox_url): + notices.append( + "NETBOX_URL uses http:// on a remote host, so the API token is sent in cleartext. " + "Use https:// unless NetBox is on this machine." + ) if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: notices.append( f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " @@ -278,7 +307,7 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: ) return RunConfig( - netbox_url=_text(env, "NETBOX_URL"), + netbox_url=netbox_url, netbox_token=_text(env, "NETBOX_TOKEN"), ignore_ssl_errors=(_text(env, "IGNORE_SSL_ERRORS", "False") or "False").casefold() in {"true", "1", "yes"}, graphql_page_size=_positive_int(env, "GRAPHQL_PAGE_SIZE", DEFAULT_GRAPHQL_PAGE_SIZE), diff --git a/core/export.py b/core/export.py index c0dd57ac..d08b2d4c 100644 --- a/core/export.py +++ b/core/export.py @@ -3,17 +3,20 @@ Entry point: ``Exporter(config, handle, export_dir, force_overwrite, vendor_slugs).run()`` """ +import contextlib import hashlib import os import re import threading +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import Any, List, Optional, Sequence +from typing import Any import requests import yaml +from core.component_registry import COMPONENT_TYPES, MODULE_TYPE_RELATIONS from core.export_manifest import ( is_entry_fresh, load_manifest, @@ -23,6 +26,7 @@ from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.nb_serializer import ( COMPONENT_ENDPOINT_NAMES, + module_bays_missing_position, serialize_device_type, serialize_module_type, serialize_rack_type, @@ -106,7 +110,7 @@ class ExportItem: kind: str # "device-type" | "module-type" | "rack-type" nb_record: Any - repo_yaml: Optional[dict] # None when absent from repo + repo_yaml: dict | None # None when absent from repo serialized: dict # What we will write reason: str # "absent" | "differs" | "images-missing" mfr_name: str @@ -140,6 +144,40 @@ def _yaml_equal(a: dict, b: dict) -> bool: return _normalize_for_compare(a) == _normalize_for_compare(b) +def _relation_names() -> set: + """Every relation key the registry knows, so this does not become a second source.""" + return {relation for component in COMPONENT_TYPES for relation in component.relations} | set(MODULE_TYPE_RELATIONS) + + +def _carry_unqueried_relations(repo_yaml: dict, serialized: dict) -> dict: + """Return *serialized* with relations the server never answered for taken from the repo. + + Below NetBox 4.7 module_bay_types is not queried at all, so its absence means "not asked", + not "cleared". Writing NetBox's answer as it stands would delete the restriction from + every definition the export touches for any other reason. + """ + relations = _relation_names() + result = dict(serialized) + for relation in relations: + if relation in repo_yaml and relation not in result: + result[relation] = repo_yaml[relation] + for key, repo_value in repo_yaml.items(): + if not isinstance(repo_value, list) or not isinstance(result.get(key), list): + continue + by_name = {e.get("name"): e for e in repo_value if isinstance(e, dict)} + merged = [] + for entry in result[key]: + if isinstance(entry, dict): + repo_entry = by_name.get(entry.get("name")) + if isinstance(repo_entry, dict): + carried = {r: repo_entry[r] for r in relations if r in repo_entry and r not in entry} + if carried: + entry = {**entry, **carried} + merged.append(entry) + result[key] = merged + return result + + def _repo_supersedes(repo_yaml: dict, nb_serialized: dict) -> bool: """Return True when *repo_yaml* already contains every field NetBox would write. @@ -161,8 +199,24 @@ def _norm_mfr(d: dict) -> dict: return d return {**d, "manufacturer": _canon_mfr_slug(d["manufacturer"])} - nrepo = _normalize_for_compare(_norm_mfr(repo_yaml)) - nnb = _normalize_for_compare(_norm_mfr(nb_serialized)) + # The serializer always writes the schema-required positions; an entry that omits it + # means the default, so filling it here keeps those definitions from reading as differing. + def _default_positions(d: dict) -> dict: + ports = d.get("front-ports") + if not isinstance(ports, list): + return d + filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] + return {**d, "front-ports": filled} + + def _sort_port_mappings(d: dict) -> dict: + mappings = d.get("port-mappings") + if not isinstance(mappings, list) or not all(isinstance(mapping, dict) for mapping in mappings): + return d + fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") + return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} + + nrepo = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml)))) + nnb = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized)))) return _is_subset(nnb, nrepo) @@ -196,7 +250,7 @@ def _is_subset(sub: Any, sup: Any) -> bool: class Exporter: """Exports NetBox device/module/rack types to a local directory in DTL format.""" - def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendor_slugs: Optional[Sequence[str]]): + def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendor_slugs: Sequence[str] | None): """Initialize the Exporter from the resolved run configuration.""" self.config = config self.handle = handle @@ -218,7 +272,7 @@ def __init__(self, config, handle, export_dir: str, force_overwrite: bool, vendo handle=handle, page_size=config.graphql_page_size, ) - self._module_image_details: Optional[dict] = None + self._module_image_details: dict | None = None def _get_module_image_details(self) -> dict: """Return module type image details, fetching from NetBox at most once per run.""" @@ -241,8 +295,11 @@ def run(self, progress=None) -> None: ) self.handle.log(f"Export-diff: fetching NetBox device/module/rack types{scope}") + # Decided before the first query: the selection depends on the answer. + self.graphql.detect_module_bay_type_support() + # ── Fetch all types from NetBox ────────────────────────────────────── - by_model, by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) + by_model, _by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) all_mt = self.graphql.get_module_types(manufacturer_slugs=self.vendor_slugs) all_rt = self.graphql.get_rack_types(manufacturer_slugs=self.vendor_slugs) @@ -316,9 +373,9 @@ def _compare_vendors_to_items( repo_dt_by_slug, repo_mt_by_key, progress, - ) -> tuple[List[ExportItem], int]: + ) -> tuple[list[ExportItem], int]: """Compare stale device/module types per vendor and return export items.""" - items: List[ExportItem] = [] + items: list[ExportItem] = [] skipped_fresh = 0 compare_task = ( progress.add_task("Comparing vendors", total=len(all_vendor_slugs)) @@ -386,9 +443,9 @@ def _compare_vendors_to_items( return items, skipped_fresh - def _compare_racks_to_items(self, all_rt, manifest, repo_rt_by_key, progress) -> tuple[List[ExportItem], int]: + def _compare_racks_to_items(self, all_rt, manifest, repo_rt_by_key, progress) -> tuple[list[ExportItem], int]: """Compare stale rack types and return export items.""" - items: List[ExportItem] = [] + items: list[ExportItem] = [] skipped_fresh = 0 rack_records = [record for models in all_rt.values() for record in models.values()] rack_task = ( @@ -443,15 +500,15 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: # top-level fields (e.g. comments, profile) that NetBox does not return # in its serialized output. Component lists are left as NB authoritative. to_write = item.serialized + if item.repo_yaml and not self.graphql.supports_module_bay_types: + to_write = _carry_unqueried_relations(item.repo_yaml, to_write) if item.reason == "differs" and item.repo_yaml: # Only preserve scalar/metadata repo fields not present in the NB output. # Exclude list-valued keys (component sections such as interfaces, power-ports, # console-ports, etc.) so that NB remains authoritative for all components. - extra = { - k: v for k, v in item.repo_yaml.items() if k not in item.serialized and not isinstance(v, list) - } + extra = {k: v for k, v in item.repo_yaml.items() if k not in to_write and not isinstance(v, list)} if extra: - to_write = {**item.serialized, **extra} + to_write = {**to_write, **extra} written = self._write_yaml(dest, to_write) if not written: skipped_overwrite += 1 @@ -463,6 +520,12 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: continue written_count += 1 + bays = module_bays_missing_position(to_write) + if bays: + self.handle.log( + f"[yellow]{item.mfr_name}/{item.filename}: module bay(s) {', '.join(bays)} have " + f"no position, which the library schema requires[/yellow]" + ) images_ok = self._download_type_images(item) if images_ok: update_entry(manifest, f"{item.kind}s", item.manifest_key, item.nb_record.last_updated) @@ -624,13 +687,7 @@ def _fetch_vendor_components(self, mfr_slug: str) -> tuple: def _fetch_one(endpoint_name): if not getattr(_thread_local, "graphql", None): - client = NetBoxGraphQLClient( - self.graphql.url, - self.graphql.token, - self.graphql.ignore_ssl, - self.graphql.handle, - self.graphql.DEFAULT_PAGE_SIZE, - ) + client = self.graphql.clone() _thread_local.graphql = client with _clients_lock: _clients.append(client) @@ -643,10 +700,8 @@ def _fetch_one(endpoint_name): results = list(pool.map(_fetch_one, COMPONENT_ENDPOINT_NAMES)) finally: for client in _clients: - try: + with contextlib.suppress(Exception): client.close() - except Exception: - pass for endpoint_name, records in results: for rec in records: @@ -662,7 +717,7 @@ def _fetch_one(endpoint_name): def _determine_export_set_for_device_types( self, nb_records: list, repo_dt_by_slug: dict, components_by_dt_id: dict - ) -> List[ExportItem]: + ) -> list[ExportItem]: """Build the list of device types that need exporting to the repo. Includes records absent from the repo, those whose serialized form @@ -678,7 +733,7 @@ def _determine_export_set_for_device_types( manifest_key = f"{mfr_name}/{rec.slug}" repo_yaml = repo_dt_by_slug.get((mfr_slug, rec.slug)) - reason: Optional[str] + reason: str | None if repo_yaml is None: reason = "absent" elif _repo_supersedes(repo_yaml, serialized): @@ -704,7 +759,7 @@ def _determine_export_set_for_device_types( def _determine_export_set_for_module_types( self, nb_records: list, repo_mt_by_key: dict, components_by_mt_id: dict - ) -> List[ExportItem]: + ) -> list[ExportItem]: """Build the list of module types that need exporting to the repo. Includes records absent from the repo and those whose serialized form @@ -741,7 +796,7 @@ def _determine_export_set_for_module_types( ) return items - def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: dict) -> List[ExportItem]: + def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: dict) -> list[ExportItem]: """Build the list of rack types that need exporting to the repo. Includes records absent from the repo and those whose serialized form @@ -778,7 +833,7 @@ def _determine_export_set_for_rack_types(self, nb_records: list, repo_rt_by_key: ) return items - def _check_missing_images(self, front_url, rear_url, mfr_name: str, slug: str) -> Optional[str]: + def _check_missing_images(self, front_url, rear_url, mfr_name: str, slug: str) -> str | None: """Return ``'images-missing'`` if any expected local image is absent; else None. DTL stores images under ``elevation-images//.{front,rear}.{png,jpg,jpeg,gif}`` @@ -819,7 +874,7 @@ def _download_type_images(self, item: ExportItem) -> bool: """Download images for *item*. Returns True if all downloads succeeded.""" if item.kind == "device-type": return self._download_device_type_images(item) - elif item.kind == "module-type": + if item.kind == "module-type": return self._download_module_type_images(item) return True # rack types have no images @@ -920,7 +975,7 @@ def _download_module_type_images(self, item: ExportItem) -> bool: return ok def _download_image( - self, url_path: str, dest: Path, content_type_out: "Optional[list]" = None + self, url_path: str, dest: Path, content_type_out: "list | None" = None ) -> "str | _SkipSentinel | None": """Download an image from NetBox and write to *dest*. diff --git a/core/export_manifest.py b/core/export_manifest.py index b959ddeb..d9ae91c0 100644 --- a/core/export_manifest.py +++ b/core/export_manifest.py @@ -11,6 +11,11 @@ _EMPTY: dict = {"device-types": {}, "module-types": {}, "rack-types": {}} +# Bump whenever the serialized output shape changes (a new stanza, a field that starts or +# stops being written). An unchanged NetBox record has the same last_updated forever, so +# without this an old manifest skips the very types the new shape was added for. +EXPORT_SCHEMA_REVISION = 2 + def load_manifest(path: Path) -> dict: """Load manifest from *path*. Returns an empty manifest on any error.""" @@ -33,13 +38,15 @@ def save_manifest(path: Path, data: dict) -> None: def is_entry_fresh(manifest: dict, kind: str, key: str, last_updated: str) -> bool: - """Return True if the manifest entry for *key* matches *last_updated*.""" + """Return True if *key* was written by this exporter and the record has not changed.""" section = manifest.get(kind) if not isinstance(section, dict): return False entry = section.get(key) if not isinstance(entry, dict): return False + if entry.get("schema") != EXPORT_SCHEMA_REVISION: + return False return entry.get("last_updated") == last_updated @@ -48,4 +55,4 @@ def update_entry(manifest: dict, kind: str, key: str, last_updated: str) -> None section = manifest.get(kind) if not isinstance(section, dict): manifest[kind] = {} - manifest[kind][key] = {"last_updated": last_updated} + manifest[kind][key] = {"last_updated": last_updated, "schema": EXPORT_SCHEMA_REVISION} diff --git a/core/graphql_client.py b/core/graphql_client.py index 70baabfd..d66c376a 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -5,13 +5,15 @@ compatible with the existing REST-based code in ``netbox_api.py``. """ +import copy import threading import time from collections.abc import Sequence import requests -from core.component_registry import BY_ENDPOINT +from core.compat import supports_module_bay_types +from core.component_registry import BY_ENDPOINT, MODULE_TYPE_RELATIONS, relation_selection # Module-level dedup: tracks (url, requested_page_size) pairs that have already # emitted the page-size clamping warning so the message appears at most once @@ -54,7 +56,8 @@ def __getattr__(self, key): try: value = self[key] except KeyError: - raise AttributeError(f"'DotDict' has no attribute '{key}'") + # The dict miss is an implementation detail of attribute lookup, not the error. + raise AttributeError(f"'DotDict' has no attribute '{key}'") from None if isinstance(value, dict) and not isinstance(value, DotDict): value = DotDict(value) self[key] = value @@ -102,6 +105,9 @@ def _to_dotdict(obj): # connections, which is transient during a long paginated run. _RETRYABLE_STATUSES = {429, 500, 502, 503, 504} +# One small metadata read; a run that cannot get it fails at the next query anyway. +_STATUS_TIMEOUT_SECONDS = 30 + def _response_body_detail(response): """Return the response body as a suffix for an error message, truncated and stripped.""" @@ -109,7 +115,7 @@ def _response_body_detail(response): return "" try: body = response.text.strip() - except Exception: # pragma: no cover - a body that cannot be decoded is not worth failing on + except Exception: # pragma: no cover - a body that cannot be decoded is not worth failing on # noqa: BLE001 return "" if not body: return "" @@ -136,7 +142,7 @@ class NetBoxGraphQLClient: or raise the server's ``MAX_PAGE_SIZE`` setting to match. """ - def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): + def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000, supports_module_bay_types=False): """Store connection parameters for later use in :meth:`query`. Args: @@ -148,6 +154,8 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): ``print`` when not provided. page_size: Default number of items per GraphQL page (default: 5 000). + supports_module_bay_types: True when NetBox is >= 4.7 and its schema + exposes the module bay type relation. """ self.DEFAULT_PAGE_SIZE = page_size self.url = url.rstrip("/") @@ -155,22 +163,28 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): self.token = token self.ignore_ssl = ignore_ssl self._handle = handle + self.supports_module_bay_types = supports_module_bay_types + + self._session = self._new_session() - self._session = requests.Session() + def _new_session(self): + """Return an HTTP session carrying this client's auth and TLS settings.""" + session = requests.Session() # v2 tokens start with "nbt_" prefix (format: nbt_.); # v1 tokens are plain 40-char hex strings using legacy Token auth. auth_scheme = "Bearer" if self.token.startswith("nbt_") else "Token" - self._session.headers.update( + session.headers.update( { "Authorization": f"{auth_scheme} {self.token}", "Content-Type": "application/json", } ) - self._session.verify = not self.ignore_ssl + session.verify = not self.ignore_ssl if self.ignore_ssl: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + return session @property def handle(self): @@ -178,8 +192,40 @@ def handle(self): return self._handle def clone(self): - """Return an independent client with the same connection settings.""" - return type(self)(self.url, self.token, self.ignore_ssl, self.handle, self.DEFAULT_PAGE_SIZE) + """Return an independent client with the same settings and its own HTTP session. + + Copied rather than reconstructed, so a setting added to ``__init__`` travels with + the clone instead of quietly reverting to its default. The session is the one + thing a worker must not share, so it is the one thing rebuilt here. + """ + clone = copy.copy(self) + clone._session = self._new_session() + return clone + + def detect_module_bay_type_support(self): + """Ask NetBox for its version and record whether the relation can be selected. + + The importer learns this from its pynetbox client; the export entry point has no + such client, so it asks here. Both sides decide from core.compat, so the line + cannot drift between them. + """ + status_url = f"{self.url}/api/status/" + try: + response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) + response.raise_for_status() + payload = response.json() + except requests.RequestException as exc: + raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc + except ValueError as exc: + # A proxy error page answers 200 with HTML, so the body is not JSON. + raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + # Reading an absent version as "" would answer "unsupported" for a 4.7 server and + # export without the relation, so demand the field rather than defaulting it. + version = payload.get("netbox-version") if isinstance(payload, dict) else None + if not isinstance(version, str) or not version.strip(): + raise GraphQLError(f"No netbox-version in the status payload from {status_url}: {payload!r}") + self.supports_module_bay_types = supports_module_bay_types(version) + return self.supports_module_bay_types def close(self): """Close the underlying HTTP session.""" @@ -254,6 +300,7 @@ def query(self, graphql_query, variables=None, _retries=3): raise GraphQLSchemaError(messages) return body.get("data", {}) + return None def query_all(self, graphql_query, list_key, page_size=None, variables=None, on_page=None): """Auto-paginate a GraphQL list query using offset/limit. @@ -477,6 +524,11 @@ def get_module_types(self, manufacturer_slugs=None): sequence of non-blank strings. """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) + module_bay_type_selection = ( + "".join(f"{relation_selection(name)}\n " for name in MODULE_TYPE_RELATIONS) + if self.supports_module_bay_types + else "" + ) query = f""" query($pagination: OffsetPaginationInput{var_decl}) {{ @@ -490,7 +542,7 @@ def get_module_types(self, manufacturer_slugs=None): weight weight_unit last_updated - manufacturer {{ + {module_bay_type_selection}manufacturer {{ id name slug @@ -698,7 +750,7 @@ def get_module_type_image_details(self): def _front_port_field_variants(fields): """Yield successive field-list tiers for the front_port_templates fallback. - Tier 1: mappings block (NetBox 4.5+) + Tier 1: mappings block and positions (NetBox 4.5+) Tier 2: rear_port_position scalar (<4.5) Tier 3: neither (field removed entirely) """ @@ -707,7 +759,8 @@ def _front_port_field_variants(fields): for f in fields: if "mappings" in f: fallback.extend(["rear_port_position", "rear_port { id name }"]) - else: + elif f != "positions": + # positions arrived with the mapping model, so no pre-4.5 tier may ask for it. fallback.append(f) yield fallback stripped = [f for f in fallback if f != "rear_port_position" and "rear_port" not in f] @@ -801,6 +854,9 @@ def get_component_templates(self, endpoint_name, manufacturer_slug=None, on_page raise ValueError("manufacturer_slug must be None or a non-empty string") fields = component.graphql_fields + if not self.supports_module_bay_types: + # Selecting a field the server's schema lacks fails the whole query. + fields = [f for f in fields if f not in component.graphql_relation_fields] list_key = component.list_key parent_fields = "device_type { id }" diff --git a/core/import_run.py b/core/import_run.py index f81fd7bf..c99a6ef0 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -1,22 +1,26 @@ """Import pipeline planning and execution.""" +import os from collections import Counter from contextlib import contextmanager from dataclasses import dataclass -from datetime import datetime, timedelta -import os +from datetime import UTC, datetime, timedelta from typing import Any -from core.change_detector import ChangeDetector, ChangeType, IMAGE_PROPERTIES +from core.change_detector import IMAGE_PROPERTIES, ChangeDetector, ChangeType from core.component_cache import NullTaskDisplay, RichTaskDisplay from core.config import RunConfig from core.errors import VendorSelectionError from core.outcomes import EntityKind, Outcome - _PROGRESS_DESC_WIDTH = 28 +def _as_aware(moment): + """Return *moment* with a timezone attached; a naive value means local time.""" + return moment if moment.tzinfo is not None else moment.astimezone() + + @dataclass(frozen=True) class RunSelection: """Selected vendors and source paths for one import run.""" @@ -49,8 +53,6 @@ class RunSummary: """Snapshot of the result of one completed import run.""" counter: Counter - modules: bool - rack_types: bool outcome_counts: dict failure_lines: tuple duplicate_definitions: tuple @@ -66,12 +68,10 @@ def capture(cls, netbox, repo, started_at): """ return cls( counter=Counter(netbox.counter), - modules=netbox.modules, - rack_types=netbox.rack_types, outcome_counts=netbox.outcomes.summary_by_kind(), failure_lines=tuple(netbox.outcomes.render_failure_report()), duplicate_definitions=tuple(repo.duplicate_definitions), - elapsed=datetime.now() - started_at, + elapsed=datetime.now(UTC) - started_at, ) def outcome_count(self, kind, outcome): @@ -455,10 +455,6 @@ def _process_rack_types(config, netbox, handle, progress, rack_types, vendor_nam if not rack_types: return - if not netbox.rack_types: - handle.log("Rack types require NetBox >= 4.1. Skipping rack type import.") - return - handle.verbose_log(f"{len(rack_types)} Rack-Types Found") all_rack_types = netbox.get_existing_rack_types() @@ -521,21 +517,19 @@ def _log_run_summary(handle, summary): handle.log(f"{counter['components_removed']} components removed") handle.verbose_log(f"{counter['images']} images uploaded") handle.log(f"{counter['manufacturer']} manufacturers created") - if summary.modules: - handle.log(f"{counter['module_added']} modules created") - handle.log(f"{counter['module_updated']} modules updated") - module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) - if module_failed: - handle.log(f"{module_failed} modules failed to create or update") - module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) - if module_partial: - handle.log(f"{module_partial} modules partially updated") - if summary.rack_types: - handle.log(f"{counter['rack_type_added']} rack types created") - handle.log(f"{counter['rack_type_updated']} rack types updated") - rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) - if rack_failed: - handle.log(f"{rack_failed} rack types failed") + handle.log(f"{counter['module_added']} modules created") + handle.log(f"{counter['module_updated']} modules updated") + module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) + if module_failed: + handle.log(f"{module_failed} modules failed to create or update") + module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) + if module_partial: + handle.log(f"{module_partial} modules partially updated") + handle.log(f"{counter['rack_type_added']} rack types created") + handle.log(f"{counter['rack_type_updated']} rack types updated") + rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) + if rack_failed: + handle.log(f"{rack_failed} rack types failed") for line in summary.failure_lines: handle.log(line) @@ -590,7 +584,7 @@ def __init__(self, config, repo, netbox, reporter, progress_factory, *, started_ netbox (NetBox): Connected NetBox interface. reporter (LogHandler): Run message sink. progress_factory: Context manager factory for the Rich progress display. - started_at (datetime | None): Start time used for elapsed-time reporting. + started_at (datetime | None): Start time for elapsed-time reporting; naive means local. """ if not isinstance(config, RunConfig): raise TypeError("config must be a RunConfig") @@ -599,7 +593,8 @@ def __init__(self, config, repo, netbox, reporter, progress_factory, *, started_ self.netbox = netbox self.reporter = reporter self.progress_factory = progress_factory - self.started_at = started_at or datetime.now() + # Normalize here: a naive value would otherwise survive the run and raise in capture(). + self.started_at = _as_aware(started_at) if started_at is not None else datetime.now(UTC) self.progress: Any = None self.task_registry = None self.vendor_task_id = None @@ -673,30 +668,24 @@ def plan_vendor(self, selection, vendor): self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] ) - if self.netbox.modules: - module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None - if module_hint is not None and vendor["slug"] not in module_hint: - module_types = [] - else: - module_types = _parse_vendor_files( - self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] - ) - else: + module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None + if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] - - if self.netbox.rack_types: - rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None - if rack_hint is not None and vendor["slug"] not in rack_hint: - rack_types = [] - else: - rack_types = _parse_vendor_files( - self.repo, - selection.racks_path, - vendor["name"], - self.config.slugs or [], - ) else: + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) + + rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None + if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] + else: + rack_types = _parse_vendor_files( + self.repo, + selection.racks_path, + vendor["name"], + self.config.slugs or [], + ) return VendorPlan( vendor=vendor, @@ -738,17 +727,16 @@ def apply(self, plan): ) cache.pump() - if self.netbox.modules: - _process_module_types( - self.config, - self.netbox, - self.reporter, - self.progress, - plan.module_types, - vendor_name=plan.vendor["name"], - task_registry=self.task_registry, - ) - cache.pump() + _process_module_types( + self.config, + self.netbox, + self.reporter, + self.progress, + plan.module_types, + vendor_name=plan.vendor["name"], + task_registry=self.task_registry, + ) + cache.pump() _process_rack_types( self.config, diff --git a/core/log_handler.py b/core/log_handler.py index 2a53eab9..3842cd3c 100644 --- a/core/log_handler.py +++ b/core/log_handler.py @@ -19,7 +19,7 @@ def __init__(self, verbose: bool): def _timestamp(self): """Return the current time formatted as HH:MM:SS.""" - return datetime.now().strftime("%H:%M:%S") + return datetime.now().astimezone().strftime("%H:%M:%S") def set_console(self, console): """Set the Rich Console instance used for output, or None to fall back to print().""" @@ -74,6 +74,6 @@ def log_ports_created(self, created_ports, parent_type: str, port_type: str = "p for port in created_ports: self.verbose_log( f"{port_type} Template Created: {port.name} - " - + f"{port.type if hasattr(port, 'type') else ''} - {getattr(port, parent_attribute).id} - " - + f"{port.id}" + f"{port.type if hasattr(port, 'type') else ''} - {getattr(port, parent_attribute).id} - " + f"{port.id}" ) diff --git a/core/module_bay_types.py b/core/module_bay_types.py new file mode 100644 index 00000000..76697cd6 --- /dev/null +++ b/core/module_bay_types.py @@ -0,0 +1,264 @@ +"""Resolve module-bay-type reference names to NetBox ids. + +A device type's bay says which classes of module it accepts, and a module type says +which classes it belongs to. Both sides name those classes as plain strings, so +something has to turn a name into the id of one specific NetBox object, creating it +when the target instance has never seen it. + +That is this module. Callers pass the owning manufacturer and the names; they get +ids back. The scope rule, the catalog files, the lookup, the creation, and the cache +stay in here, so the four call sites (device-type create, module-type create, change +detection, export) do not each restate them. +""" + +import os +import re + +import pynetbox +import requests +import yaml + +from core.errors import FatalError + +# Directory in the devicetype-library holding the catalog, one directory per manufacturer. +CATALOG_DIRNAME = "module-bay-types" + +# Second resolution scope, for a class one vendor defines and another fills. +FALLBACK_MANUFACTURER = "Generic" + + +def manufacturer_slug(name): + """Slugify a manufacturer name the way the library loader does. + + ``core.repo`` reduces every YAML ``manufacturer`` to this slug, because upstream data + disagrees on case (RuggedCOM vs RuggedCom). The catalog keys on the same slug so both + sides meet on one value. + """ + return re.sub(r"\W+", "-", (name or "").lower()) + + +class ModuleBayTypeError(FatalError): + """A reference could not be resolved to exactly one NetBox object. + + Raised per definition so the caller can log it and continue with the next one. + Never raised for a difference this module could paper over: an unresolved name and + a conflicting identity are both reported rather than guessed at. + """ + + +class ModuleBayCatalogError(FatalError): + """The catalog itself could not be read, so no name in it can be trusted. + + A sibling of :class:`ModuleBayTypeError`, never a subclass: every caller recovers from + that one per definition, which would turn a single unreadable catalog into one skipped + component after another instead of ending the run once. + """ + + +class ModuleBayTypeCatalog: + """Turn module-bay-type names into NetBox ids, creating what is missing. + + ``ids_for(manufacturer, names)`` resolves each name in the owning manufacturer's + scope, then in ``Generic``, and raises :class:`ModuleBayTypeError` if neither has + it. Resolution creates the NetBox object, and the manufacturer that owns it, when + they do not exist yet. Returned order is not meaningful; callers compare as sets. + + Export does not use this class: it reads the names off the records NetBox returned, + because an export run resolves nothing and so has no id-to-name mapping of its own. + """ + + def __init__(self, netbox, repo_path, handle): + """Store the NetBox client, the library checkout to read the catalog from, and the log handle.""" + self._netbox = netbox + self._catalog_dir = os.path.join(repo_path, CATALOG_DIRNAME) + self._handle = handle + self._entries = None + self._load_error = None + self._ids = {} + self._manufacturer_ids = {} + + def ids_for(self, manufacturer, names): + """Return the NetBox id for each name, resolved against *manufacturer* then Generic. + + *manufacturer* is a manufacturer slug, as produced by :func:`manufacturer_slug`. + An empty list is a valid instruction to hold no classes; anything that is not a + list of non-empty strings is refused rather than read as empty, because reading a + malformed value as empty would drop a restriction the author asked for. + """ + self._validate(names) + # Duplicates collapse: the relationship is a set. + return sorted({self._id_for(manufacturer, name) for name in names}) + + def identities_for(self, manufacturer, names): + """Return the identity each name resolves to, without touching NetBox. + + The identity is ``(manufacturer slug, slug)``: the object a reference means, not + the name it is written as. Two manufacturers may both define a class called + ``X``, so a name alone cannot say which object is intended. + + This is the query half of the module. :meth:`ids_for` is the command half and + creates what is missing; this one reads only the catalog files, so change + detection can ask what a reference means without causing anything to exist. + """ + self._validate(names) + return frozenset( + (manufacturer_slug(entry["manufacturer"]), entry["slug"]) + for entry in (self._lookup(manufacturer, name) for name in names) + ) + + @staticmethod + def _validate(names): + """Reject anything that is not a list of non-empty names.""" + if names is None or not isinstance(names, list): + raise ModuleBayTypeError(f"module_bay_types must be a list of names, got {names!r}") + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ModuleBayTypeError(f"module_bay_types entries must be non-empty names, got {name!r}") + + def _id_for(self, manufacturer, name): + entry = self._lookup(manufacturer, name) + cache_key = (manufacturer_slug(entry["manufacturer"]), entry["name"]) + if cache_key not in self._ids: + self._ids[cache_key] = self._netbox_id(entry) + return self._ids[cache_key] + + def _lookup(self, manufacturer, name): + """Find the catalog entry for *name*, owner scope first, then Generic.""" + entries = self._load_catalog() + for scope in (manufacturer, manufacturer_slug(FALLBACK_MANUFACTURER)): + entry = entries.get((scope, name)) + if entry is not None: + return entry + raise ModuleBayTypeError( + f"Module bay type {name!r} is not in the catalog for manufacturer " + f"{manufacturer!r} or {FALLBACK_MANUFACTURER!r}" + ) + + def _load_catalog(self): + """Read every catalog file once, indexed by (manufacturer, name). + + A failure is cached too: it is terminal for the run, and re-walking the tree for + every later lookup only repeats the same error more slowly. + """ + if self._entries is not None: + return self._entries + if self._load_error is not None: + raise self._load_error + try: + self._entries = self._read_catalog() + except ModuleBayCatalogError as exc: + self._load_error = exc + raise + return self._entries + + def _read_catalog(self): + """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + + def _unreadable(exc): + # os.walk skips a directory it cannot read; a short catalog resolves to the + # wrong entry rather than failing, so make the traversal error terminal. + raise ModuleBayCatalogError( + f"Module bay type catalog directory {exc.filename!r} could not be read: {exc}" + ) from exc + + entries = {} + for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable): + for filename in sorted(files): + if not filename.endswith((".yaml", ".yml")): + continue + path = os.path.join(root, filename) + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + except (OSError, yaml.YAMLError) as exc: + raise ModuleBayCatalogError( + f"Module bay type catalog file {path!r} could not be read: {exc}" + ) from exc + if data is None: + continue # an empty or comment-only document is not an entry + if not isinstance(data, dict): + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is not a mapping, got {type(data).__name__}" + ) + # Reject a half-written entry here; the readers index on name and dereference slug. + invalid = [ + field + for field in ("name", "slug", "manufacturer") + if not isinstance(data.get(field), str) or not data[field].strip() + ] + if invalid: + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" + ) + key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) + if key in entries: + raise ModuleBayCatalogError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + entries[key] = data + return entries + + @staticmethod + def _request(action, description): + """Run one NetBox request, reporting a rejection as a catalog error. + + Callers resolve one definition at a time and recover from ModuleBayTypeError. A + raw RequestError or a dropped connection escapes that recovery and ends the run, + which can leave a parent half created and every later definition unprocessed. + """ + try: + return action() + except pynetbox.RequestError as exc: + raise ModuleBayTypeError(f"NetBox rejected {description}: {exc}") from exc + except requests.exceptions.RequestException as exc: + raise ModuleBayTypeError(f"NetBox could not be reached for {description}: {exc}") from exc + + def _netbox_id(self, entry): + """Return the id of the NetBox object for *entry*, creating it if absent.""" + manufacturer_id = self._manufacturer_id(entry["manufacturer"]) + existing = self._request( + lambda: list( + self._netbox.dcim.module_bay_types.filter(manufacturer_id=manufacturer_id, name=entry["name"]) + ), + f"the lookup of module bay type {entry['name']!r}", + ) + for record in existing: + if record.slug != entry["slug"]: + raise ModuleBayTypeError( + f"NetBox already has module bay type {entry['name']!r} for " + f"{entry['manufacturer']!r} with slug {record.slug!r}, but the catalog " + f"says {entry['slug']!r}. Resolve the conflict in NetBox; this import " + f"will not rename it." + ) + return record.id + + payload = {"name": entry["name"], "slug": entry["slug"], "manufacturer": manufacturer_id} + if entry.get("description"): + payload["description"] = entry["description"] + created = self._request( + lambda: self._netbox.dcim.module_bay_types.create(payload), + f"creating module bay type {entry['name']!r}", + ) + self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") + return created.id + + def _manufacturer_id(self, name): + """Return the id of *name*, creating the manufacturer when the catalog needs it. + + A Generic-scoped class is reachable from any vendor, so an import filtered to one + manufacturer still has to create the manufacturer that owns the class. + """ + if name in self._manufacturer_ids: + return self._manufacturer_ids[name] + found = self._request( + lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), + f"the lookup of manufacturer {name!r}", + ) + if found: + self._manufacturer_ids[name] = found[0].id + else: + created = self._request( + lambda: self._netbox.dcim.manufacturers.create({"name": name, "slug": manufacturer_slug(name)}), + f"creating manufacturer {name!r}", + ) + self._handle.verbose_log(f"Manufacturer Created: {name} - {created.id}") + self._manufacturer_ids[name] = created.id + return self._manufacturer_ids[name] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index c6535506..9ca1094a 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -4,10 +4,10 @@ comparison against existing repo YAML files. """ -import warnings -from typing import Any, Sequence +from collections.abc import Sequence +from typing import Any -from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES +from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS # Row order sets the component key order of the serialized YAML. COMPONENT_ENDPOINT_NAMES = [component.endpoint for component in COMPONENT_TYPES] @@ -26,7 +26,6 @@ "feed_leg": None, "maximum_draw": None, "allocated_draw": None, - "positions": 1, # rear port default; include only when > 1 } # Device type scalar field order for output. @@ -109,9 +108,7 @@ def _should_include(field: str, val: Any) -> bool: return False if isinstance(val, str) and val == "": return False - if field in _OMIT_IF_EQUAL and val == _OMIT_IF_EQUAL[field]: - return False - return True + return not (field in _OMIT_IF_EQUAL and val == _OMIT_IF_EQUAL[field]) def _serialize_component(record: Any, fields: Sequence[str]) -> dict: @@ -125,48 +122,103 @@ def _serialize_component(record: Any, fields: Sequence[str]) -> dict: return result +def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: + """Return the catalog name of each related object, which is how YAML names them. + + The names come off the record the query returned, not from the import-side catalog: + an export run resolves nothing, so it has no id-to-name mapping of its own. + + An empty relation writes no key. Emitting an empty list instead would add the key to + every bay in the library and make _repo_supersedes report every existing definition as + differing, and it tells a fresh import nothing that omitting it does not. + """ + result = {} + for relation in relations: + names = sorted( + name for name in (getattr(item, "name", None) for item in getattr(record, relation, None) or []) if name + ) + if names: + result[relation] = names + return result + + def _serialize_front_port(record: Any) -> dict: - """Serialize a front port template, including rear_port mapping.""" + """Serialize a front port template's own fields. + + The rear-port linkage is no longer written here: NetBox 4.5 moved it to a through + table and the library schema follows, carrying it in a top-level ``port-mappings`` + stanza built by :func:`_port_mappings`. + """ result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) - mappings = getattr(record, "mappings", None) or [] - if mappings: - if len(mappings) > 1: - port_name = getattr(record, "name", "") - warnings.warn( - f"Front port '{port_name}' has {len(mappings)} mappings; " - "only the first will be exported. " - "Full multi-mapping support requires DTL schema update (see issue #78).", - UserWarning, - stacklevel=4, - ) - m = mappings[0] - rear_port = getattr(m, "rear_port", None) - if rear_port: - result["rear_port"] = rear_port.name - rear_pos = getattr(m, "rear_port_position", None) - rear_pos = _coerce_numeric(rear_pos) - if rear_pos is not None and rear_pos > 1: - result["rear_port_position"] = rear_pos - else: - # Legacy: pre-4.5 NetBox returns rear_port / rear_port_position as direct scalar fields - legacy_rp = getattr(record, "rear_port", None) - if legacy_rp: - result["rear_port"] = legacy_rp.name - legacy_pos = getattr(record, "rear_port_position", None) - legacy_pos = _coerce_numeric(legacy_pos) - if legacy_pos is not None and legacy_pos > 1: - result["rear_port_position"] = legacy_pos + # positions is schema-required but arrived in 4.5, so a pre-4.5 record has none. + result.setdefault("positions", 1) return result +def _port_mappings(records: list) -> list: + """Return the ``port-mappings`` stanza for a type's front port templates. + + Every mapping is written, not just the first: one front port may occupy several + positions across rear ports, which is what the through table exists to express. + + A server below 4.5 has no through table and answers with ``rear_port`` and + ``rear_port_position`` scalars instead. Those describe one mapping, so they are + written as one entry rather than dropped. + """ + stanza = [] + for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): + name = getattr(record, "name", None) + mappings = [] + for mapping in getattr(record, "mappings", None) or []: + rear_port = getattr(mapping, "rear_port", None) + if not rear_port: + continue + mappings.append( + { + "front_port": name, + "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, + "rear_port": rear_port.name, + "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, + } + ) + stanza.extend( + sorted(mappings, key=lambda m: (m["front_port_position"], m["rear_port_position"], m["rear_port"] or "")) + ) + if getattr(record, "mappings", None): + continue + legacy = getattr(record, "rear_port", None) + if legacy: + stanza.append( + { + "front_port": name, + "front_port_position": 1, + "rear_port": legacy.name, + "rear_port_position": _coerce_numeric(getattr(record, "rear_port_position", None)) or 1, + } + ) + return stanza + + +def module_bays_missing_position(serialized: dict) -> list: + """Return the names of module bays the library schema would reject. + + NetBox leaves ``position`` blank on a bay that names no physical slot, and a blank + string writes no key, but the schema requires one on every module bay. + """ + return [bay.get("name", "?") for bay in serialized.get("module-bays", []) if "position" not in bay] + + def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" + component = BY_ENDPOINT[endpoint_name] out = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): if endpoint_name == "front_port_templates": - out.append(_serialize_front_port(record)) + serialized = _serialize_front_port(record) else: - out.append(_serialize_component(record, BY_ENDPOINT[endpoint_name].fields)) + serialized = _serialize_component(record, component.fields) + serialized.update(_serialize_relations(record, component.relations)) + out.append(serialized) return out @@ -177,6 +229,9 @@ def _add_components(result: dict, type_id: int, components_by_id: dict) -> None: records = type_components.get(component.endpoint, []) if records: result[component.yaml_key] = _serialize_component_list(component.endpoint, records) + mappings = _port_mappings(type_components.get("front_port_templates", [])) + if mappings: + result["port-mappings"] = mappings def serialize_device_type(nb_record: Any, components_by_dt_id: dict) -> dict: @@ -236,6 +291,7 @@ def serialize_module_type(nb_record: Any, components_by_mt_id: dict) -> dict: if _should_include(field, val): result[field] = val + result.update(_serialize_relations(nb_record, MODULE_TYPE_RELATIONS)) _add_components(result, nb_record.id, components_by_mt_id) return result diff --git a/core/netbox_api.py b/core/netbox_api.py index 220efae1..02f2566c 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1,21 +1,23 @@ """NetBox REST and GraphQL API client for importing device and module type libraries.""" -from collections import Counter -from contextlib import contextmanager -from functools import lru_cache +import glob import hashlib import json -import re +import os import tempfile import time +from collections import Counter +from contextlib import contextmanager, suppress +from dataclasses import replace +from functools import lru_cache +from pathlib import Path +from typing import Any + import pynetbox import requests -import os -import glob -from pathlib import Path -from typing import Any, Optional from core.change_detector import ChangeDetector, ChangeType +from core.compat import parse_netbox_version, supports_module_bay_types from core.component_cache import ComponentCache from core.component_registry import ( BY_YAML_KEY, @@ -24,9 +26,10 @@ LINK_POWER_PORT, LINK_REAR_PORTS, MODULE_TYPE_COMPONENTS, + MODULE_TYPE_RELATIONS, ) -from core.formatting import log_property_diffs from core.errors import FatalError, UnknownError +from core.formatting import log_property_diffs from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.normalization import values_equal from core.outcomes import EntityKind, Outcome, OutcomeRegistry @@ -50,6 +53,35 @@ def __init__(self, ignore_ssl_errors: bool, cause=None): ) +# Oldest NetBox this importer is tested against; see the CI matrix and the README. +MINIMUM_NETBOX_VERSION = (4, 3) + + +def _relation_identities_differ(catalog, manufacturer, declared, related, *, names_differ): + """Return True when the assigned objects are not the ones the references resolve to. + + Comparing names alone reports equality when NetBox holds a same-named class from + another manufacturer's scope, so the identity is compared where both sides can supply + one. Where they cannot, the name comparison the caller already made stands. + """ + from core.module_bay_types import ModuleBayTypeError, manufacturer_slug + + if not catalog or not manufacturer: + return names_differ + current = set() + for item in related: + slug = getattr(item, "slug", None) + owner = getattr(getattr(item, "manufacturer", None), "slug", None) + if not slug or not owner: + return names_differ + current.add((manufacturer_slug(owner), slug)) + try: + return catalog.identities_for(manufacturer, declared) != frozenset(current) + except ModuleBayTypeError: + # A matching name must not make an unresolvable reference look applied. + return True + + class NetBoxError(FatalError): """A fatal error reported by the NetBox integration.""" @@ -216,7 +248,7 @@ def _is_image_hash_changed(local_path: str, hash_cache: dict, log_fn=None) -> bo return current != cached -def _load_image_hash_cache(path: Optional[str], log_fn=None) -> dict: +def _load_image_hash_cache(path: str | None, log_fn=None) -> dict: """Load the image-hash cache from *path* (JSON), returning an empty dict when it cannot be read. An absent file is the normal first run and stays quiet. Anything else means the @@ -267,10 +299,8 @@ def _save_image_hash_cache(path: str, cache: dict) -> bool: return True except OSError: if tmp_path is not None: - try: + with suppress(OSError): os.unlink(tmp_path) - except OSError: - pass return False @@ -329,6 +359,7 @@ def _retry_on_connection_error(func, *args, **kwargs): raise wait = _RETRY_BACKOFF[attempt] if attempt < len(_RETRY_BACKOFF) else _RETRY_BACKOFF[-1] time.sleep(wait) + return None # Module type scalar properties that can be compared and updated. @@ -401,7 +432,40 @@ def _image_dir_for_yaml(src_file: str, src_segment: str, dst_segment: str) -> "P return Path(*parts) -# from pynetbox import RequestError as APIRequestError +def _is_mapping_removal(prop_change): + """Return True when a ``_mappings`` change only takes mappings away.""" + return ( + prop_change.property_name == "_mappings" + and isinstance(prop_change.old_value, (set, frozenset)) + and isinstance(prop_change.new_value, (set, frozenset)) + and prop_change.new_value < prop_change.old_value + ) + + +def _without_gated_mapping_clears(changes, remove_components, handle=None): + """Drop mapping clears unless removal is enabled. + + Clearing a front-to-rear linkage deletes data, but it reaches NetBox as a property + change rather than a COMPONENT_REMOVED, so it would otherwise ignore the flag and + contradict the "will not remove components" notice. Filtering here rather than at the + write boundary keeps the actionable count and the applied changes in agreement. + """ + if remove_components: + return changes + kept, gated = [], [] + for change in changes: + remaining = [pc for pc in change.property_changes if not _is_mapping_removal(pc)] + if len(remaining) == len(change.property_changes): + kept.append(change) + continue + gated.append(change.component_name) + if remaining: + kept.append(replace(change, property_changes=remaining)) + if gated and handle is not None: + handle.log( + f"Kept existing port mappings on {sorted(gated)}; use --remove-components with --update to clear them." + ) + return kept def _count_actionable_component_changes(changes, remove_components): @@ -462,10 +526,8 @@ def __init__(self, config, handle): self.handle = handle self.netbox: Any = None self.ignore_ssl = config.ignore_ssl_errors - self.modules = False - self.new_filters = False + self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) - self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts self.remove_unmanaged_types = config.remove_unmanaged_types self.verify_images = config.verify_images @@ -477,7 +539,7 @@ def __init__(self, config, handle): _cache_dir = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "nb-dt-import" try: _cache_dir.mkdir(parents=True, exist_ok=True) - self._image_hash_cache_path: Optional[str] = str(_cache_dir / "image-hashes.json") + self._image_hash_cache_path: str | None = str(_cache_dir / "image-hashes.json") except OSError as exc: self.handle.log( "[yellow]Warning: could not create image hash cache directory " @@ -495,6 +557,7 @@ def __init__(self, config, handle): self.ignore_ssl, handle=self.handle, page_size=config.graphql_page_size, + supports_module_bay_types=self.module_bay_types, ) try: self.existing_manufacturers = self.get_manufacturers() @@ -506,9 +569,9 @@ def __init__(self, config, handle): self.handle, self.counter, self.ignore_ssl, - self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, + module_bay_types_supported=self.module_bay_types, repo_path=config.repo_path, max_threads=config.preload_threads, ) @@ -570,10 +633,10 @@ def connect_api(self): raise UnknownError("NetBox API Error", cause=e) from e def verify_compatibility(self): - """Check the connected NetBox version and configure feature flags accordingly. + """Refuse a server below the supported floor and set the feature flags above it. - Sets ``self.modules = True`` for NetBox >= 3.2 and ``self.new_filters = True`` - for >= 4.1. Logs the detected version when the new-filter flag is enabled. + Only the flags that still vary across supported releases are set here: the 4.3 + floor makes everything introduced at or below 4.1 unconditional. """ # nb.version should be the version in the form '3.2' # Strip non-numeric suffixes (e.g. "4.1-beta") before converting to int. @@ -601,26 +664,26 @@ def verify_compatibility(self): msg += f"\nResponse body (may be from an intermediate proxy):\n{body}" msg += f"\nHint: Verify that {self.url} is reachable and not blocked by a proxy." raise NetBoxError(msg) from e - _raw = [int(re.sub(r"\D.*", "", x.strip()) or "0") for x in nb_version.split(".")] - version_split = (_raw + [0, 0])[:2] # pad to (major, minor) to guard against single-component strings + version_split = parse_netbox_version(nb_version) - # Later than 3.2 - # Might want to check for the module-types entry as well? - if version_split[0] > 3 or (version_split[0] == 3 and version_split[1] >= 2): - self.modules = True - - # check if version >= 4.1 in order to use new filter names (https://github.com/netbox-community/netbox/issues/15410) - if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 1): - self.new_filters = True - self.rack_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using new filters.") + # Below the floor the run dies later naming a schema field, not the real cause. + if tuple(version_split) < MINIMUM_NETBOX_VERSION: + minimum = ".".join(str(part) for part in MINIMUM_NETBOX_VERSION) + raise NetBoxError( + f"NetBox {nb_version} is not supported: this importer requires NetBox {minimum} or later. " + f"Older releases fail part way through with a GraphQL schema error rather than here." + ) # NetBox 4.5 replaced FrontPortTemplate.rear_port (FK) + rear_port_position (int) # with a ManyToMany through table (PortMapping). The creation and read APIs differ. # https://github.com/netbox-community/netbox/issues/20564 if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 5): self.m2m_front_ports = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + self.handle.log(f"Netbox version {nb_version} found. Using M2M front/rear port mappings.") + + if supports_module_bay_types(nb_version): + self.module_bay_types = True + self.handle.log(f"Netbox version {nb_version} found. Module bay types are supported.") def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" @@ -733,9 +796,8 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er netbox=self.netbox, device_type_id=dt.id, device_type_yaml=device_type, - new_filters=self.new_filters, ) - except Exception as exc: # defensive: classifier must never break the run + except Exception as exc: # defensive: classifier must never break the run # noqa: BLE001 self.handle.verbose_log(f"Failure classifier raised {type(exc).__name__}: {exc}") return False, None @@ -768,7 +830,7 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er try: for step in resolution.remediation_steps: step() - except Exception as exc: + except Exception as exc: # a failed remediation is reported, not raised # noqa: BLE001 self.handle.log(f"Auto-resolve failed for {dt.model}: {exc}") return False, resolution @@ -1043,7 +1105,10 @@ def _handle_existing_device_type( # Apply component changes component_errors = [] if dt_change.component_changes: - actionable_count = _count_actionable_component_changes(dt_change.component_changes, remove_components) + component_changes = _without_gated_mapping_clears( + dt_change.component_changes, remove_components, self.handle + ) + actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1053,11 +1118,11 @@ def _handle_existing_device_type( self.device_types.update_components( device_type, dt.id, - dt_change.component_changes, + component_changes, parent_type="device", ) if remove_components: - self.device_types.remove_components(dt.id, dt_change.component_changes, parent_type="device") + self.device_types.remove_components(dt.id, component_changes, parent_type="device") after_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1160,9 +1225,13 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima yaml_key = component.yaml_key if yaml_key not in device_type: continue - if yaml_key == "module-bays" and not self.modules: - continue - self.device_types.create_components(yaml_key, device_type[yaml_key], dt_id, context=src_file) + self.device_types.create_components( + yaml_key, + device_type[yaml_key], + dt_id, + context=src_file, + manufacturer=device_type.get("manufacturer"), + ) if component_errors: # The type exists but not all of its components do. self.outcomes.record( @@ -1404,11 +1473,11 @@ def filter_new_module_types(module_types, all_module_types): Returns: list[dict]: Module types not found in *all_module_types*. """ - new_module_types = [] - for module_type in module_types: - if NetBox._find_existing_module_type(module_type, all_module_types) is None: - new_module_types.append(module_type) - return new_module_types + return [ + module_type + for module_type in module_types + if NetBox._find_existing_module_type(module_type, all_module_types) is None + ] def _log_module_property_diffs(self, mfr_slug, model, fields_info, component_changes=None): """Emit diff-u style lines for changed module type properties and component changes. @@ -1522,6 +1591,8 @@ def filter_actionable_module_types(self, module_types, all_module_types, only_ne if not values_equal(module_type[f], nb_val): changed_fields_info.append((f, nb_val, module_type[f])) + changed_fields_info += self._type_relation_changes(module_type, existing_module) + component_changes = detector._compare_components(module_type, existing_module.id, parent_type="module") if changed_fields_info or component_changes: @@ -1574,6 +1645,58 @@ def _fetch_module_type_existing_images(self): ) return module_type_existing_images + def _type_relation_changes(self, module_type, existing_module): + """Return (field, current, wanted) for each of the type's own relations that differs. + + A relation is a list, so the scalar comparison loop never sees it. A field the + query did not return is skipped rather than read as empty, which would otherwise + report a change on every run. + """ + changes: list[tuple[str, list[str], list[str]]] = [] + if not self.module_bay_types: + return changes + for field in MODULE_TYPE_RELATIONS: + if field not in module_type: + continue + netbox_value = getattr(existing_module, field, _MISSING) + if netbox_value is _MISSING: + continue + related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] + declared = module_type[field] + if not isinstance(declared, list) or any(not isinstance(x, str) or not x.strip() for x in declared): + # Malformed or bare key: leave the relation unmanaged rather than clear it. + continue + wanted = sorted(set(declared)) + current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) + if _relation_identities_differ( + self.device_types.module_bay_type_catalog, + self.device_types._manufacturer_slug(module_type.get("manufacturer")), + declared, + related, + names_differ=wanted != current, + ): + changes.append((field, current, wanted)) + return changes + + def _resolve_type_relations(self, payload): + """Return *payload* with its own relation fields resolved from names to ids. + + Raises ModuleBayTypeError if a name does not resolve, so the caller can report the + module type rather than write it with the restriction silently dropped. + """ + names = {field: payload[field] for field in MODULE_TYPE_RELATIONS if field in payload} + if not names: + return payload + if not self.module_bay_types: + # The server predates ModuleBayType; sending the field would be rejected. + return {k: v for k, v in payload.items() if k not in names} + manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) + resolved = { + field: self.device_types.module_bay_type_catalog.ids_for(manufacturer, value) + for field, value in names.items() + } + return {**payload, **resolved} + def _try_update_module_type(self, curr_mt, module_type_res, src_file): """Apply pending field updates to an existing module type in NetBox. @@ -1590,6 +1713,18 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + if self.module_bay_types: + from core.module_bay_types import ModuleBayTypeError + + for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): + try: + updates[field] = self.device_types.module_bay_type_catalog.ids_for( + self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted + ) + except ModuleBayTypeError as exc: + # Not fatal to the run, but not a success either; the caller records it. + self.handle.log(f"Skipped {field} on {curr_mt.get('model')}: {exc} (Context: {src_file})") + return False, False if not updates: return True, False try: @@ -1622,7 +1757,12 @@ def _create_module_type_components(self, curr_mt, module_type_id, src_file): yaml_key = component.yaml_key if yaml_key in curr_mt: self.device_types.create_components( - yaml_key, curr_mt[yaml_key], module_type_id, parent_type="module", context=src_file + yaml_key, + curr_mt[yaml_key], + module_type_id, + parent_type="module", + context=src_file, + manufacturer=curr_mt.get("manufacturer"), ) if component_errors: # The module type exists but not all of its components do. @@ -1651,6 +1791,7 @@ def _apply_module_type_component_updates( self.device_types.ensure_components_ready(manufacturer_slug=curr_mt["manufacturer"]["slug"]) identity = f"{module_type_res.manufacturer.name}/{module_type_res.model}" component_changes = self.change_detector._compare_components(curr_mt, module_type_res.id, parent_type="module") + component_changes = _without_gated_mapping_clears(component_changes, remove_components, self.handle) if component_changes: actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_updated = self.counter["components_updated"] @@ -1756,7 +1897,7 @@ def _process_single_module_type( if module_type_res is not None: self.handle.verbose_log( f"Module Type Cached: {module_type_res.manufacturer.name} - " - + f"{module_type_res.model} - {module_type_res.id}" + f"{module_type_res.model} - {module_type_res.id}" ) # Upload images before the scalar PATCH so attachments are created # even if the property update later fails (module already exists in @@ -1776,14 +1917,27 @@ def _process_single_module_type( ) else: try: - module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, curr_mt) + from core.module_bay_types import ModuleBayTypeError + + try: + payload = self._resolve_type_relations(curr_mt) + except ModuleBayTypeError as exc: + self.handle.log(f"Error creating Module Type: {exc} (Context: {src_file})") + self._record_failure( + EntityKind.MODULE_TYPE, + self._yaml_identity(curr_mt), + str(exc), + src_file, + ) + return False + module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, payload) self.counter["module_added"] += 1 is_new = True manufacturer_slug = curr_mt["manufacturer"]["slug"] all_module_types.setdefault(manufacturer_slug, {})[curr_mt["model"]] = module_type_res self.handle.verbose_log( f"Module Type Created: {module_type_res.manufacturer.name} - " - + f"{module_type_res.model} - {module_type_res.id}" + f"{module_type_res.model} - {module_type_res.id}" ) except pynetbox.RequestError as excep: self.handle.log(f"Error creating Module Type: {excep.error} (Context: {src_file})") @@ -2126,7 +2280,7 @@ class _FrontPortRecordWithMappings: All other attribute accesses are forwarded to the underlying record. """ - __slots__ = ("_record", "_mappings_canonical") + __slots__ = ("_mappings_canonical", "_mappings_m2m", "_record") def __init__(self, record): """Wrap *record* and pre-compute a canonical mappings list for ChangeDetector compatibility. @@ -2137,7 +2291,7 @@ def __init__(self, record): """ object.__setattr__(self, "_record", record) mappings_raw = getattr(record, "mappings", None) - canonical: Optional[list] + canonical: list | None if mappings_raw is not None: # NetBox >= 4.5: mappings is a list of PortTemplateMapping objects canonical = [] @@ -2160,10 +2314,18 @@ def __init__(self, record): else: # NetBox < 4.5: rear_port_position is a direct scalar field rp_pos = getattr(record, "rear_port_position", None) + # The pre-4.5 query asks for rear_port { id name }; keeping the name is what lets + # a move to a different rear port be seen at all. + legacy_rp = getattr(record, "rear_port", None) + legacy_name = ( + (legacy_rp.get("name") if isinstance(legacy_rp, dict) else getattr(legacy_rp, "name", None)) + if legacy_rp is not None + else None + ) canonical = ( [ { - "rear_port_name": None, + "rear_port_name": legacy_name, "front_port_position": 1, "rear_port_position": rp_pos, } @@ -2172,6 +2334,10 @@ def __init__(self, record): else None # Both mappings and rear_port_position absent; skip comparison. ) object.__setattr__(self, "_mappings_canonical", canonical) + # Which model the record came from, recorded rather than inferred: an empty M2M list + # carries no names to infer from, and reading it as pre-4.5 drops the rear port name + # the patch needs. + object.__setattr__(self, "_mappings_m2m", mappings_raw is not None) def __getattr__(self, name): """Delegate attribute access to the wrapped record.""" @@ -2187,11 +2353,11 @@ def __init__( handle, counter, ignore_ssl, - new_filters, *, graphql, repo_path, m2m_front_ports=False, + module_bay_types_supported=False, max_threads=8, ): """Initialize empty DeviceTypes cache structures; no data is fetched at construction time. @@ -2204,8 +2370,8 @@ def __init__( handle (LogHandler): Sink for creation and error messages. counter (Counter): Shared operation counter updated during creation. ignore_ssl (bool): Whether SSL certificate verification is disabled. - new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. + module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. m2m_front_ports (bool): Whether NetBox uses the 4.5+ M2M port mapping model. max_threads (int): Maximum number of concurrent threads for component preloading. @@ -2214,20 +2380,20 @@ def __init__( self.handle = handle self.counter = counter self.ignore_ssl = ignore_ssl - self.new_filters = new_filters self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports + self.module_bay_types_supported = module_bay_types_supported self.max_threads = max_threads self.components = ComponentCache( netbox, graphql, handle, - new_filters, max_threads, wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None + self._module_bay_type_catalog = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2342,7 +2508,7 @@ def _create_generic( payload = extract_error_payload(excep.error) per_item = payload if isinstance(payload, list) and len(payload) == len(to_create) else [] reported = 0 - for item, error in zip(to_create, per_item): + for item, error in zip(to_create, per_item, strict=False): if error: reported += 1 self._log_component_error( @@ -2428,7 +2594,11 @@ def _apply_mappings_change(self, comp_name, new_mappings, yaml_mappings, update_ update_data["rear_port"] = None update_data["rear_port_position"] = None return - first = next(iter(new_mappings)) + if len(new_mappings) > 1: + self._log_component_error( + f'Multiple mappings for front port "{comp_name}" on NetBox < 4.5: only first mapping applied' + ) + first = sorted(new_mappings)[0] if len(first) != 3: # Legacy NetBox (<4.5): ChangeDetector emits 2-tuples (fp_pos, rp_pos) # because rear port names are unavailable via the GraphQL API. @@ -2486,6 +2656,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, if change.component_name in existing: comp = existing[change.component_name] update_data = {"id": comp.id} + unresolved = False for pc in change.property_changes: if comp_type == "front-ports" and pc.property_name == "_mappings": yaml_front_port = next( @@ -2501,8 +2672,21 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, parent_type, ) continue + if pc.property_name in component.relations: + # The comparison works in names; NetBox wants ids. + from core.module_bay_types import ModuleBayTypeError + + try: + update_data[pc.property_name] = self.module_bay_type_catalog.ids_for( + self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value + ) + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{change.component_name}': {exc}") + unresolved = True + break + continue update_data[pc.property_name] = pc.new_value - if len(update_data) > 1: # has fields beyond just "id" + if not unresolved and len(update_data) > 1: # has fields beyond just "id" updates.append(update_data) success_count = 0 @@ -2547,7 +2731,13 @@ def _apply_additions_for_type(self, comp_type, changes, yaml_data, device_type_i if not components_to_add: return - self.create_components(comp_type, components_to_add, device_type_id, parent_type=parent_type) + self.create_components( + comp_type, + components_to_add, + device_type_id, + parent_type=parent_type, + manufacturer=yaml_data.get("manufacturer"), + ) def update_components(self, yaml_data, device_type_id, component_changes, parent_type="device"): """Update existing components and add new components based on detected changes. @@ -2778,7 +2968,7 @@ def link_rear_ports(items, pid): else: if len(resolved) > 1: ctx = f" (Context: {context})" if context else "" - self.handle.log( + self._log_component_error( f'Multiple mappings for {label} "{port["name"]}" on NetBox < 4.5: ' f"only first mapping applied{ctx}" ) @@ -2837,7 +3027,66 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): f"Connection error bridging interfaces after {_MAX_RETRIES} retries: {e} (Context: {context})" ) - def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None): + @property + def module_bay_type_catalog(self): + """The module-bay-type catalog, built once per run from the library checkout.""" + if self._module_bay_type_catalog is None: + from core.module_bay_types import ModuleBayTypeCatalog + + self._module_bay_type_catalog = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_type_catalog + + @staticmethod + def _manufacturer_slug(manufacturer): + """Return the manufacturer slug, whatever shape the caller happens to hold. + + ``core.repo`` rewrites every YAML ``manufacturer`` to ``{"slug": ...}`` before the + importer sees it, so that is the usual shape; a plain name is slugified here. + """ + from core.module_bay_types import manufacturer_slug + + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or manufacturer_slug(manufacturer.get("name")) + return manufacturer_slug(getattr(manufacturer, "name", manufacturer)) + + def _resolve_relations(self, component, items, manufacturer): + """Turn each relation field's names into NetBox ids, dropping items that cannot resolve. + + A component whose restriction cannot be resolved is skipped and logged rather than + created without it: creating the bay anyway would silently discard the restriction. + """ + if not component.relations: + return items + if not self.module_bay_types_supported: + # The server predates ModuleBayType; drop the field rather than have it rejected. + return [{k: v for k, v in item.items() if k not in component.relations} for item in items] + from core.module_bay_types import ModuleBayTypeError + + manufacturer = self._manufacturer_slug(manufacturer) + resolved = [] + for item in items: + names = {field: item[field] for field in component.relations if field in item} + if not names: + resolved.append(item) + continue + if not manufacturer: + # No scope to resolve in, and NetBox wants ids: sending the names would fail. + self._log_component_error( + f"Skipped {component.label} '{item.get('name', 'Unknown')}': no manufacturer to " + f"resolve {', '.join(sorted(names))} in" + ) + continue + try: + replacements = { + field: self.module_bay_type_catalog.ids_for(manufacturer, value) for field, value in names.items() + } + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") + continue + resolved.append({**item, **replacements}) + return resolved + + def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None, manufacturer=None): """Create component templates of one kind for one parent, skipping those that exist. The registry row for *yaml_key* supplies the endpoint, the cache name, the log label @@ -2850,6 +3099,8 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co parent_id (int): NetBox ID of the parent device or module type. parent_type (str): ``"device"`` or ``"module"``. context (str | None): Optional context string appended to log messages. + manufacturer (dict | str | None): Owning manufacturer, used to resolve any + relation fields the registry row declares. """ component = BY_YAML_KEY[yaml_key] label = component.create_label(parent_type) @@ -2869,7 +3120,7 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co self._create_generic( component, - items, + self._resolve_relations(component, items, manufacturer), parent_id, parent_type=parent_type, post_process=post_process, @@ -2900,7 +3151,7 @@ def upload_images(self, baseurl, token, images, device_type): file_handles = {} try: for field, path in images.items(): - file_handles[field] = (os.path.basename(path), open(path, "rb")) + file_handles[field] = (os.path.basename(path), open(path, "rb")) # noqa: SIM115 response = requests.patch( url, headers=headers, @@ -2918,11 +3169,9 @@ def upload_images(self, baseurl, token, images, device_type): except OSError as e: self.handle.log(f"Error reading image file for device type {device_type}: {e}") finally: - for _, (_, fh) in file_handles.items(): - try: + for _, fh in file_handles.values(): + with suppress(Exception): fh.close() - except Exception: - pass def upload_image_attachment(self, baseurl, token, image_path, object_type, object_id): """Upload an image as an Image Attachment to a NetBox object. diff --git a/core/normalization.py b/core/normalization.py index c8846ec9..d807f38a 100644 --- a/core/normalization.py +++ b/core/normalization.py @@ -1,6 +1,11 @@ """Shared value-normalization helpers for YAML-vs-NetBox comparisons.""" +def is_explicit_list(value): + """Only an explicit YAML list manages a relation; null leaves it unmanaged.""" + return isinstance(value, list) + + def normalize_values(yaml_val, nb_val): """Normalize a YAML/NetBox value pair for comparison. diff --git a/core/outcomes.py b/core/outcomes.py index cf0b75e8..a9a9c798 100644 --- a/core/outcomes.py +++ b/core/outcomes.py @@ -16,7 +16,6 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List, Optional class EntityKind(str, Enum): @@ -48,9 +47,9 @@ class OutcomeRecord: kind: EntityKind identity: str # human-readable identifier (e.g. "Supermicro/SuperServer-6028TR-HTR") outcome: Outcome - reason: Optional[str] = None - blocking_objects: List[str] = field(default_factory=list) - hint: Optional[str] = None + reason: str | None = None + blocking_objects: list[str] = field(default_factory=list) + hint: str | None = None class OutcomeRegistry: @@ -62,7 +61,7 @@ class OutcomeRegistry: def __init__(self) -> None: """Initialise an empty registry.""" - self._records: List[OutcomeRecord] = [] + self._records: list[OutcomeRecord] = [] def record( self, @@ -70,9 +69,9 @@ def record( identity: str, outcome: Outcome, *, - reason: Optional[str] = None, - blocking_objects: Optional[List[str]] = None, - hint: Optional[str] = None, + reason: str | None = None, + blocking_objects: list[str] | None = None, + hint: str | None = None, ) -> None: """Append a new outcome record.""" self._records.append( @@ -87,27 +86,27 @@ def record( ) @property - def records(self) -> List[OutcomeRecord]: + def records(self) -> list[OutcomeRecord]: """All recorded outcomes (read-only view).""" return list(self._records) - def failures(self) -> List[OutcomeRecord]: + def failures(self) -> list[OutcomeRecord]: """Return only the FAILED records.""" return [r for r in self._records if r.outcome == Outcome.FAILED] - def partials(self) -> List[OutcomeRecord]: + def partials(self) -> list[OutcomeRecord]: """Return only the PARTIAL records (some but not all changes applied).""" return [r for r in self._records if r.outcome == Outcome.PARTIAL] - def summary_by_kind(self) -> Dict[EntityKind, Dict[Outcome, int]]: + def summary_by_kind(self) -> dict[EntityKind, dict[Outcome, int]]: """Aggregate counts grouped by ``(kind, outcome)``.""" - agg: Dict[EntityKind, Dict[Outcome, int]] = {} + agg: dict[EntityKind, dict[Outcome, int]] = {} for r in self._records: agg.setdefault(r.kind, {}).setdefault(r.outcome, 0) agg[r.kind][r.outcome] += 1 return agg - def render_failure_report(self) -> List[str]: + def render_failure_report(self) -> list[str]: """Render a multi-line operator-facing failure report. Returns an empty list when no failures or partials were recorded. @@ -118,7 +117,7 @@ def render_failure_report(self) -> List[str]: if not failures and not partials: return [] - lines: List[str] = [] + lines: list[str] = [] lines.append("=" * 60) lines.append("FAILED / PARTIAL UPDATE REPORT") lines.append("=" * 60) diff --git a/core/repo.py b/core/repo.py index 2022cb57..4c368911 100644 --- a/core/repo.py +++ b/core/repo.py @@ -4,15 +4,17 @@ import json import os import pickle +from collections.abc import Sequence from glob import glob from re import sub as re_sub -from typing import Optional, Sequence from urllib.parse import urlparse -from git import Repo, exc + import yaml +from git import Repo, exc from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +from core.normalization import is_explicit_list # Top-level directories that make a checkout a device-type library. LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") @@ -78,7 +80,7 @@ def find_class(self, module, name): _INDEX_MAX_BYTES = 10 * 1024 * 1024 # 10 MiB — DTL index files are typically <1 MiB -def _resolve_index_path(base_dir: str, stem: str) -> "Optional[str]": +def _resolve_index_path(base_dir: str, stem: str) -> "str | None": """Return the path to a DTL index file, preferring the JSON form over the legacy pickle. Upstream DTL replaced ``tests/known-*.pickle`` with ``tests/known-*.json`` (GHSA-492p-5wp7-2w7c). @@ -95,8 +97,8 @@ def _resolve_index_path(base_dir: str, stem: str) -> "Optional[str]": def _vendor_slugs_from_index( - index_path: "Optional[str]", slugs_lower: list, slug_format, subdir_filter: "Optional[str]" = None -) -> "Optional[set]": + index_path: "str | None", slugs_lower: list, slug_format, subdir_filter: "str | None" = None +) -> "set | None": """Load a (model_name, vendor_dir) index and return the set of vendor slugs matching *slugs_lower*. *index_path* may point at a ``.json`` (current) or ``.pickle`` (legacy) file; the loader is @@ -108,7 +110,7 @@ def _vendor_slugs_from_index( return None try: entries = _safe_index_load(index_path) - except Exception: + except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return None result = set() for model_name, vendor_dir in entries: @@ -121,7 +123,7 @@ def _vendor_slugs_from_index( return result -def _safe_abs_path(repo_root: str, relpath: str) -> "Optional[str]": +def _safe_abs_path(repo_root: str, relpath: str) -> "str | None": """Return the absolute path for *relpath* inside *repo_root*, or None if it escapes the root.""" abs_path = os.path.normpath(os.path.join(repo_root, *relpath.replace("\\", "/").split("/"))) return abs_path if abs_path.startswith(os.path.normpath(repo_root) + os.sep) else None @@ -214,9 +216,7 @@ def validate_git_url(url): if url.startswith("https://"): parsed = urlparse(url) if parsed.scheme == "https" and parsed.hostname: - # Optional: enforce an allowlist if desired - # if parsed.hostname not in ("github.com", "gitlab.com"): - # return False, f"Host not allowed: {parsed.hostname}" + # Any HTTPS host is accepted: the library is forked and self-hosted widely. return True, None return False, "Invalid HTTPS URL" @@ -272,6 +272,59 @@ def validate_repo_path(repo_path): return True, "" +def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): + """Return ``({front_port_name: [mapping, ...]}, error)`` for the pre-4.5 inline format. + + The inline keys are removed from each entry as they are read, so the caller is left with + one representation to reason about. + """ + inline_mappings: dict = {} + for fp in front_ports: + rp_name = fp.get("rear_port") + if rp_name is None: + continue + fp_name = fp.get("name") + if rear_ports_declared and rp_name not in rear_by_name: + return {}, f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" + rp_pos = fp.pop("rear_port_position", 1) + fp.pop("rear_port") + inline_mappings.setdefault(fp_name, []).append( + {"rear_port": rp_name, "front_port_position": 1, "rear_port_position": rp_pos} + ) + return inline_mappings, None + + +def _conflicting_mapping(inline_mappings, stanza_mappings): + """Return an error when the two formats cannot both be honoured. + + Carrying both is allowed only while they agree, which is what a half-finished migration + looks like; disagreeing is the case where guessing a winner would silently pick one. + A stanza speaks for the whole file, so an inline linkage it omits cannot be honoured + either. A port the stanza alone names is not in question: it had no inline linkage. + """ + if not (inline_mappings and stanza_mappings): + return None + + def _shape(mappings): + return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) + + for name in sorted(inline_mappings): + if name not in stanza_mappings: + return ( + f"Error: front port '{name}' declares an inline rear_port but the port-mappings " + f"stanza does not list it; the stanza is authoritative, so add '{name}' to it " + f"or remove the inline rear_port keys" + ) + inline = _shape(inline_mappings[name]) + stanza = _shape(stanza_mappings[name]) + if inline != stanza: + return ( + f"Error: front port '{name}' has conflicting mapping definitions " + f"(inline: {inline}, port-mappings stanza: {stanza})" + ) + return None + + def normalize_port_mappings(data): """Normalize port mapping definitions in a parsed YAML device/module type dict. @@ -309,9 +362,9 @@ def normalize_port_mappings(data): """ front_ports = data.get("front-ports") or [] port_mappings_stanza = data.get("port-mappings") - - if not front_ports and "port-mappings" not in data: - return None + stanza_authoritative = is_explicit_list(port_mappings_stanza) + if port_mappings_stanza is not None and not stanza_authoritative: + return f"Error: port-mappings must be a list: {port_mappings_stanza!r}" front_by_name = {fp["name"]: fp for fp in front_ports if fp.get("name")} rear_ports_declared = "rear-ports" in data @@ -320,28 +373,16 @@ def normalize_port_mappings(data): # --- Old inline format --- # Collect rear_port references declared directly on front-port entries. - inline_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - for fp in front_ports: - rp_name = fp.get("rear_port") - if rp_name is None: - continue - fp_name = fp.get("name") - if rear_ports_declared and rp_name not in rear_by_name: - return f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" - rp_pos = fp.pop("rear_port_position", 1) - fp.pop("rear_port") - inline_mappings.setdefault(fp_name, []).append( - { - "rear_port": rp_name, - "front_port_position": 1, - "rear_port_position": rp_pos, - } - ) + inline_mappings, error = _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared) + if error: + return error # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - if "port-mappings" in data: + if stanza_authoritative: for entry in port_mappings_stanza or []: + if not isinstance(entry, dict): + return f"Error: port-mappings entry must be a mapping: {entry!r}" fp_name = entry.get("front_port") rp_name = entry.get("rear_port") if not fp_name or not rp_name: @@ -357,29 +398,23 @@ def normalize_port_mappings(data): "rear_port_position": entry.get("rear_port_position", 1), } ) - del data["port-mappings"] - - # --- Conflict detection --- - # Accept both formats simultaneously only when they describe identical mappings. - if inline_mappings and stanza_mappings: - all_names = set(inline_mappings) | set(stanza_mappings) - for name in all_names: - inline = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in inline_mappings.get(name, []) - ) - stanza = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in stanza_mappings.get(name, []) + data.pop("port-mappings", None) + + conflict = _conflicting_mapping(inline_mappings, stanza_mappings) + if conflict: + return conflict + + if stanza_authoritative: + if not stanza_mappings and inline_mappings: + return ( + "Error: port-mappings is empty but front port(s) " + f"{sorted(inline_mappings)} still declare an inline rear_port" ) - if inline != stanza: - return ( - f"Error: front port '{name}' has conflicting mapping definitions " - f"(inline: {inline}, port-mappings stanza: {stanza})" - ) + for fp in front_ports: + fp["_mappings"] = stanza_mappings.get(fp.get("name"), []) + return None - effective = stanza_mappings if stanza_mappings else inline_mappings - for fp_name, mappings in effective.items(): + for fp_name, mappings in inline_mappings.items(): if fp_name in front_by_name: front_by_name[fp_name]["_mappings"] = mappings @@ -398,7 +433,7 @@ def parse_single_file(file): `src` set to the file path. str: Error string beginning with "Error:" describing YAML parsing or other failure. """ - with open(file, "r") as stream: + with open(file) as stream: try: data = yaml.safe_load(stream) manufacturer = data["manufacturer"] @@ -414,7 +449,7 @@ def parse_single_file(file): return data except yaml.YAMLError as excep: return f"Error: {excep}" - except Exception as e: + except Exception as e: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return f"Error: {e}" @@ -589,7 +624,7 @@ def clone_repo(self): except Exception as git_error: raise UnknownError("Git Repository Error", cause=git_error) from git_error - def get_devices(self, base_path, vendors: Optional[Sequence[str]] = None): + def get_devices(self, base_path, vendors: Sequence[str] | None = None): """Discover device YAML files and vendor directories under a base path. Args: @@ -666,7 +701,7 @@ def resolve_slug_files(self, slugs): device_files: dict = {} # vendor_slug -> [abs_path] try: known_slugs = _safe_index_load(device_index) - except Exception: + except Exception: # an unreadable index or definition is skipped, not fatal # noqa: BLE001 return None for entry_slug, relpath in known_slugs: @@ -733,7 +768,7 @@ def discover_vendors(self, devices_path, modules_path, racks_path): # Return sorted list by slug return sorted(vendors_dict.values(), key=lambda v: v["slug"]) - def parse_files(self, files: list, slugs: Optional[Sequence[str]] = None, progress=None): + def parse_files(self, files: list, slugs: Sequence[str] | None = None, progress=None): """Parse YAML device files into device type dicts, optionally filtering and tracking progress. Args: diff --git a/core/update_failure_resolver.py b/core/update_failure_resolver.py index 7e2eb1f6..205610e4 100644 --- a/core/update_failure_resolver.py +++ b/core/update_failure_resolver.py @@ -20,11 +20,10 @@ from __future__ import annotations import json +from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from typing import Any, Callable, List, Optional - -from core.compat import device_type_filter_kwargs +from typing import Any class FailureKind(str, Enum): @@ -59,10 +58,10 @@ class FailureResolution: kind: FailureKind description: str = "" - blocking_objects: List[str] = field(default_factory=list) - dependent_devices_count: Optional[int] = None - dependent_devices_sample: List[str] = field(default_factory=list) - remediation_steps: List[Callable[[], None]] = field(default_factory=list) + blocking_objects: list[str] = field(default_factory=list) + dependent_devices_count: int | None = None + dependent_devices_sample: list[str] = field(default_factory=list) + remediation_steps: list[Callable[[], None]] = field(default_factory=list) hint: str = "" @property @@ -93,7 +92,7 @@ def extract_error_payload(error: Any) -> Any: if isinstance(error, (bytes, bytearray)): try: error = error.decode("utf-8", errors="replace") - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return error if isinstance(error, str): try: @@ -122,7 +121,7 @@ def _matches_subdevice_role_constraint(payload: Any) -> bool: return False -def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> tuple[int, List[str]]: +def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, list[str]]: """Query NetBox for devices using *device_type_id*. Returns ``(count, sample_names)`` where ``sample_names`` is up to 5 names @@ -133,13 +132,11 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ - filter_kwargs = device_type_filter_kwargs(device_type_id, new_filters=new_filters) + filter_kwargs = {"device_type_id": device_type_id} try: devices = list(netbox.dcim.devices.filter(**filter_kwargs, limit=5)) - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return -1, [] sample = [getattr(d, "name", None) or str(getattr(d, "id", "?")) for d in devices[:5]] if len(devices) < 5: @@ -147,12 +144,12 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b # We capped at limit=5; query the real total separately. try: total = netbox.dcim.devices.count(**filter_kwargs) - except Exception: + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 total = len(devices) return total, sample -def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> Optional[List[Any]]: +def _list_device_bay_templates(netbox: Any, device_type_id: int) -> list[Any] | None: """Return all ``DeviceBayTemplate`` records attached to *device_type_id*. Returns ``None`` when the NetBox query itself fails (network error, 5xx, etc.) @@ -161,16 +158,10 @@ def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ try: - return list( - netbox.dcim.device_bay_templates.filter( - **device_type_filter_kwargs(device_type_id, new_filters=new_filters) - ) - ) - except Exception: + return list(netbox.dcim.device_bay_templates.filter(device_type_id=device_type_id)) + except Exception: # pynetbox raises many types; a probe failure is not fatal # noqa: BLE001 return None @@ -180,7 +171,6 @@ def classify_device_type_update_failure( netbox: Any, device_type_id: int, device_type_yaml: dict, - new_filters: bool = False, ) -> FailureResolution: """Classify a ``pynetbox.RequestError`` raised while updating a device type. @@ -192,7 +182,6 @@ def classify_device_type_update_failure( device_type_yaml: Parsed YAML dict for this device-type (used to detect whether the YAML *also* lists device bays — in which case we cannot blindly delete them). - new_filters: When True, use updated filter parameter names (NetBox ≥ 4.1). Returns: A :class:`FailureResolution` describing the constraint and (when safe) @@ -207,7 +196,7 @@ def classify_device_type_update_failure( ) # SUBDEVICE_ROLE_FLIP path ------------------------------------------------- - blocking_templates = _list_device_bay_templates(netbox, device_type_id, new_filters=new_filters) + blocking_templates = _list_device_bay_templates(netbox, device_type_id) if blocking_templates is None: return FailureResolution( kind=FailureKind.MANUAL_REQUIRED, @@ -216,7 +205,7 @@ def classify_device_type_update_failure( ) blocking_names = [getattr(t, "name", str(getattr(t, "id", "?"))) for t in blocking_templates] - dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id, new_filters=new_filters) + dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id) # YAML must NOT redefine device-bays — otherwise deleting them would just # cause our own component-creation step to fail or thrash. This catches diff --git a/nb-dt-import.py b/nb-dt-import.py index 6d53f2da..cbd5cb68 100644 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """Import NetBox device and module types from the community library.""" -from contextlib import contextmanager -from datetime import datetime import sys +from contextlib import contextmanager +from datetime import UTC, datetime -from pynetbox.core.query import RequestError as NetBoxRequestError import requests +from pynetbox.core.query import RequestError as NetBoxRequestError from rich.panel import Panel from rich.progress import ( BarColumn, @@ -80,10 +80,7 @@ def _effective_speed(task, primary_attr): def render(self, task): """Render the current or finished speed.""" - if task.finished: - speed = self._effective_speed(task, "finished_speed") - else: - speed = self._effective_speed(task, "speed") + speed = self._effective_speed(task, "finished_speed") if task.finished else self._effective_speed(task, "speed") if speed is None: return Text("- it/s") return Text(f"{speed:.1f} it/s") @@ -135,7 +132,7 @@ def _run_export_diff(config: RunConfig, handle): def _run(config: RunConfig): """Build and execute the selected run pipeline.""" - started_at = datetime.now() + started_at = datetime.now(UTC) handle = LogHandler(config.verbose) for notice in config.notices: handle.log(notice) @@ -161,40 +158,41 @@ def main(): try: config = resolve_run_config() except ConfigError as exc: - raise SystemExit(str(exc)) + raise SystemExit(str(exc)) from exc try: return _run(config) except FatalError as exc: if config.verbose and exc.formatted_traceback: print(exc.formatted_traceback, end="") - raise SystemExit(str(exc)) + raise SystemExit(str(exc)) from exc except requests.exceptions.ConnectionError as exc: + detail = _fmt_connection_error(config.netbox_url, exc) print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: {_fmt_connection_error(config.netbox_url, exc)}", + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: {detail}", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc except GraphQLError as exc: print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: NetBox GraphQL request failed — {exc}\n" - f"[{datetime.now().strftime('%H:%M:%S')}] This may be a temporary connectivity issue. " + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: NetBox GraphQL request failed — {exc}\n" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] This may be a temporary connectivity issue. " "Check that NetBox is reachable and try again.", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc except NetBoxRequestError as exc: print( - f"[{datetime.now().strftime('%H:%M:%S')}] Error: NetBox REST API request failed — {exc}\n" - f"[{datetime.now().strftime('%H:%M:%S')}] Check that NetBox is reachable and" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Error: NetBox REST API request failed — {exc}\n" + f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Check that NetBox is reachable and" " the API token has the required permissions.", file=sys.stderr, ) - raise SystemExit(1) + raise SystemExit(1) from exc if __name__ == "__main__": try: main() except KeyboardInterrupt: - print(f"[{datetime.now().strftime('%H:%M:%S')}] Interrupted by user (Ctrl-C). Exiting.") - raise SystemExit(130) + print(f"[{datetime.now().astimezone().strftime('%H:%M:%S')}] Interrupted by user (Ctrl-C). Exiting.") + raise SystemExit(130) from None diff --git a/pyproject.toml b/pyproject.toml index 99ff4081..c1cad3cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,8 +31,25 @@ dev = [ line-length = 120 [tool.ruff.lint] -select = ["E", "F", "W", "D", "C901"] +select = [ + "E", "F", "W", "D", "C901", + "I", # isort + "UP", # pyupgrade; safe at the 3.12 floor + "B", # bugbear + "DTZ", # datetimes must carry a timezone + "BLE", # no bare `except Exception` + "S", # bandit + "ERA", # no commented-out code + "SIM", "RET", "PERF", "C4", "RUF", "ISC", + "PIE", "FLY", "RSE", "PGH", "TID", "INP", "A", "LOG", "G", + "PT", # pytest style +] ignore = [ + # Conflicts with the formatter, which owns implicit string concatenation. + "ISC001", + # StrEnum.__str__ returns the bare value where (str, Enum) returns ClassName.MEMBER, + # so the rewrite would change logged and serialized output. + "UP042", # Docstring style choices (pick D211+D212 convention) "D203", # no-blank-line-before-class (incompatible with D211) "D213", # multi-line-summary-second-line (incompatible with D212) @@ -52,7 +69,16 @@ ignore = [ max-complexity = 15 [tool.ruff.lint.per-file-ignores] -"tests/**" = ["D102", "D103"] # test functions don't need docstrings +"tests/**" = [ + "D102", "D103", # test functions don't need docstrings + "S101", # asserting is what a test does + "S105", "S106", "S107", # fixture credentials are not secrets + "S108", "S603", "S607", # tmp paths and subprocess calls are the test's subject + "INP001", # tests/ is not an importable package + "ERA001", # commented sample payloads document the case under test + "SIM117", # nested `with` keeps one patch per line and reads better here + "A002", # `id` mirrors the NetBox field name the stub stands in for +] [tool.mypy] python_version = "3.12" diff --git a/tests/conftest.py b/tests/conftest.py index 364bd128..32544ae5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import os -import pytest from unittest.mock import MagicMock, patch +import pytest + from core.config import RunConfig @@ -76,8 +77,14 @@ def mock_git_repo(request): @pytest.fixture def mock_pynetbox(): - """Mock pynetbox to prevent API calls.""" + """Mock pynetbox to prevent API calls. + + Defaults the reported version to the oldest supported release, so a test that does not + care about version gating still constructs a NetBox; the importer refuses anything + older. Individual tests override it to exercise a specific release. + """ with patch("core.netbox_api.pynetbox") as mock_nb: + mock_nb.api.return_value.version = "4.3" yield mock_nb @@ -108,6 +115,13 @@ def mock_graphql_requests(request): } } mock_session.post.return_value = response + # /api/status/ is a GET; without this the probe reads a synthesized MagicMock and + # any shape check on the payload sees something no NetBox would ever return. + status = MagicMock() + status.status_code = 200 + status.raise_for_status = MagicMock() + status.json.return_value = {"netbox-version": "4.7.0"} + mock_session.get.return_value = status yield mock_session.post diff --git a/tests/helpers.py b/tests/helpers.py index 9bbad389..c6f0f381 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,13 @@ """Shared test utilities for the NetBox device-type importer test suite.""" +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlparse + +import pynetbox def paginate_dispatch(data_dict): @@ -49,3 +56,195 @@ def recording_handle(): console = RecordingConsole() handle.set_console(console) return handle, console + + +class FakeNetBox: + """A local HTTP server answering the slice of the NetBox REST API the importer uses. + + Collections are addressed by their URL segment with dashes turned into underscores, + so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates and PATCH + bulk-updates behave the way pynetbox expects, so a real client drives it and the + serialising, filtering and paginating under test are the production ones. + + ``/graphql/`` answers every query with an empty list, which is enough to bring a real + :class:`~core.netbox_api.NetBox` up; the REST half is where the assertions live. + + A test using this must carry the ``real_http`` marker. Without it the suite patches + ``requests.Session`` and no request ever leaves the client. + """ + + _IGNORED_FILTERS = ("limit", "offset", "brief", "exclude") + + # GraphQL collections are named "_list"; the client asks for one per query. + _LIST_KEY = re.compile(r"\b(\w+_list)\b") + + def __init__(self, errors=None, netbox_version="4.7.0", **collections): + """Start the server with *collections* seeded, keyed by collection name. + + *errors* maps a collection name to an HTTP status the server answers it with, so a + test can drive a rejection the importer has to survive. *netbox_version* is what + ``/api/status/`` reports, which is how the export side decides what it may select. + """ + self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} + self.errors = dict(errors or {}) + self.netbox_version = netbox_version + self.requests = [] + self._server = HTTPServer(("127.0.0.1", 0), self._handler()) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def url(self): + """Return the base URL a client should talk to.""" + return f"http://127.0.0.1:{self._server.server_port}" + + def api(self, token="test-token"): + """Return a real pynetbox client pointed at this server.""" + return pynetbox.api(self.url, token=token) + + def close(self): + """Stop serving and release the listening socket.""" + self._server.shutdown() + self._server.server_close() + + def collection(self, name): + """Return the stored records for a collection, creating it empty when unseen.""" + return self.collections.setdefault(name, []) + + def sent(self, verb, name): + """Return the payload of each *verb* request made to collection *name*. + + A bulk create or update sends a list; its entries are returned individually, so a + caller asserting on what was written does not have to care which shape was used. + """ + out = [] + for method, collection, payload in self.requests: + if method != verb or collection != name: + continue + out.extend(payload) if isinstance(payload, list) else out.append(payload) + return out + + def matches(self, name, query): + """Filter a collection the way the NetBox REST API filters it.""" + out = [] + for record in self.collection(name): + ok = True + for key, values in query.items(): + if key in self._IGNORED_FILTERS: + continue + field = key[:-3] if key.endswith("_id") else key + actual = record.get(field) + if isinstance(actual, dict): + actual = actual.get("id") + if str(actual) not in values: + ok = False + break + if ok: + out.append(record) + return out + + def _create(self, name, payload): + """Store the new record(s) and echo them back the way NetBox does. + + A list payload is a bulk create, which is how the importer adds components, and it + answers with a list. + """ + if isinstance(payload, list): + return [self._create_one(name, item) for item in payload] + return self._create_one(name, payload) + + def _create_one(self, name, payload): + """Store one new record and return it as NetBox would echo it back.""" + collection = self.collection(name) + record = {"id": 1000 + len(collection), **payload} + # NetBox echoes a foreign key back as a nested object, not the id it was given. + if isinstance(record.get("manufacturer"), int): + owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) + record["manufacturer"] = {"id": record["manufacturer"], "name": owner.get("name")} + collection.append(record) + return record + + def _patch(self, name, payload): + """Merge a bulk update into the stored records and return the updated ones.""" + updated = [] + for entry in payload if isinstance(payload, list) else [payload]: + record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) + if record is None: + continue + record.update(entry) + updated.append(record) + return updated + + def _handler(self): + """Build the request handler class bound to this server's state.""" + state = self + + class Handler(BaseHTTPRequestHandler): + def _collection_name(self): + return urlparse(self.path).path.rstrip("/").rsplit("/", 1)[-1].replace("-", "_") + + def _body(self): + return json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + + def _reply(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _refused(self, name): + """Answer with the configured status for *name*, if the test set one.""" + status = state.errors.get(name) + if status is None: + return False + self._reply(status, {"detail": f"{name} refused with {status}"}) + return True + + def do_GET(self): + parsed = urlparse(self.path) + name = self._collection_name() + state.requests.append(("GET", name, parsed.query)) + if name == "status": + self._reply(200, {"netbox-version": state.netbox_version}) + return + if self._refused(name): + return + results = state.matches(name, parse_qs(parsed.query)) + self._reply(200, {"count": len(results), "next": None, "previous": None, "results": results}) + + def do_POST(self): + name = self._collection_name() + payload = self._body() + if name == "graphql": + state.requests.append(("POST", name, payload)) + key = state._LIST_KEY.search(payload.get("query", "")) + self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) + return + state.requests.append(("POST", name, payload)) + if self._refused(name): + return + self._reply(201, state._create(name, payload)) + + def do_PATCH(self): + name = self._collection_name() + payload = self._body() + state.requests.append(("PATCH", name, payload)) + if self._refused(name): + return + self._reply(200, state._patch(name, payload)) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + return Handler + + +def write_module_bay_type(root, manufacturer, slug, name, description=None): + """Write one module-bay-type catalog file where the devicetype-library puts it.""" + directory = root / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + body = f"name: {name}\nslug: {slug}\nmanufacturer: {manufacturer}\n" + if description: + body += f"description: {description}\n" + (directory / f"{slug}.yaml").write_text(body, encoding="utf-8") diff --git a/tests/integration/test_import.py b/tests/integration/test_import.py index acb110f7..961697e5 100644 --- a/tests/integration/test_import.py +++ b/tests/integration/test_import.py @@ -19,29 +19,29 @@ Test scenarios -------------- A. All component types created with correct field values - – interfaces (including mgmt_only), power-ports (draw values), + - interfaces (including mgmt_only), power-ports (draw values), console-ports, console-server-ports, power-outlets (power_port link), rear-ports (positions), front-ports (M2M rear_ports), device-bays, module-bays (position). B. Device-type properties stored correctly - – u_height (decimal), is_full_depth (bool), weight, weight_unit, + - u_height (decimal), is_full_depth (bool), weight, weight_unit, airflow, part_number, comments. C. Image linkage - – front_image and rear_image URLs are set on the device type (not just + - front_image and rear_image URLs are set on the device type (not just "uploaded" as orphan files) and the URLs return HTTP 200. D. GraphQL schema consistency - – Query every device-type schema field and every + - Query every device-type schema field and every registry field directly through the GraphQL client so a removed/renamed schema field raises an explicit error rather than a silent false-positive. E. Front-port multi-position linkage - – FP1 → RP1 position 1; FP2 → RP1 position 2 (same rear port). + - FP1 → RP1 position 1; FP2 → RP1 position 2 (same rear port). F. Module-type component creation - – All module component types created; front-port rear_port mapping set. + - All module component types created; front-port rear_port mapping set. G. Idempotency - – Second run: 0 new, 0 modified device types and module types. + - Second run: 0 new, 0 modified device types and module types. H. Update mode - – Delete one interface via REST API; re-run with --update; verify it is + - Delete one interface via REST API; re-run with --update; verify it is recreated with the original type value. Usage:: @@ -67,8 +67,8 @@ import urllib3 from core.change_detector import get_device_type_properties -from core.config import resolve_run_config from core.component_registry import COMPONENT_TYPES +from core.config import resolve_run_config from core.graphql_client import NetBoxGraphQLClient urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -399,7 +399,7 @@ def test_graphql_schema() -> None: ok("get_manufacturers() returned TestVendor") # ── Device types: every schema property present ── - dt_by_model, dt_by_slug = client.get_device_types() + _dt_by_model, dt_by_slug = client.get_device_types() fd = dt_by_slug.get(("testvendor", "testvendor-full-device")) if fd is None: fail("get_device_types() did not return full-device") @@ -426,7 +426,7 @@ def test_graphql_schema() -> None: expected_fields = component.graphql_fields try: records = client.get_component_templates(endpoint_name) - except Exception as exc: + except Exception as exc: # the test reports whatever the client raised # noqa: BLE001 fail( f"get_component_templates({endpoint_name!r}) raised {type(exc).__name__}: {exc} — " f"likely a GraphQL schema error (field removed or renamed)." diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index 3a9b6434..bb79c675 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -1,8 +1,6 @@ from types import SimpleNamespace from unittest.mock import MagicMock -from core.component_cache import ComponentCache -from core.graphql_client import DotDict from core.change_detector import ( ChangeDetector, ChangeReport, @@ -11,6 +9,8 @@ DeviceTypeChange, PropertyChange, ) +from core.component_cache import ComponentCache +from core.graphql_client import DotDict def _cache(**records): @@ -19,7 +19,7 @@ def _cache(**records): Populating through the real object means these tests read the same index the importer builds, rather than a dict that merely looks like it. """ - cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), new_filters=True, max_threads=1) + cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), max_threads=1) for endpoint_name, items in records.items(): cache.populate(endpoint_name, items) return cache @@ -428,6 +428,54 @@ def _make_netbox_comp(self, canonical, **attrs): """Build a netbox component with _mappings_canonical and explicit attributes.""" return SimpleNamespace(_mappings_canonical=canonical, **attrs) + def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): + """mappings=[] is 4.5+ saying "none", not a pre-4.5 record without the field. + + Inferring the model from the data cannot tell those apart when the list is empty, and + a positions-only tuple makes _build_mappings_patch return None, so the mapping is + never added and nothing is logged. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + netbox_comp = _FrontPortRecordWithMappings(SimpleNamespace(name="FP1", type="8p8c", mappings=[])) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "adding a mapping is a change" + tup = next(iter(changes[0].new_value)) + assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" + assert tup[0] == "RP1" + + def test_a_legacy_rear_port_change_is_detected_by_name(self): + """The pre-4.5 query asks for rear_port { id name }, so the name is available. + + Discarding it left only positions to compare, and RP1/1 -> RP2/1 produced no change, + no PATCH and no warning: the definition silently never synced. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + legacy = SimpleNamespace(name="FP1", type="8p8c", rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) + netbox_comp = _FrontPortRecordWithMappings(legacy) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "moving the front port to a different rear port is a change" + assert next(iter(changes[0].new_value))[0] == "RP2" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { @@ -603,12 +651,14 @@ def test_a_reader_failure_propagates(self): from core.change_detector import _load_device_type_properties - with patch( - "core.change_detector.load_properties_for_type", - side_effect=RuntimeError("schema unavailable"), + with ( + patch( + "core.change_detector.load_properties_for_type", + side_effect=RuntimeError("schema unavailable"), + ), + pytest.raises(RuntimeError, match="schema unavailable"), ): - with pytest.raises(RuntimeError, match="schema unavailable"): - _load_device_type_properties("/nonexistent") + _load_device_type_properties("/nonexistent") # --------------------------------------------------------------------------- diff --git a/tests/test_component_cache.py b/tests/test_component_cache.py index a6d9e330..49387cb3 100644 --- a/tests/test_component_cache.py +++ b/tests/test_component_cache.py @@ -7,6 +7,7 @@ import threading import time +from typing import ClassVar import pytest @@ -19,7 +20,6 @@ from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError - # ── Fakes ───────────────────────────────────────────────────────────────────── @@ -187,7 +187,6 @@ def make_cache(netbox=None, graphql=None, handle=None, **kwargs): netbox or FakeNetBox(), graphql or FakeGraphQL(), handle or FakeHandle(), - kwargs.pop("new_filters", True), kwargs.pop("max_threads", 4), **kwargs, ) @@ -268,7 +267,8 @@ def test_a_miss_filters_by_device_type_and_caches_the_answer(self): first = cache.get("interface_templates", "device", 1, endpoint) second = cache.get("interface_templates", "device", 1, endpoint) - assert set(first) == {"eth0"} and first == second + assert set(first) == {"eth0"} + assert first == second assert endpoint.filter_calls == [{"device_type_id": 1}] def test_a_miss_filters_by_module_type_for_a_module_parent(self): @@ -279,14 +279,6 @@ def test_a_miss_filters_by_module_type_for_a_module_parent(self): assert endpoint.filter_calls == [{"module_type_id": 5}] - def test_old_netbox_filter_names_are_used_when_asked(self): - cache = make_cache(new_filters=False) - endpoint = FakeEndpoint() - - cache.get("interface_templates", "device", 1, endpoint) - - assert endpoint.filter_calls == [{"devicetype_id": 1}] - def test_an_empty_result_still_becomes_a_hit(self): """Otherwise every parent with no components is re-read on each lookup.""" cache = make_cache() @@ -358,7 +350,7 @@ def test_each_prefetch_worker_owns_and_closes_its_graphql_client(self): """Concurrent endpoint requests must not share a requests session.""" class WorkerClient: - instances = [] + instances: ClassVar[list] = [] def __init__(self, *args, **kwargs): self.thread_id = None @@ -386,7 +378,7 @@ def test_close_waits_for_workers_and_closes_their_graphql_clients(self): """Cancelling a prefetch must not leak the sessions of workers already running.""" class WorkerClient: - instances = [] + instances: ClassVar[list] = [] started = threading.Event() def __init__(self, *args, **kwargs): @@ -426,7 +418,7 @@ def test_a_prefetch_cannot_be_collected_for_another_vendor(self): cache = make_cache() cache.begin_prefetch(manufacturer_slug="cisco") - with pytest.raises(ValueError, match="cisco.*juniper"): + with pytest.raises(ValueError, match=r"cisco.*juniper"): cache.ensure_ready(manufacturer_slug="juniper") cache.close() diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 2b19c784..f1d05dbe 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -128,10 +128,18 @@ def test_the_query_selects_exactly_these_fields(self): "label", "description", "color", + "positions", _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], - "module_bay_templates": ["id", "name", "position", "label", "description"], + "module_bay_templates": [ + "id", + "name", + "position", + "label", + "description", + "module_bay_types { id name slug manufacturer { slug } }", + ], } def test_only_the_front_port_query_selects_a_nested_block(self): @@ -146,13 +154,17 @@ class TestDerivedComparisonAndExport: @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_every_compared_property_is_fetched(self, component): """A property compared but never queried reads as missing and is skipped in silence.""" - queried = set(component.graphql_fields) | {"_mappings"} + # A relation is selected as a nested block, so take the field name each selection + # opens with. Adding component.relations here instead would make the assertion + # hold even if the query stopped selecting them. + queried = {selection.split(None, 1)[0] for selection in component.graphql_fields} | {"_mappings"} assert set(component.compare_properties) <= queried @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_the_export_writes_every_scalar_the_query_reads(self, component): """Export fields are the query's scalars: an unexported scalar drops out of a round trip.""" - scalars = [name for name in component.graphql_fields if name != "id" and name not in component.graphql_extra] + non_scalar = set(component.graphql_extra) | set(component.graphql_relation_fields) + scalars = [name for name in component.graphql_fields if name != "id" and name not in non_scalar] assert list(component.fields) == scalars def test_front_ports_compare_the_mapping_the_query_selects(self): diff --git a/tests/test_config.py b/tests/test_config.py index 1aae1838..0c77c249 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,6 +2,7 @@ import os import pathlib +import re import subprocess import sys @@ -92,7 +93,9 @@ class TestRequiredVariables: """Required environment values fail with a typed configuration error.""" def test_one_missing_variable_uses_the_catalogue_error(self): - with pytest.raises(EnvironmentVariableError, match='Environment variable "NETBOX_TOKEN" is not set.'): + with pytest.raises( + EnvironmentVariableError, match=re.escape('Environment variable "NETBOX_TOKEN" is not set.') + ): resolve_run_config(argv=[], env={"NETBOX_URL": "http://netbox.local"}) def test_all_missing_variables_use_one_typed_error(self): @@ -139,3 +142,55 @@ def test_a_branch_set_beside_a_real_url_needs_no_notice(self): config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + +class TestInsecureNetboxUrlIsReported: + """The API token travels in an Authorization header, so cleartext transport leaks it.""" + + @pytest.mark.parametrize( + "url", + [ + "http://netbox.example.com", + "http://netbox.example.com:8000/", + "http://10.0.0.5:8000", + ], + ) + def test_http_to_a_remote_host_is_reported(self, url): + config = _resolve(NETBOX_URL=url) + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + @pytest.mark.parametrize( + "url", + [ + "http://localhost:8000", + "http://127.0.0.1:8000/", + "http://[::1]:8000", + ], + ) + def test_http_to_loopback_needs_no_notice(self, url): + """A loopback URL never leaves the host, and the integration tests rely on it.""" + config = _resolve(NETBOX_URL=url) + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_https_needs_no_notice(self): + config = _resolve(NETBOX_URL="https://netbox.example.com") + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + +class TestCleartextCheckMatchesWhatRequestsWillDo: + """The notice is worthless if the URL it parses is not the URL the token is sent to.""" + + def test_a_backslash_authority_is_not_treated_as_loopback(self): + """The host reads as localhost here, but requests targets the address before the backslash.""" + config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_an_unparseable_url_is_reported_not_raised(self): + """Urlparse raises on some authorities; an advisory notice must not abort the run.""" + config = _resolve(NETBOX_URL="http://user[foo]@198.18.0.1") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_docker_workflow.py b/tests/test_docker_workflow.py index 8b8ce602..19fb5d4f 100644 --- a/tests/test_docker_workflow.py +++ b/tests/test_docker_workflow.py @@ -4,7 +4,6 @@ import yaml - _WORKFLOW = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "docker.yml" diff --git a/tests/test_errors.py b/tests/test_errors.py index 16a91d64..cbd7e0d6 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -23,8 +23,10 @@ ), ( SSLVerificationError(False), - "SSL verification failed. IGNORE_SSL_ERRORS is False. " - "Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING.", + ( + "SSL verification failed. IGNORE_SSL_ERRORS is False. " + "Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING." + ), ), (GitCommandError("my-repo"), 'Git error for repo "my-repo".'), (GitInvalidRepositoryError("my-repo"), 'The repo "my-repo" is not a valid git repo.'), diff --git a/tests/test_export_manifest.py b/tests/test_export_manifest.py index 8ffeb796..0cfddd01 100644 --- a/tests/test_export_manifest.py +++ b/tests/test_export_manifest.py @@ -1,7 +1,8 @@ """Tests for core/export_manifest.py.""" import json -from core.export_manifest import load_manifest, save_manifest, is_entry_fresh, update_entry + +from core.export_manifest import is_entry_fresh, load_manifest, save_manifest, update_entry class TestLoadManifest: @@ -93,11 +94,10 @@ class TestIsEntryFresh: """Tests for is_entry_fresh function.""" def test_fresh_when_last_updated_matches(self): - manifest = { - "device-types": {"Nokia/acme-x": {"last_updated": "2024-01-01T00:00:00Z"}}, - "module-types": {}, - "rack-types": {}, - } + """Built through update_entry: a hand-written entry cannot carry the schema revision.""" + manifest = {"device-types": {}, "module-types": {}, "rack-types": {}} + update_entry(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") + assert is_entry_fresh(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") is True def test_stale_when_last_updated_differs(self): @@ -125,3 +125,32 @@ def test_updates_existing_entry(self): manifest = {"device-types": {"Nokia/acme-x": {"last_updated": "old"}}, "module-types": {}, "rack-types": {}} update_entry(manifest, "device-types", "Nokia/acme-x", "2024-02-01T00:00:00Z") assert manifest["device-types"]["Nokia/acme-x"]["last_updated"] == "2024-02-01T00:00:00Z" + + +class TestExportSchemaRevision: + """last_updated alone cannot see a change in what the exporter writes.""" + + def test_an_entry_from_an_older_exporter_is_not_fresh(self): + """The record did not change, but the serialized shape did, so it must be rewritten.""" + from core.export_manifest import is_entry_fresh + + manifest = {"device-types": {"Acme/x": {"last_updated": "2026-01-01T00:00:00Z"}}} + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is False + + def test_an_entry_this_exporter_wrote_is_fresh(self): + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is True + + def test_a_changed_record_is_still_not_fresh(self): + """The revision must not paper over the timestamp check it sits beside.""" + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-02-02T00:00:00Z") is False diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 18c06bf2..7bfa5bd1 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1,15 +1,16 @@ """Tests for core/export.py — Exporter class.""" from dataclasses import replace +from unittest.mock import MagicMock, patch import pytest -from unittest.mock import MagicMock, patch import yaml from core.config import RunConfig from core.export import ( - ExportItem, + _SKIP, Exporter, + ExportItem, _canon_mfr_slug, _is_subset, _make_filename, @@ -17,11 +18,9 @@ _repo_supersedes, _sanitize_attachment_filename, _yaml_equal, - _SKIP, ) from core.log_handler import LogHandler - # ── Fixtures ────────────────────────────────────────────────────────────────── @@ -239,6 +238,120 @@ def test_yaml_equal_normalizes_component_order_and_numbers(self): class TestRepoSupersedes: """Tests for _repo_supersedes / _is_subset (asymmetric containment).""" + def test_an_empty_relation_does_not_make_every_definition_differ(self): + """The serializer omits an empty relation, and this is why it has to. + + _is_subset requires every NetBox leaf to be present in the repo YAML, and + _normalize_for_compare does not drop empty lists. A serialized + "module_bay_types: []" would therefore be absent from every library definition + and re-export the whole library on a NetBox 4.7 server. + """ + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +module-bays: + - {name: Slot 0, position: '0'} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + components = {1: {"module_bay_templates": [bay]}} + serialized = serialize_device_type(record, components) + + assert serialized["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert _repo_supersedes(repo, serialized), "an unchanged definition must not re-export" + bay.module_bay_types = [DotDict(name="Control")] + populated = serialize_device_type(record, components) + assert populated["module-bays"][0]["module_bay_types"] == ["Control"] + assert not _repo_supersedes(repo, populated), "a new restriction must trigger export" + + def test_a_default_positions_does_not_make_every_front_port_differ(self): + """The export writes the schema-required positions; it must not re-export the library. + + A library entry that omits positions means the default, 1. _is_subset requires every + NetBox leaf to be present in the repo YAML, so a serialized positions: 1 compared + against an entry that omits it would report every such definition as differing. + """ + from types import SimpleNamespace + + from core.nb_serializer import _serialize_front_port + + legacy = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + serialized = _serialize_front_port(legacy) + assert serialized == {"name": "FP1", "type": "8p8c", "positions": 1} + + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [serialized]} + + assert _repo_supersedes(repo, nb), "an omitted positions is the default, not a difference" + + def test_a_non_default_positions_still_differs(self): + """Only the default may be treated as absent, or a real change would be suppressed.""" + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c", "positions": 4}]} + + assert _repo_supersedes(repo, nb) is False + + @pytest.mark.parametrize("reverse_repo", [False, True]) + @pytest.mark.parametrize("reverse_netbox", [False, True]) + def test_reordered_netbox_mappings_do_not_trigger_export(self, reverse_repo, reverse_netbox): + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Panel +slug: acme-panel +u_height: 1 +is_full_depth: false +front-ports: + - {name: FP1, type: lc-upc, positions: 2} +port-mappings: + - {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} + - {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 3} +""") + record = DotDict( + id=1, manufacturer=DotDict(name="Acme"), model="Panel", slug="acme-panel", u_height=1, is_full_depth=False + ) + port = DotDict( + name="FP1", + type="lc-upc", + positions=2, + mappings=[ + DotDict(rear_port=DotDict(name="RP2"), front_port_position=2, rear_port_position=3), + DotDict(rear_port=DotDict(name="RP1"), front_port_position=1, rear_port_position=1), + ], + ) + if reverse_repo: + repo["port-mappings"].reverse() + if reverse_netbox: + port.mappings.reverse() + serialized = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + + assert _repo_supersedes(repo, serialized), "unchanged mappings must not trigger export" + repo["port-mappings"].append(dict(repo["port-mappings"][0])) + assert not _repo_supersedes(repo, serialized), "duplicate mappings must keep their multiplicity" + repo["port-mappings"].pop() + port.mappings.append(port.mappings[0]) + duplicated = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, duplicated), "NetBox duplicates must keep their multiplicity" + port.mappings.pop() + port.mappings[0].rear_port_position = 4 + changed = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, changed) + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -853,11 +966,17 @@ def test_fetch_vendor_components_groups_device_and_module_records(self, tmp_path def _side_effect(endpoint_name, manufacturer_slug=None): return [dt_rec, mt_rec] if endpoint_name == "interface_templates" else [] + # A worker must fetch through its own clone, so only the clone answers. + worker = MagicMock() + worker.get_component_templates.side_effect = _side_effect mock_client = MagicMock() - mock_client.get_component_templates.side_effect = _side_effect - with patch("core.export.NetBoxGraphQLClient", return_value=mock_client): - dt_result, mt_result = exporter._fetch_vendor_components("nokia") + mock_client.get_component_templates.side_effect = AssertionError("worker must use clone()") + mock_client.clone.return_value = worker + exporter.graphql = mock_client + dt_result, mt_result = exporter._fetch_vendor_components("nokia") + + assert mock_client.clone.called assert dt_result[11]["interface_templates"] == [dt_rec] assert mt_result[22]["interface_templates"] == [mt_rec] @@ -1539,3 +1658,146 @@ def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_ with pytest.raises(FileNotFoundError, match="No device-type library found"): self._exporter(tmp_path, repo)._verify_repo_available() + + +class TestModuleBayPositionWarning: + """NetBox allows a blank module-bay position; the DTL schema requires one.""" + + @staticmethod + def _item(module_bays, kind="device-type"): + return ExportItem( + kind=kind, + nb_record=_make_dt(), + repo_yaml=None, + serialized={"model": "7750-SR-7s", "module-bays": module_bays}, + reason="absent", + mfr_name="Nokia", + filename="nokia-7750-sr-7s.yaml", + manifest_key="Nokia/nokia-7750-sr-7s", + ) + + @staticmethod + def _write(tmp_path, item): + """Drive the real write path with a real LogHandler, which prints to stdout.""" + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + exporter._get_module_image_details = dict + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): + item = self._item([{"name": "Slot 0"}, {"name": "Slot 1", "position": "1"}]) + + self._write(tmp_path, item) + + out = capsys.readouterr().out + assert "Slot 0" in out + assert "position" in out + assert "nokia-7750-sr-7s.yaml" in out + assert "Slot 1" not in out, "a bay that has a position is not a problem" + + def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): + """'0' is a real position: the MX304 PSU bays use '0' and '1'.""" + item = self._item([{"name": "Slot 0", "position": "0"}]) + + self._write(tmp_path, item) + + # Assert the file was written too: silence alone would also hold if nothing ran. + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert "no position" not in capsys.readouterr().out + + def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + item = replace(self._item([]), serialized={"model": "7750-SR-7s"}) + + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written == {"model": "7750-SR-7s"}, "the write path ran, it simply had nothing to warn about" + assert "no position" not in capsys.readouterr().out + + def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): + """Device types and module types share the module-bay schema definition.""" + item = self._item([{"name": "Sub 0"}], kind="module-type") + + self._write(tmp_path, item) + + assert "Sub 0" in capsys.readouterr().out + + +class TestUnqueriedRelationsSurviveTheExport: + """A pre-4.7 server never returns module_bay_types, so the export must not strip it.""" + + @staticmethod + def _item(repo_yaml, serialized): + return ExportItem( + kind="device-type", + nb_record=_make_dt(), + repo_yaml=repo_yaml, + serialized=serialized, + reason="differs", + mfr_name="Juniper", + filename="mx304.yaml", + manifest_key="Juniper/mx304", + ) + + def _write(self, tmp_path, item, supported): + exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = supported + exporter._get_module_image_details = dict + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) + + def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path): + """Only the description changed; the relation must not be collateral damage.""" + repo = { + "model": "MX304", + "description": "old", + "module-bays": [{"name": "RE0", "position": "0", "module_bay_types": ["MX304-RE"]}], + } + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0", "position": "0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=False) + + assert written["description"] == "new", "the real change still lands" + assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] + + def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): + """An export selected for a changed description must retain the supported server's answer.""" + from core.graphql_client import DotDict + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +description: old +module-bays: + - {name: Slot 0, position: '0', module_bay_types: [Control]} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme", slug="acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + description="new", + front_image=None, + rear_image=None, + last_updated="2026-01-01T00:00:00Z", + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = True + items = exporter._determine_export_set_for_device_types( + [record], {("acme", "acme-chassis"): repo}, {1: {"module_bay_templates": [bay]}} + ) + + assert len(items) == 1 + assert items[0].reason == "differs" + exporter._write_export_items(items, {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Acme" / "Chassis.yaml").read_text()) + assert written["description"] == "new" + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 6c5b22b5..7f538d09 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -3,9 +3,9 @@ import ast import pathlib import textwrap +from unittest.mock import MagicMock, patch import pytest -from unittest.mock import MagicMock, patch import requests from core.graphql_client import DotDict, NetBoxGraphQLClient @@ -118,22 +118,154 @@ def test_logging_dependency_has_a_public_read_only_handle(self): client.handle = LogHandler(True) def test_clone_uses_an_independent_session_with_the_same_settings(self): + """Every stored setting is compared, so a new one cannot be dropped unnoticed. + + clone() re-lists constructor arguments by hand. Listing the settings here by hand + too let supports_module_bay_types default back to False in every cloned worker. + """ handle = MagicMock() sessions = [MagicMock(), MagicMock()] with patch("core.graphql_client.requests.Session", side_effect=sessions): - client = NetBoxGraphQLClient("http://netbox.local", "token", True, handle, 250) + client = NetBoxGraphQLClient( + "http://netbox.local", "token", True, handle, 250, supports_module_bay_types=True + ) clone = client.clone() assert clone is not client assert clone._session is sessions[1] - assert (clone.url, clone.token, clone.ignore_ssl, clone.handle, clone.DEFAULT_PAGE_SIZE) == ( - client.url, - client.token, - client.ignore_ssl, - client.handle, - client.DEFAULT_PAGE_SIZE, - ) + settings = lambda c: {k: v for k, v in vars(c).items() if k != "_session"} # noqa: E731 + assert settings(clone) == settings(client) + + @pytest.mark.real_http + def test_the_cloned_session_is_configured_not_merely_new(self): + """A bare requests.Session() would be independent and completely unauthenticated. + + Marked real_http only to get real Session objects; nothing here sends a request. + """ + client = NetBoxGraphQLClient("http://netbox.local", "nbt_key.secret", ignore_ssl=True) + + clone = client.clone() + + assert clone._session is not client._session + assert clone._session.headers["Authorization"] == "Bearer nbt_key.secret" + assert clone._session.headers["Content-Type"] == "application/json" + assert clone._session.verify is False + + @pytest.mark.real_http + def test_a_v1_token_clone_keeps_the_legacy_auth_scheme(self): + client = NetBoxGraphQLClient("http://netbox.local", "0123456789abcdef") + + assert client.clone()._session.headers["Authorization"] == "Token 0123456789abcdef" + + @pytest.mark.real_http + def test_an_unreachable_server_fails_the_probe_as_a_graphql_error(self): + """It is the export's first request, so a raw transport error escapes as a traceback.""" + from helpers import FakeNetBox + + from core.graphql_client import GraphQLError + + server = FakeNetBox() + url = server.url + server.close() # nothing is listening on that port any more + + with pytest.raises(GraphQLError): + NetBoxGraphQLClient(url, "tok").detect_module_bay_type_support() + + @pytest.mark.real_http + def test_a_non_json_status_body_fails_the_probe_as_a_graphql_error(self): + """A proxy error page answers 200 with HTML; json() then raises ValueError.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"gateway" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + @pytest.mark.parametrize( + ("body", "why"), + [ + (b"[]", "a JSON list has no .get, so the probe raised a bare AttributeError"), + (b"{}", "no netbox-version read as '' and silently disabled the relation"), + (b'{"netbox-version": ""}', "an empty version is not a version"), + (b'{"netbox-version": null}', "a null version is not a version"), + ], + ) + def test_a_wrong_shaped_status_body_fails_the_probe(self, body, why): + """Silently deciding 'unsupported' on a 4.7 server exports without the relation.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): + """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" + from helpers import FakeNetBox + + for version, expected in (("4.7.0", True), ("4.7.0-beta2", True), ("4.6.9", False), ("4.3.7", False)): + server = FakeNetBox(netbox_version=version) + try: + client = NetBoxGraphQLClient(server.url, "tok") + assert client.detect_module_bay_type_support() is expected, version + assert client.supports_module_bay_types is expected + finally: + server.close() + + def test_clone_carries_a_setting_it_does_not_name(self): + """clone() must not enumerate settings: the one it forgets is the one that breaks. + + supports_module_bay_types was lost exactly that way, and adding it to the argument + list only fixes the setting that was already missed. + """ + with patch("core.graphql_client.requests.Session"): + client = NetBoxGraphQLClient("http://netbox.local", "token") + client.added_after_this_test_was_written = "carried" + clone = client.clone() + + assert clone.added_after_this_test_was_written == "carried" def test_init_stores_config(self): from core.graphql_client import NetBoxGraphQLClient @@ -475,8 +607,7 @@ class TestGetManufacturers: def _make_client(self): from core.graphql_client import NetBoxGraphQLClient - client = NetBoxGraphQLClient("http://netbox.local", "tok") - return client + return NetBoxGraphQLClient("http://netbox.local", "tok") def test_returns_dict_keyed_by_name(self, mock_post): data = { @@ -1025,6 +1156,106 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + @pytest.mark.real_http + @pytest.mark.parametrize("caller", ["graphql_relation_fields", "get_module_types"]) + def test_relation_selections_use_the_shared_helper(self, caller): + from helpers import FakeNetBox + + from core.component_registry import BY_ENDPOINT + + expected = "module_bay_types { id name slug manufacturer { slug } }" + if caller == "graphql_relation_fields": + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [expected] + else: + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + assert expected in " ".join(query.split()) + finally: + server.close() + + @pytest.mark.real_http + def test_module_type_query_contains_the_complete_relation_selection(self): + from helpers import FakeNetBox + + from core.component_registry import BY_ENDPOINT + + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + finally: + server.close() + + selection = "module_bay_types { id name slug manufacturer { slug } }" + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [selection] + expected_query = """ +query($pagination: OffsetPaginationInput) { + module_type_list(pagination: $pagination) { + id model part_number airflow description comments weight weight_unit last_updated + module_bay_types { id name slug manufacturer { slug } } + manufacturer { id name slug } + } +} +""" + assert " ".join(query.split()) == " ".join(expected_query.split()) + + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): + """The prefetch runs on clones, not on the client it was cloned from. + + A clone that drops the relation reads every bay without it, and the comparison + then skips a field it never received. + """ + from core.graphql_client import NetBoxGraphQLClient + + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=True) + mock_post.side_effect = _make_paged_responses({"module_bay_template_list": []}, "module_bay_template_list") + + client.clone().get_component_templates("module_bay_templates") + + queries = [call.kwargs["json"]["query"] for call in mock_post.call_args_list] + assert queries + assert all("module_bay_types" in q for q in queries) + + def test_the_module_type_query_selects_the_relation_only_when_supported(self, mock_post): + """The selection has to track the server, in both directions. + + An unselected field reads as missing and is skipped by the comparison, and + selecting it on a server below 4.7 fails the whole query. + """ + from core.graphql_client import NetBoxGraphQLClient + + def _query_for(supported): + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=supported) + mock_post.reset_mock() + mock_post.side_effect = _make_paged_responses({"module_type_list": []}, "module_type_list") + client.get_module_types() + return mock_post.call_args_list[0].kwargs["json"]["query"] + + assert "module_bay_types" in _query_for(True) + assert "module_bay_types" not in _query_for(False) + + def test_the_front_port_fallback_drops_every_45_only_field(self, mock_post): + """Positions arrived with the mapping model, so a pre-4.5 tier must not ask for it. + + Leaving it in a fallback tier makes every tier fail the same way, and the whole + front-port preload dies on a server that would answer the older shape. + """ + from core.component_registry import BY_ENDPOINT + from core.graphql_client import NetBoxGraphQLClient + + fields = list(BY_ENDPOINT["front_port_templates"].graphql_fields) + assert "positions" in fields, "guard: the registry is expected to select positions" + + tiers = list(NetBoxGraphQLClient._front_port_field_variants(fields)) + + assert any("positions" in f for f in tiers[0]), "the 4.5 tier keeps it" + for tier in tiers[1:]: + assert not any("positions" in f for f in tier), f"pre-4.5 tier still asks for it: {tier}" + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { @@ -1543,11 +1774,11 @@ def test_primary_fails_no_mappings_field_reraises(self, mock_post): selection — meaning there is no older tier left to fall back to. """ from dataclasses import replace + from unittest.mock import patch + import core.graphql_client as gc_module from core.component_registry import BY_ENDPOINT from core.graphql_client import GraphQLError - from unittest.mock import patch - import core.graphql_client as gc_module # A NetBox whose schema dropped the mappings block entirely. stripped = {"front_port_templates": replace(BY_ENDPOINT["front_port_templates"], graphql_extra=())} @@ -1660,9 +1891,8 @@ def test_request_exception_retries_then_raises_graphql_error(self, mock_post): mock_post.side_effect = requests.exceptions.ConnectionError("connection refused") client = self._make_client() - with patch("core.graphql_client.time.sleep"): - with pytest.raises(GraphQLError, match="connection refused"): - client.query("{ test }", _retries=1) + with patch("core.graphql_client.time.sleep"), pytest.raises(GraphQLError, match="connection refused"): + client.query("{ test }", _retries=1) def test_request_exception_retries_before_exhausting(self, mock_post): """A RequestException on the first attempt stores last_exc and retries.""" @@ -1674,9 +1904,8 @@ def test_request_exception_retries_before_exhausting(self, mock_post): mock_post.side_effect = requests.exceptions.Timeout("timed out") client = self._make_client() - with patch("core.graphql_client.time.sleep"): - with pytest.raises(GraphQLError, match="timed out"): - client.query("{ test }", _retries=1) + with patch("core.graphql_client.time.sleep"), pytest.raises(GraphQLError, match="timed out"): + client.query("{ test }", _retries=1) # ── Vendor-scoped filtering tests ───────────────────────────────────────── @@ -1783,7 +2012,7 @@ def test_single_vendor_filter(self, mock_post, slugs): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=slugs) + by_model, _by_slug = client.get_device_types(manufacturer_slugs=slugs) assert ("cisco", "Catalyst 3850") in by_model assert by_model[("cisco", "Catalyst 3850")].model == "Catalyst 3850" @@ -1839,7 +2068,7 @@ def test_multiple_vendor_filter(self, mock_post, slugs): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=slugs) + by_model, _by_slug = client.get_device_types(manufacturer_slugs=slugs) assert ("cisco", "Catalyst 3850") in by_model assert ("juniper", "EX4300") in by_model @@ -1877,7 +2106,7 @@ def test_none_manufacturer_slugs_unfiltered(self, mock_post): mock_post.side_effect = _make_paged_responses(data, "device_type_list") client = self._make_client() - by_model, by_slug = client.get_device_types(manufacturer_slugs=None) + _by_model, _by_slug = client.get_device_types(manufacturer_slugs=None) # Verify no filter in the query and no manufacturer variable sent call_payload = mock_post.call_args_list[0][1]["json"] @@ -2248,12 +2477,12 @@ def test_vendor_scoped_two_queries(self, mock_post): # Verify both filters were applied via GraphQL variables (not string interpolation) calls = mock_post.call_args_list - # calls[0]: device_type filter query – data page + # calls[0]: device_type filter query - data page device_payload = calls[0][1]["json"] device_query = device_payload["query"] device_vars = device_payload["variables"] - # calls[1]: device_type filter query – empty terminator (pagination ends) - # calls[2]: module_type filter query – data page + # calls[1]: device_type filter query - empty terminator (pagination ends) + # calls[2]: module_type filter query - data page module_payload = calls[2][1]["json"] module_query = module_payload["query"] module_vars = module_payload["variables"] @@ -2421,7 +2650,7 @@ def test_get_device_types_includes_last_updated(self, mocker): } ] client = self._mock_graphql(mocker, items) - by_model, by_slug = client.get_device_types() + by_model, _by_slug = client.get_device_types() record = by_model[("acme", "M")] assert record.last_updated == "2024-01-15T10:00:00Z" @@ -2693,7 +2922,8 @@ def test_schema_error_still_falls_back_to_the_older_field_tier(self): server.shutdown() assert result == [] - assert "T1" in attempts and "T2" in attempts + assert "T1" in attempts + assert "T2" in attempts @pytest.mark.real_http @@ -2763,7 +2993,8 @@ def test_schema_error_still_triggers_the_unfiltered_fallback(self, method): finally: server.shutdown() - assert "filtered" in attempts and "unfiltered" in attempts + assert "filtered" in attempts + assert "unfiltered" in attempts class TestErrorHandlingStandards: diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 1aa14152..26b09352 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -2,12 +2,12 @@ from collections import Counter from contextlib import contextmanager +from types import SimpleNamespace import pytest +from helpers import recording_handle as _recording_handle from core.errors import VendorSelectionError -from types import SimpleNamespace - from core.import_run import ( ImportRun, RunSummary, @@ -20,7 +20,6 @@ from core.log_handler import LogHandler from core.outcomes import EntityKind, Outcome from core.repo import DTLRepo -from helpers import recording_handle as _recording_handle class _ComponentCache: @@ -59,7 +58,6 @@ class _NetBoxBoundary: """Expose run state and fail if planning starts an import.""" def __init__(self): - self.modules = True self.rack_types = True self.device_types = _DeviceTypes() self.outcomes = _Outcomes() @@ -119,7 +117,7 @@ def discover_vendors(self, _devices_path, _modules_path, _racks_path): @staticmethod def resolve_slug_files(_slugs): """Force the pipeline to use its full file scan.""" - return None + return @staticmethod def get_devices(path, vendors): @@ -271,8 +269,6 @@ def test_execute_returns_snapshot_and_owns_console_lifecycle(make_config, tmp_pa assert isinstance(summary, RunSummary) assert summary.counter["added"] == 2 - assert summary.modules is True - assert summary.rack_types is True assert progress_factory.entered is True assert progress_factory.exited is True assert handle.console is None @@ -434,37 +430,35 @@ def test_banners_omit_the_separator_when_no_vendor_is_given(make_config): class _OutcomeNetBox: """Run-state stand-in carrying a real OutcomeRegistry.""" - def __init__(self, *, modules=True, rack_types=True, counter=None): + def __init__(self, *, counter=None): from core.outcomes import OutcomeRegistry - self.modules = modules - self.rack_types = rack_types self.outcomes = OutcomeRegistry() - base = dict( - added=0, - properties_updated=0, - components_updated=0, - components_added=0, - components_removed=0, - images=0, - manufacturer=0, - module_added=0, - module_updated=0, - rack_type_added=0, - rack_type_updated=0, - ) + base = { + "added": 0, + "properties_updated": 0, + "components_updated": 0, + "components_added": 0, + "components_removed": 0, + "images": 0, + "manufacturer": 0, + "module_added": 0, + "module_updated": 0, + "rack_type_added": 0, + "rack_type_updated": 0, + } base.update(counter or {}) self.counter = Counter(base) def _summary_lines(netbox): """Render a run summary through the real LogHandler and return its lines.""" - from datetime import datetime + from datetime import UTC, datetime handle, console = _recording_handle() handle.verbose = True repo = SimpleNamespace(duplicate_definitions=[]) - summary = RunSummary.capture(netbox, repo, datetime.now()) + summary = RunSummary.capture(netbox, repo, datetime.now(UTC)) _log_run_summary(handle, summary) return console.lines @@ -539,3 +533,54 @@ def test_headline_partials_match_the_itemised_rows(): assert "2 device types partially updated" in text assert "1 modules partially updated" in text assert "Partial updates: 3" in text + + +class TestNaiveStartedAtIsNormalized: + """A naive started_at used to survive construction and fail much later, in the summary.""" + + def test_a_naive_started_at_is_stored_timezone_aware(self, make_config, tmp_path): + from datetime import datetime + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=datetime(2020, 1, 1, 12, 0, 0), # noqa: DTZ001 - a naive value is the subject of this test + ) + + assert run.started_at.tzinfo is not None, "a naive value cannot be subtracted from an aware now()" + + def test_a_run_started_naive_still_produces_a_summary(self, make_config, tmp_path): + """The real failure: RunSummary.capture subtracts started_at from an aware now().""" + from datetime import datetime + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=datetime(2020, 1, 1, 12, 0, 0), # noqa: DTZ001 - a naive value is the subject of this test + ) + + summary = RunSummary.capture(_NetBoxBoundary(), SimpleNamespace(duplicate_definitions=[]), run.started_at) + + assert summary.elapsed.total_seconds() > 0 + + def test_an_aware_started_at_is_left_alone(self, make_config, tmp_path): + from datetime import UTC, datetime + + given = datetime(2020, 1, 1, 12, 0, 0, tzinfo=UTC) + + run = ImportRun( + make_config(), + _RepositoryBoundary(tmp_path), + _NetBoxBoundary(), + LogHandler(False), + _ProgressFactory(), + started_at=given, + ) + + assert run.started_at == given diff --git a/tests/test_log_handler.py b/tests/test_log_handler.py index 3eaf98a5..5b5d9d7f 100644 --- a/tests/test_log_handler.py +++ b/tests/test_log_handler.py @@ -1,9 +1,9 @@ -from types import SimpleNamespace from inspect import signature +from types import SimpleNamespace from unittest.mock import MagicMock, patch -from core.log_handler import LogHandler from core.graphql_client import NetBoxGraphQLClient +from core.log_handler import LogHandler from core.netbox_api import DeviceTypes, NetBox from core.repo import DTLRepo @@ -166,3 +166,34 @@ def test_progress_group_supports_nested_blocks(): handle.end_progress_group() print_mock.assert_called_once_with("[12:00:00] Nested message") + + +class TestTimestampStaysOnLocalWallClock: + """The timestamp is timezone-aware now, which must not move what the operator reads to UTC.""" + + def test_the_timestamp_reads_local_time_not_utc(self, monkeypatch): + """A UTC-based fix for DTZ005 would silently shift every logged line by the local offset.""" + import time + from datetime import UTC, datetime + + monkeypatch.setenv("TZ", "Asia/Tokyo") + time.tzset() + try: + # One absolute instant: 18:04:05 UTC is 03:04:05 the next day in Tokyo. + instant = datetime(2026, 1, 1, 18, 4, 5, tzinfo=UTC) + + class _FrozenClock: + """Models a real clock: naive now() is local, now(tz) converts the same instant.""" + + @staticmethod + def now(tz=None): + if tz is not None: + return instant.astimezone(tz) + return instant.astimezone().replace(tzinfo=None) + + monkeypatch.setattr("core.log_handler.datetime", _FrozenClock) + + assert LogHandler(False)._timestamp() == "03:04:05", "local wall-clock, not the 18:04:05 UTC reading" + finally: + monkeypatch.undo() + time.tzset() diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py new file mode 100644 index 00000000..b2084a1b --- /dev/null +++ b/tests/test_module_bay_type_sync.py @@ -0,0 +1,578 @@ +"""Tests for keeping module bay type assignments in sync with NetBox. + +Resolving a reference name to an id is :mod:`core.module_bay_types` and is tested there. +This file covers the wiring around it: the relation a module type carries itself, the +component update path, and the gate that keeps the field off a server that predates it. + +Both sides are real. The catalog is written to disk and read by a real +``ModuleBayTypeCatalog``; a local HTTP server answers a real ``pynetbox`` client and the +real GraphQL client, so the payloads asserted on here are the ones that would go over the +wire. Only the version handshake is stood in for, to fix the server release under test. +""" + +import pynetbox +import pytest +from helpers import FakeNetBox, recording_handle, write_module_bay_type + +from core.change_detector import ChangeType, ComponentChange, PropertyChange +from core.component_registry import BY_YAML_KEY +from core.graphql_client import NetBoxGraphQLClient +from core.module_bay_types import ModuleBayTypeCatalog +from core.netbox_api import DeviceTypes, NetBox +from core.outcomes import EntityKind, Outcome + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +JUNIPER = {"id": 1, "name": "Juniper", "slug": "juniper"} + + +def stub(**attrs): + """Return an object carrying exactly *attrs*, the way NetBox nests a related object.""" + return type("Stub", (), attrs)() + + +def created_id(server, slug): + """Return the id the server gave the module bay type it created for *slug*.""" + return next(record["id"] for record in server.collection("module_bay_types") if record["slug"] == slug) + + +def related(name, slug=None, manufacturer_slug=None): + """Return a module bay type as NetBox returns it nested inside a relation.""" + return stub(name=name, slug=slug, manufacturer=stub(slug=manufacturer_slug)) + + +class NetBoxRecord: + """An existing NetBox object as the importer holds it: an id, plus what was fetched. + + A field the query did not ask for is genuinely absent, which is the difference the + comparison turns on, so this carries only what a test hands it. + """ + + def __init__(self, record_id, **fields): + """Store the id and the fields this record is meant to have.""" + self.id = record_id + for name, value in fields.items(): + setattr(self, name, value) + + +@pytest.fixture +def catalog_root(tmp_path): + """Write the catalog these tests resolve against and return its root.""" + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot") + return root + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server seeded with the manufacturers the catalog needs.""" + fake = FakeNetBox(manufacturers=[JUNIPER]) + yield fake + fake.close() + + +@pytest.fixture +def make_device_types(server, catalog_root): + """Build a real DeviceTypes talking to the local server, with its cache already primed.""" + + def _make(module_bay_types_supported=True): + handle, console = recording_handle() + device_types = DeviceTypes( + server.api(), + handle, + {}, + False, + graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), + repo_path=str(catalog_root), + module_bay_types_supported=module_bay_types_supported, + ) + device_types.components.ensure_ready() + return device_types, console + + return _make + + +@pytest.fixture +def netbox(make_config, mock_pynetbox, server, catalog_root): + """Build a real NetBox against the local server, reporting the release that has the feature.""" + mock_pynetbox.api.return_value.version = "4.7" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, console = recording_handle() + config = make_config(netbox_url=server.url, repo_path=str(catalog_root)) + nb = NetBox(config, handle) + assert nb.module_bay_types, "a 4.7 server supports module bay types; the rest of this file assumes it" + + # connect_api ran against the patched pynetbox; every call under test uses a real client. + api = server.api() + nb.netbox = api + nb.device_types.netbox = api + nb.device_types.components.netbox = api + nb.device_types.components.ensure_ready() + return nb, console + + +class TestModuleTypeOwnRelation: + """A module type says which classes it belongs to, and that has to stay in sync.""" + + def test_a_missing_class_is_reported_as_a_change(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", [], ["MX304-RE"])] + + def test_the_right_class_is_left_alone(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): + """Juniper owns MX304-RE. A Generic object of that name is not the one referenced.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "generic")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + + def test_names_are_compared_when_netbox_returned_no_identity(self, netbox): + """A record carrying only a name cannot answer the scope question; names still can.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_unresolvable_reference_is_reported_even_when_the_name_matches(self, netbox): + """A name that happens to match must not make an unresolvable reference look applied. + + Comparison cannot say which object the name means, so the write path has to try, + fail, and refuse. Reporting no change here skips that entirely. + """ + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["NO-SUCH-CLASS"]} + + assert nb._type_relation_changes(module_type, existing) == [ + ("module_bay_types", ["NO-SUCH-CLASS"], ["NO-SUCH-CLASS"]) + ] + + def test_a_matching_but_unresolvable_name_does_not_read_as_updated(self, netbox, server): + """The end of that path: the module type is refused, not patched with its scalars.""" + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + assert nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") == (False, False) + assert not server.sent("PATCH", "module_types") + + def test_a_field_the_query_did_not_return_is_skipped(self, netbox): + """Reading an absent field as empty would report a change on every run.""" + nb, _ = netbox + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, NetBoxRecord(7)) == [] + + def test_an_omitted_key_leaves_the_relation_unmanaged(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + + assert nb._type_relation_changes({"model": "JNP304-RE"}, existing) == [] + + def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox): + """A bare key parses as None; clearing on it would drop a restriction nobody removed.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": None} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_names_are_compared_when_the_definition_names_no_manufacturer(self, netbox): + """Without an owning manufacturer there is no scope, so the name comparison stands.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC", "mx304-lmic", "juniper")]) + module_type = {"model": "JNP304-RE", "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_older_server_reports_no_relation_changes(self, netbox): + """Below 4.7 the relation does not exist, so there is nothing to compare.""" + nb, _ = netbox + nb.module_bay_types = False + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + +class TestModuleTypeCreatePayload: + """What the create path sends for a module type's own relation.""" + + def test_names_are_replaced_by_ids(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + resolved = nb._resolve_type_relations(payload) + + created = server.sent("POST", "module_bay_types") + assert [p["slug"] for p in created] == ["mx304-re"] + assert resolved["module_bay_types"] == [created_id(server, "mx304-re")] + assert resolved["model"] == "JNP304-RE" + + def test_a_payload_without_the_relation_is_untouched(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + + assert nb._resolve_type_relations(payload) == payload + assert not server.sent("POST", "module_bay_types") + + def test_an_older_server_never_sees_the_field(self, netbox, server): + """Sending a field the server does not have would fail the whole create.""" + nb, _ = netbox + nb.module_bay_types = False + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._resolve_type_relations(payload) == {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + assert not server.sent("POST", "module_bay_types") + + +class TestModuleTypeUpdate: + """The update path patches the relation, and survives one it cannot resolve.""" + + def test_a_changed_relation_is_patched_as_ids(self, netbox, server): + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (True, True) + assert server.sent("PATCH", "module_types") == [{"id": 7, "module_bay_types": [created_id(server, "mx304-re")]}] + + def test_an_unresolvable_reference_is_logged_and_the_run_goes_on(self, netbox, server): + """One bad module type must not end the run, and must not be written without it.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False), "an unapplied relation must not read as success" + assert not server.sent("PATCH", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + def test_a_scalar_change_is_held_back_when_the_relation_cannot_resolve(self, netbox, server): + """Patching the description while dropping the restriction is a half-applied write.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False) + assert not server.sent("PATCH", "module_types"), "the scalar PATCH must not go out alone" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestModuleTypeCreateRefusal: + """A module type whose relation cannot be resolved is reported, not written without it.""" + + def test_an_unresolvable_reference_skips_the_module_type(self, netbox, server): + nb, console = netbox + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + created = nb._process_single_module_type(curr_mt, "juniper/jnp304-re.yaml", {}, {}, only_new=False) + + assert created is False + assert not server.sent("POST", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + # The registry is the only tally of failures; a path that merely logs is absent + # from the run summary and from the itemised report. + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [(r.kind, r.identity) for r in failed] == [(EntityKind.MODULE_TYPE, "juniper/JNP304-RE")] + assert "NO-SUCH-CLASS" in failed[0].reason + + +class TestModuleTypeUpdateOutcome: + """The run summary must show the module type whose relation could not be applied.""" + + def test_an_unresolvable_relation_is_recorded_as_a_failure(self, netbox, server): + """Driven through the parent operation, because that is where the outcome is recorded.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[], description="old" + ) + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + nb._process_single_module_type( + curr_mt, "juniper/jnp304-re.yaml", {"juniper": {"JNP304-RE": existing}}, {}, only_new=False + ) + + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [r.kind for r in failed] == [EntityKind.MODULE_TYPE], "the failure must reach the run summary" + assert not server.sent("PATCH", "module_types"), "nothing may be written for a type it could not apply" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestComponentCreatePayload: + """A module bay is created with the restriction its definition asked for, or not at all.""" + + def test_names_are_replaced_by_ids(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert resolved == [{"name": "FPC 0", "module_bay_types": [created_id(server, "mx304-lmic")]}] + + def test_an_item_without_the_field_passes_through(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == items + assert not server.sent("POST", "module_bay_types") + + def test_an_unresolvable_restriction_drops_the_bay_rather_than_relaxing_it(self, make_device_types, server): + """Creating the bay without its restriction would silently accept any module.""" + device_types, console = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [ + {"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}, + {"name": "FPC 1", "module_bay_types": ["MX304-LMIC"]}, + ] + + with device_types.collect_component_errors() as collected: + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert [item["name"] for item in resolved] == ["FPC 1"] + # Collected, not just printed: the parent's outcome reason is built from these. + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + assert any("FPC 0" in line for line in console.lines) + + def test_an_older_server_never_sees_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == [{"name": "FPC 0"}] + assert not server.sent("POST", "module_bay_types") + + def test_a_netbox_rejection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A 403 while resolving must not escape the per-component recovery. + + The callers recover from ModuleBayTypeError only, so a raw RequestError ends the + run with a traceback and leaves a partly created parent behind. + """ + device_types, _ = make_device_types() + server.errors["module_bay_types"] = 403 + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e and "403" in e] + + def test_a_lost_connection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A dropped connection is not a RequestError, so it escaped the same recovery.""" + device_types, _ = make_device_types() + server.close() # the port stops answering; resolution now hits a refused connection + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e] + + def test_a_component_kind_with_no_relations_is_untouched(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "xe-0/0/0", "type": "100gbase-x-qsfp28"}] + + assert device_types._resolve_relations(BY_YAML_KEY["interfaces"], items, {"slug": "juniper"}) == items + + def test_an_unknown_manufacturer_drops_the_bay_rather_than_sending_names(self, make_device_types): + """Without a scope the names cannot become ids, and NetBox rejects raw names.""" + device_types, console = make_device_types() + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [] + assert any("FPC 0" in line for line in console.lines) + + def test_an_unknown_manufacturer_still_passes_through_a_bay_with_no_restriction(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == items + + def test_an_older_server_strips_the_field_even_without_a_manufacturer(self, make_device_types): + """The manufacturer question must not decide whether an unsupported field is sent.""" + device_types, _ = make_device_types(module_bay_types_supported=False) + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [{"name": "FPC 0"}] + + +class TestComponentCreateWiring: + """Through create_components(), so the resolution is proved to be wired in, not just present.""" + + def test_a_created_bay_carries_resolved_ids(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert [p["name"] for p in posted] == ["FPC 0"] + assert posted[0]["module_bay_types"] == [created_id(server, "mx304-lmic")] + + def test_an_older_server_creates_the_bay_without_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert posted + assert "module_bay_types" not in posted[0] + + def test_an_unresolvable_bay_is_never_posted(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + with device_types.collect_component_errors() as collected: + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + assert not server.sent("POST", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e] + + +class TestComponentUpdatePayload: + """An existing bay whose restriction changed is patched with ids, not names.""" + + @staticmethod + def _change(name, new_value): + return ComponentChange( + component_type="module-bays", + component_name=name, + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange(property_name="module_bay_types", old_value=[], new_value=new_value)], + ) + + def test_a_changed_restriction_is_patched_as_ids(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + device_types._apply_updates_for_type( + "module-bays", [self._change("FPC 0", ["MX304-LMIC"])], {"manufacturer": {"slug": "juniper"}}, 3, "device" + ) + + assert server.sent("PATCH", "module_bay_templates") == [ + {"id": 55, "module_bay_types": [created_id(server, "mx304-lmic")]} + ] + + def test_a_rejected_patch_is_collected_rather_than_raised(self, make_device_types, server): + """The bay resolves; NetBox refuses the write. That must land in the entity's report.""" + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + server.errors["module_bay_templates"] = 400 + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["MX304-LMIC"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert [e for e in collected if "55" in e] + + def test_an_unresolvable_restriction_is_logged_and_nothing_is_patched(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["NO-SUCH-CLASS"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert not server.sent("PATCH", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + + +class TestCatalogWiring: + """The catalog is built once per run, from the library checkout the run is using.""" + + def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): + device_types, _ = make_device_types() + + catalog = device_types.module_bay_type_catalog + + assert isinstance(catalog, ModuleBayTypeCatalog) + assert device_types.module_bay_type_catalog is catalog + assert not hasattr(device_types, "module_bay_types") + assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py new file mode 100644 index 00000000..e629b393 --- /dev/null +++ b/tests/test_module_bay_types.py @@ -0,0 +1,345 @@ +"""Tests for module bay type reference resolution. + +Driven through the public interface (``ids_for`` / ``identities_for``) against catalog files +the tests write themselves and a real ``pynetbox`` client talking HTTP to a local server. +Nothing here is mocked: the client serialises, filters and paginates for real, which is +where the semantics that matter actually live. +""" + +import pytest +from helpers import FakeNetBox, write_module_bay_type + +from core.module_bay_types import ModuleBayCatalogError, ModuleBayTypeCatalog, ModuleBayTypeError + + +class Handle: + """Capture what the catalog logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +DEFAULT_MANUFACTURERS = ( + {"id": 1, "name": "Juniper", "slug": "juniper"}, + {"id": 2, "name": "Cisco", "slug": "cisco"}, + {"id": 3, "name": "Nokia", "slug": "nokia"}, +) + + +@pytest.fixture +def library(tmp_path): + """Write the catalog the resolution tests resolve against, and return its root. + + QSFP-DD is defined twice on purpose, by Juniper and by Generic, so the owner-scope + rule has two candidates to choose between. + """ + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot compatibility") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot compatibility") + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "Juniper MX304 QSFP-DD cage") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "QSFP-DD pluggable transceiver form factor") + return root + + +@pytest.fixture +def catalog(library): + """Build a catalog wired to a real pynetbox client and a local NetBox-shaped server.""" + servers = [] + + def _make(module_bay_types=(), manufacturers=None, root=None): + server = FakeNetBox( + manufacturers=DEFAULT_MANUFACTURERS if manufacturers is None else manufacturers, + module_bay_types=module_bay_types, + ) + servers.append(server) + return ModuleBayTypeCatalog(server.api(), str(root or library), Handle()), server + + yield _make + for server in servers: + server.close() + + +@pytest.mark.real_http +class TestResolution: + """Names resolve to ids, in the owning manufacturer's scope and then in Generic.""" + + def test_resolves_in_owner_manufacturer_scope(self, catalog): + cat, server = catalog() + ids = cat.ids_for("juniper", ["MX304-RE"]) + assert len(ids) == 1 + created = server.sent("POST", "module_bay_types") + assert created == [ + { + "name": "MX304-RE", + "slug": "mx304-re", + "manufacturer": 1, + "description": "Juniper MX304 routing-engine slot compatibility", + } + ] + + def test_falls_back_to_generic_for_another_manufacturer(self, catalog): + """A Cisco optic reaches the Generic form factor, and Generic is created on demand.""" + cat, server = catalog() + assert len(cat.ids_for("cisco", ["QSFP-DD"])) == 1 + made = server.sent("POST", "manufacturers") + assert made == [{"name": "Generic", "slug": "generic"}] + + def test_owner_scope_wins_over_generic(self, catalog): + """Juniper and Generic both define QSFP-DD, and a Juniper reference means Juniper's.""" + cat, server = catalog() + cat.ids_for("juniper", ["QSFP-DD"]) + created = server.sent("POST", "module_bay_types") + assert [p["manufacturer"] for p in created] == [1] + assert not server.sent("POST", "manufacturers") + + def test_existing_object_is_reused_not_recreated(self, catalog): + cat, server = catalog( + module_bay_types=[{"id": 77, "name": "MX304-RE", "slug": "mx304-re", "manufacturer": {"id": 1}}] + ) + assert cat.ids_for("juniper", ["MX304-RE"]) == [77] + assert not server.sent("POST", "module_bay_types") + + def test_repeated_resolution_is_cached(self, catalog): + cat, server = catalog() + first = cat.ids_for("juniper", ["MX304-RE"]) + before = len(server.requests) + + assert cat.ids_for("juniper", ["MX304-RE"]) == first + assert len(first) == 1, "an empty result would make the request count meaningless" + assert len(server.requests) == before + + def test_order_is_not_meaningful(self, catalog): + cat, _ = catalog() + a = cat.ids_for("juniper", ["MX304-RE", "MX304-LMIC"]) + b = cat.ids_for("juniper", ["MX304-LMIC", "MX304-RE"]) + assert len(a) == 2 + assert sorted(a) == sorted(b) + + +@pytest.mark.real_http +class TestRefusals: + """An unresolved name and a conflicting identity are reported, never papered over.""" + + def test_unresolved_name_raises_rather_than_dropping(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["NO-SUCH-CLASS"]) + assert "NO-SUCH-CLASS" in str(exc.value) + + def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): + """The catalog says mx304-re; NetBox holds mx304_re. Never silently rename.""" + cat, server = catalog( + module_bay_types=[{"id": 88, "name": "MX304-RE", "slug": "mx304_re", "manufacturer": {"id": 1}}] + ) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["MX304-RE"]) + assert "mx304_re" in str(exc.value) + assert "mx304-re" in str(exc.value) + assert not server.sent("POST", "module_bay_types") + + +@pytest.mark.real_http +class TestCatalogReading: + """The catalog is read from real files on disk, including the shapes that are rejected.""" + + def test_entry_without_a_description_is_created_without_one(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + cat, server = catalog(root=tmp_path) + cat.ids_for("generic", ["SFP"]) + created = server.sent("POST", "module_bay_types") + generic = next(m for m in server.collection("manufacturers") if m["slug"] == "generic") + assert created == [{"name": "SFP", "slug": "sfp", "manufacturer": generic["id"]}] + + def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + (tmp_path / "module-bay-types" / "Generic" / "README.md").write_text("not a catalog entry\n") + (tmp_path / "module-bay-types" / "Generic" / "blank.yaml").write_text("# only a comment\n") + cat, _ = catalog(root=tmp_path) + assert len(cat.ids_for("generic", ["SFP"])) == 1 + + def test_a_yaml_document_that_is_not_a_mapping_is_refused(self, tmp_path, catalog): + """Silently skipping it shrinks the catalog, and a Generic entry then answers instead.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + directory = tmp_path / "module-bay-types" / "Juniper" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("- name: SFP\n slug: sfp\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("Juniper", ["SFP"]) + + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): + """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "slug" in str(exc.value) + assert "sfp.yaml" in str(exc.value) + + def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog): + """A parser error escaping the boundary ends the run before it reports anything.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "sfp.yaml" in str(exc.value) + + def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catalog, monkeypatch): + """The load caches only on success, so a bad catalog re-walked the tree every time.""" + import core.module_bay_types as module + + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + + walks = [] + real_walk = module.os.walk + monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) + + for _ in range(3): + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("generic", ["SFP"]) + assert len(walks) == 1 + + def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): + """Two files claiming the same (manufacturer, name) would make a reference ambiguous.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + with pytest.raises(ModuleBayCatalogError) as exc: + cat.ids_for("generic", ["SFP"]) + assert "Duplicate" in str(exc.value) + assert "SFP" in str(exc.value) + + +@pytest.mark.real_http +class TestMalformedReferences: + """A reference list that is not a list of names is refused, never read as empty.""" + + def test_a_bare_key_is_refused_rather_than_clearing(self, catalog): + """`module_bay_types:` with no value parses as None; treating it as [] would clear.""" + cat, server = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", None) + assert "list of names" in str(exc.value) + assert not server.sent("POST", "module_bay_types") + assert not server.sent("POST", "manufacturers") + + def test_a_non_string_entry_is_refused(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [{"name": "MX304-RE"}]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", ["MX304-RE", 7]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [" "]) + + def test_an_empty_list_is_allowed_and_holds_no_classes(self, catalog): + """`module_bay_types: []` is an explicit instruction, not a malformed value.""" + cat, server = catalog() + assert cat.ids_for("juniper", []) == [] + assert not server.sent("POST", "module_bay_types") + assert not server.sent("POST", "manufacturers") + + def test_duplicate_names_collapse(self, catalog): + """The relationship is a set, so a repeated name must not produce a repeated id.""" + cat, _ = catalog() + once = cat.ids_for("juniper", ["MX304-RE"]) + + assert len(once) == 1 + assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once + + +@pytest.mark.real_http +class TestACatalogFailureIsNotAPerComponentSkip: + """A broken catalog is terminal for the run, but every caller recovers per component.""" + + @staticmethod + def _broken_catalog(tmp_path, catalog): + root = tmp_path / "broken" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "fine") + # A half-written entry: the readers index on name and dereference slug. + (root / "module-bay-types" / "Juniper" / "bad.yaml").write_text( + "name: 123\nslug: bad\nmanufacturer: Juniper\n", encoding="utf-8" + ) + return catalog(root=root)[0] + + def test_a_malformed_entry_is_not_reported_as_a_relation_change(self, tmp_path, catalog): + """_relation_change recovers from ModuleBayTypeError, so a load failure must not be one.""" + from types import SimpleNamespace + + from core.change_detector import _relation_change + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + netbox_comp = SimpleNamespace(name="RE0", module_bay_types=[]) + + with pytest.raises(ModuleBayCatalogError): + _relation_change( + "module_bay_types", + {"name": "RE0", "module_bay_types": ["MX304-RE"]}, + netbox_comp, + catalog=broken, + manufacturer="Juniper", + ) + + def test_the_catalog_failure_is_not_a_module_bay_type_error(self, tmp_path, catalog): + """Sibling, not subclass: an `except ModuleBayTypeError` must not swallow it.""" + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + + with pytest.raises(ModuleBayCatalogError) as caught: + broken.identities_for("Juniper", ["MX304-RE"]) + + assert not isinstance(caught.value, ModuleBayTypeError), "per-name catches would swallow it" + + def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, catalog): + """The split must not promote a per-name miss into a run-ending failure.""" + resolver, _server = catalog() + + with pytest.raises(ModuleBayTypeError): + resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) + + +@pytest.mark.real_http +class TestUnreadableCatalogDirectory: + """os.walk swallows a directory it cannot read, which silently shrinks the catalog.""" + + def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, catalog): + """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" + import os + + if not hasattr(os, "geteuid"): + pytest.skip("no POSIX ownership, so the permission bits mean nothing here") + if os.geteuid() == 0: + pytest.skip("root ignores the permission bits this test relies on") + + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "the owner-scoped entry") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "the fallback entry") + vendor_dir = root / "module-bay-types" / "Juniper" + os.chmod(vendor_dir, 0o000) + try: + resolver, _server = catalog(root=root) + + with pytest.raises(ModuleBayCatalogError): + resolver.identities_for("Juniper", ["QSFP-DD"]) + finally: + os.chmod(vendor_dir, 0o700) diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 31245d0b..bb3af27d 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -312,14 +312,14 @@ def test_items_per_second_column_uses_elapsed_fallback_when_finished_speed_missi @pytest.fixture(scope="session", autouse=True) -def _real_library_root(tmp_path_factory): +def real_library_root(tmp_path_factory): """Point the repo mocks at a real library tree: the pipeline stats these paths before reading them.""" global _LIBRARY_ROOT root = tmp_path_factory.mktemp("library") for name in ("device-types", "module-types", "rack-types"): (root / name).mkdir() _LIBRARY_ROOT = root - yield root + return root def _make_mock_repo(device_types=None): @@ -335,13 +335,11 @@ def _make_mock_repo(device_types=None): return mock_repo -def _make_mock_netbox(modules=False, rack_types=False): +def _make_mock_netbox(): """Return a pre-configured NetBox mock.""" from collections import Counter mock_nb = MagicMock() - mock_nb.modules = modules - mock_nb.rack_types = rack_types mock_nb.device_types.existing_device_types = {} mock_nb.device_types.existing_device_types_by_slug = {} mock_nb.count_device_type_images.return_value = 0 @@ -481,7 +479,7 @@ def test_explicit_total_skips_update_in_finally(self, nb_dt_import): # --------------------------------------------------------------------------- -# filter_device_types_by_change_keys – empty-keys branch +# filter_device_types_by_change_keys - empty-keys branch # --------------------------------------------------------------------------- @@ -496,7 +494,7 @@ def test_empty_change_keys_returns_empty_list(self, nb_dt_import): # --------------------------------------------------------------------------- -# select_device_types_* – None report branches +# select_device_types_* - None report branches # --------------------------------------------------------------------------- @@ -560,15 +558,17 @@ def test_cleanup_on_exception(self, nb_dt_import): mock_progress.add_task.return_value = 1 mock_dt = MagicMock() - with pytest.raises(ValueError): - with import_run_module._image_progress_scope(mock_progress, mock_dt, total=3): - raise ValueError("boom") + with ( + pytest.raises(ValueError, match="boom"), + import_run_module._image_progress_scope(mock_progress, mock_dt, total=3), + ): + raise ValueError("boom") assert mock_dt._image_progress is None # --------------------------------------------------------------------------- -# main() – comprehensive branch coverage +# main() - comprehensive branch coverage # --------------------------------------------------------------------------- @@ -828,9 +828,9 @@ def test_missing_env_var_triggers_system_exit(self, nb_dt_import): patch("nb_dt_import.NetBox"), patch.dict(os.environ, {}, clear=True), patch("core.config.load_dotenv"), + pytest.raises(SystemExit), ): - with pytest.raises(SystemExit): - nb_dt_import.main() + nb_dt_import.main() def test_vendors_and_slugs_flags_log_lines(self, nb_dt_import, capsys): """--vendors and --slugs args cause their respective log lines to execute.""" @@ -884,7 +884,7 @@ def test_modules_with_types_to_process(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([module_type], {}, []) MockNetBox.return_value = mock_nb MockNetBox.filter_new_module_types.return_value = [] @@ -912,7 +912,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([], {}, change_log) mock_nb.filter_new_module_types.return_value = [] MockNetBox.return_value = mock_nb @@ -930,7 +930,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): mock_nb.log_module_type_changes.assert_called_once_with(change_log) def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): - """When netbox.modules is True, module_added/updated counters are logged.""" + """Module counters are always logged: every supported release has module types.""" with ( patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), patch("nb_dt_import.DTLRepo") as MockRepo, @@ -938,7 +938,7 @@ def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): patch("nb_dt_import.LogHandler") as MockLogHandler, ): MockRepo.return_value = _make_mock_repo() - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() MockNetBox.return_value = mock_nb nb_dt_import.main() @@ -987,19 +987,6 @@ class TestProcessRackTypes: def _make_args(self, only_new=False): return SimpleNamespace(only_new=only_new) - def test_rack_types_disabled_logs_warning_and_returns(self, nb_dt_import): - """netbox.rack_types=False with actual rack types: warning logged, no further processing.""" - handle = MagicMock() - netbox = MagicMock() - netbox.rack_types = False - - rack_type = {"manufacturer": {"slug": "apc"}, "model": "AR1300", "slug": "apc-ar1300"} - import_run_module._process_rack_types(self._make_args(), netbox, handle, None, [rack_type]) - - handle.log.assert_called_once() - assert "4.1" in handle.log.call_args[0][0] - netbox.get_existing_rack_types.assert_not_called() - def test_empty_rack_types_returns_early(self, nb_dt_import): """rack_types=[]: returns immediately without any logging or API calls.""" handle = MagicMock() @@ -1103,9 +1090,8 @@ def test_entry_point_connection_error_message_references_netbox(self, capsys): with patch("core.repo.DTLRepo") as MockDTLRepo, patch("core.netbox_api.NetBox"): MockDTLRepo.side_effect = _requests.exceptions.ConnectionError("Remote end closed") - with patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]): - with pytest.raises(SystemExit): - runpy.run_path(_NB_DT_IMPORT_PATH, run_name="__main__") + with patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), pytest.raises(SystemExit): + runpy.run_path(_NB_DT_IMPORT_PATH, run_name="__main__") captured = capsys.readouterr() assert "connection" in captured.err.lower() or "netbox" in captured.err.lower() @@ -1252,7 +1238,7 @@ def test_module_type_only_vendor_uses_scoped_preload(self, nb_dt_import): """ mt = {"manufacturer": {"slug": "acbel"}, "model": "M1", "slug": "acbel-m1"} - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Acbel", "slug": "acbel"}] @@ -1367,12 +1353,10 @@ class TestLogRunSummary: def test_rack_types_counters_are_logged(self, nb_dt_import): """When netbox.rack_types is True, rack_type_added/updated counters are logged.""" - from datetime import datetime + from datetime import UTC, datetime handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = True from collections import Counter mock_nb.counter = Counter( @@ -1390,7 +1374,7 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): ) mock_nb.outcomes.render_failure_report.return_value = [] mock_repo = SimpleNamespace(duplicate_definitions=[]) - summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now()) + summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now(UTC)) import_run_module._log_run_summary(handle, summary) @@ -1400,12 +1384,10 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): def test_duplicate_definitions_are_logged(self, nb_dt_import): """When dtl_repo has duplicate_definitions, each entry is logged with kept/ignored.""" - from datetime import datetime + from datetime import UTC, datetime handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = False from collections import Counter mock_nb.counter = Counter( @@ -1430,7 +1412,7 @@ def test_duplicate_definitions_are_logged(self, nb_dt_import): } ] mock_nb.outcomes.render_failure_report.return_value = [] - summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now()) + summary = import_run_module.RunSummary.capture(mock_nb, mock_repo, datetime.now(UTC)) import_run_module._log_run_summary(handle, summary) @@ -1500,9 +1482,9 @@ def test_main_turns_a_fatal_error_into_system_exit(self, nb_dt_import, make_conf with ( patch.object(nb_dt_import, "resolve_run_config", return_value=make_config(verbose=False)), patch.object(nb_dt_import, "_run", side_effect=error), + pytest.raises(SystemExit) as exc_info, ): - with pytest.raises(SystemExit) as exc_info: - nb_dt_import.main() + nb_dt_import.main() assert str(exc_info.value) == 'An unknown error occurred: "Git Repository Error"' assert capsys.readouterr().out == "" @@ -1517,9 +1499,9 @@ def test_main_prints_the_cause_traceback_only_in_verbose_mode(self, nb_dt_import with ( patch.object(nb_dt_import, "resolve_run_config", return_value=make_config(verbose=True)), patch.object(nb_dt_import, "_run", side_effect=error), + pytest.raises(SystemExit), ): - with pytest.raises(SystemExit): - nb_dt_import.main() + nb_dt_import.main() output = capsys.readouterr().out assert "Traceback (most recent call last)" in output @@ -1914,9 +1896,9 @@ def __exit__(self, exc_type, exc, tb): patch("nb_dt_import.NetBox", return_value=mock_nb), patch("core.import_run._process_device_types", side_effect=RuntimeError("boom")), patch("nb_dt_import.get_progress_panel", return_value=_Ctx()), + pytest.raises(RuntimeError, match="boom"), ): - with pytest.raises(RuntimeError, match="boom"): - nb_dt_import.main() + nb_dt_import.main() mock_nb.device_types.components.close.assert_called() @@ -1988,7 +1970,7 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c if files == ["cisco-module-types.yaml"] else [] ) - netbox = _make_mock_netbox(modules=True) + netbox = _make_mock_netbox() slug_resolved = { "device_files": {"empty": [], "cisco": ["resolved.yaml"]}, "module_vendors": {"cisco"}, @@ -2051,19 +2033,19 @@ def test_import_run_stops_preload_in_finally_on_error(self, make_config): ) netbox = _make_mock_netbox() - with ( + with ( # noqa: PT012 patch("core.import_run._process_device_types", side_effect=RuntimeError("boom")), patch("core.import_run._finalize_task_registry") as mock_finalize, + pytest.raises(RuntimeError, match="boom"), ): - with pytest.raises(RuntimeError, match="boom"): - run = import_run_module.ImportRun( - config, - dtl_repo, - netbox, - handle, - lambda _show_remaining_time: nullcontext(progress), - ) - run.execute() + run = import_run_module.ImportRun( + config, + dtl_repo, + netbox, + handle, + lambda _show_remaining_time: nullcontext(progress), + ) + run.execute() netbox.device_types.components.close.assert_called() mock_finalize.assert_called_once_with(progress, {}) @@ -2144,9 +2126,9 @@ class TestExportDiffVendorFilterEndToEnd: """ def test_single_vendor_reaches_the_graphql_request( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco") payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2157,10 +2139,10 @@ def test_single_vendor_reaches_the_graphql_request( assert "Nothing to export" in capsys.readouterr().out def test_multiple_vendors_are_sent_as_a_json_list( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): """A tuple would not compare equal here, and NetBox would not accept it as a list variable.""" - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco,Juniper") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco,Juniper") payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2171,10 +2153,10 @@ def test_multiple_vendors_are_sent_as_a_json_list( assert "Nothing to export" in capsys.readouterr().out def test_no_vendors_queries_every_manufacturer( - self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, _real_library_root + self, nb_dt_import, monkeypatch, tmp_path, capsys, mock_post, real_library_root ): """Without --vendors, config.vendors is (), which the GraphQL layer rejects if it reaches it.""" - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root) + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root) payloads = [call.kwargs["json"] for call in mock_post.call_args_list] filters = _filters_by_list_field(payloads) @@ -2190,7 +2172,7 @@ class TestExportDiffVendorFilterOverRealHTTP: """Same run against a local HTTP server, so the filter is asserted as it is serialized on the wire.""" @staticmethod - def _serve(): + def _serve(netbox_version="4.7.0"): """Serve empty GraphQL pages and record every decoded request body.""" import json import threading @@ -2199,16 +2181,22 @@ def _serve(): bodies = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers["Content-Length"]) - bodies.append(json.loads(self.rfile.read(length))) - payload = b'{"data": {}}' + def _reply(self, payload): self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) + def do_GET(self): + """Answer the version probe the export runs before its first query.""" + self._reply(json.dumps({"netbox-version": netbox_version}).encode()) + + def do_POST(self): + length = int(self.headers["Content-Length"]) + bodies.append(json.loads(self.rfile.read(length))) + self._reply(b'{"data": {}}') + def log_message(self, *args): """Silence the default stderr access log.""" @@ -2216,11 +2204,11 @@ def log_message(self, *args): threading.Thread(target=server.serve_forever, daemon=True).start() return f"http://127.0.0.1:{server.server_port}", server, bodies - def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypatch, tmp_path, real_library_root): url, server, bodies = self._serve() monkeypatch.setenv("NETBOX_URL", url) try: - _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "Cisco,Juniper") + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, real_library_root, "Cisco,Juniper") finally: server.shutdown() server.server_close() @@ -2229,3 +2217,34 @@ def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypa assert set(filters) == set(_LIST_FIELDS) for field, variables in filters.items(): assert variables["manufacturer_slugs"] == ["cisco", "juniper"], field + + def _queries_for_version(self, nb_dt_import, monkeypatch, tmp_path, library_root, version): + """Run one whole export against a server reporting *version* and return its queries.""" + url, server, bodies = self._serve(version) + monkeypatch.setenv("NETBOX_URL", url) + try: + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, library_root, "Juniper") + finally: + server.shutdown() + server.server_close() + return [payload["query"] for payload in bodies] + + def test_a_47_server_is_asked_for_the_module_bay_type_relation( + self, nb_dt_import, monkeypatch, tmp_path, real_library_root + ): + """Not selecting it exports every bay without its restriction, silently. + + Asserted on the module-type query, which this run always issues; the component + queries only run once there is something to export. + """ + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, real_library_root, "4.7.0") + + module_types = [q for q in queries if "module_type_list(" in q] + assert module_types + assert all("module_bay_types" in q for q in module_types) + + def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, real_library_root): + """Selecting a field the schema lacks fails the whole query.""" + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, real_library_root, "4.6.9") + + assert not [q for q in queries if "module_bay_types" in q] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 9250234d..95e9452d 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -441,13 +441,33 @@ def test_rack_type_manufacturer_as_name_string(self): class TestFrontPortSerialization: - """Tests for front port rear_port extraction.""" + """Front port scalars. The rear-port linkage lives in TestPortMappingsStanza.""" - def test_front_port_rear_port_extracted_from_mapping(self): + def test_components_sorted_by_name(self): from types import SimpleNamespace - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) + iface_z = SimpleNamespace( + name="eth9", + type="1000base-t", + label="", + description="", + mgmt_only=False, + enabled=True, + poe_mode=None, + poe_type=None, + rf_role=None, + ) + iface_a = SimpleNamespace( + name="eth0", + type="1000base-t", + label="", + description="", + mgmt_only=False, + enabled=True, + poe_mode=None, + poe_type=None, + rf_role=None, + ) record = _dotdict( id=1, model="X", @@ -465,115 +485,108 @@ def test_front_port_rear_port_extracted_from_mapping(self): front_image=None, rear_image=None, ) - components = {1: {"front_port_templates": [fp]}} + components = {1: {"interface_templates": [iface_z, iface_a]}} result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] + names = [i["name"] for i in result["interfaces"]] + assert names == ["eth0", "eth9"] - def test_front_port_rear_port_position_included_when_gt_1(self): - from types import SimpleNamespace - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=3) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, +class TestRelationSerialization: + """A module bay's restriction has to survive the trip back out to YAML.""" + + @staticmethod + def _bay(name, module_bay_types=None, **extra): + """Build a module bay template as the GraphQL query returns it.""" + return _dotdict( + name=name, + position=None, + label="", description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + module_bay_types=module_bay_types, + **extra, ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port_position"] == 3 - def test_front_port_rear_port_position_zero_omitted(self): - """Position 0 is not a valid DTL value and should be omitted.""" - from types import SimpleNamespace + def test_a_bay_exports_the_names_of_its_classes(self): + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = self._bay("FPC 0", [_dotdict(id=9, name="MX304-LMIC"), _dotdict(id=8, name="QSFP-DD")]) + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC", "QSFP-DD"]}] - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=0) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) + def test_a_bay_with_no_classes_writes_no_key(self): + """An empty list here would add the key to every bay in the library. + + See test_an_empty_relation_does_not_make_every_definition_differ for the effect + that has on the export diff. + """ + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + + result = serialize_module_type(record, {1: {"module_bay_templates": [self._bay("FPC 0", [])]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_server_that_never_returned_the_field_omits_the_key(self): + """Below 4.7 the relation is not selected, and absent must not become empty.""" + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = _dotdict(name="FPC 0", position=None, label="", description="") + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_module_type_exports_the_classes_it_belongs_to(self): record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, + id=5, + model="JNP304-RE", + manufacturer=_make_mfr(name="Juniper", slug="juniper"), part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + module_bay_types=[_dotdict(id=9, name="MX304-RE")], ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert "rear_port_position" not in result["front-ports"][0] - def test_front_port_multiple_mappings_warns_and_uses_first(self): - """When a front port has >1 mappings a UserWarning is raised and only the first is used.""" - import warnings - from types import SimpleNamespace + assert serialize_module_type(record, {})["module_bay_types"] == ["MX304-RE"] - m1 = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - m2 = SimpleNamespace(rear_port=SimpleNamespace(name="RP2"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[m1, m2]) + def test_a_device_type_bay_exports_its_classes_too(self): record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + id=2, model="MX304", slug="mx304", manufacturer=_make_mfr(), u_height=None, is_full_depth=None ) - components = {1: {"front_port_templates": [fp]}} - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert len(caught) == 1 - assert issubclass(caught[0].category, UserWarning) - assert "FP1" in str(caught[0].message) - assert "2 mappings" in str(caught[0].message) - assert "issue #78" in str(caught[0].message) - - def test_front_port_legacy_rear_port_scalars(self): - """pre-4.5 NetBox: record has rear_port/rear_port_position as direct attrs (no mappings).""" + bay = self._bay("RE0", [_dotdict(id=9, name="MX304-RE")]) + + result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] + + +class TestPortMappingsStanza: + """Export writes the NetBox 4.5 port-mappings stanza the DTL schema now requires.""" + + @staticmethod + def _mapping(rear_port, rear_position=1, front_position=1): from types import SimpleNamespace - fp = SimpleNamespace( - name="FP1", - type="8p8c", + return SimpleNamespace( + rear_port=SimpleNamespace(name=rear_port), + rear_port_position=rear_position, + front_port_position=front_position, + ) + + @staticmethod + def _front_port(name, mappings=None, positions=1, **extra): + from types import SimpleNamespace + + return SimpleNamespace( + name=name, + type="lc-upc", label="", description="", color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=3, + positions=positions, + mappings=mappings or [], + **extra, ) - record = _dotdict( + + def _device(self): + return _dotdict( id=1, model="X", slug="acme-x", @@ -590,13 +603,63 @@ def test_front_port_legacy_rear_port_scalars(self): front_image=None, rear_image=None, ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert result["front-ports"][0]["rear_port_position"] == 3 - def test_front_port_legacy_rear_port_position_1_omitted(self): - """pre-4.5: rear_port_position == 1 should be omitted (same as mappings path).""" + def test_a_front_port_carries_positions_and_no_inline_rear_port(self): + """The schema dropped rear_port from front-port entries and made positions required.""" + fp = self._front_port("FP1", [self._mapping("RP1")], positions=1) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "lc-upc", "positions": 1}] + + def test_every_mapping_reaches_the_stanza_not_just_the_first(self): + """The issue: a crossover front port mapped to two rear ports lost the second.""" + fp = self._front_port("FP1", [self._mapping("RP1", 1), self._mapping("RP2", 3, front_position=2)], positions=2) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 1}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2", "rear_port_position": 3}, + ] + + def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): + """The shape the library actually carries: many front ports onto one MPO rear port.""" + ports = [self._front_port(f"FP{i}", [self._mapping("MPO1", i)]) for i in (1, 2, 3)] + + result = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert [(m["front_port"], m["rear_port_position"]) for m in result["port-mappings"]] == [ + ("FP1", 1), + ("FP2", 2), + ("FP3", 3), + ] + + def test_mappings_sort_by_numeric_positions_then_rear_port_name(self): + import yaml + + fp = self._front_port( + "FP1", + [ + self._mapping("RP2", "10.0", "2.0"), + self._mapping("RP2", 2, 2), + self._mapping("RP1", "2.0", 2), + self._mapping("RP1", None, None), + ], + positions=2, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == yaml.safe_load(""" +- {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} +- {front_port: FP1, front_port_position: 2, rear_port: RP1, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 10} +""") + + def test_a_pre_45_server_still_exports_its_mappings(self): + """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" from types import SimpleNamespace fp = SimpleNamespace( @@ -605,75 +668,48 @@ def test_front_port_legacy_rear_port_position_1_omitted(self): label="", description="", color="", - mappings=None, rear_port=SimpleNamespace(name="RP1"), - rear_port_position=1, + rear_port_position=4, ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] - def test_components_sorted_by_name(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} + ] + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_legacy_front_port_carries_the_schema_required_positions(self): + """The schema requires positions, but it arrived in 4.5, so a pre-4.5 record needs the default.""" from types import SimpleNamespace - iface_z = SimpleNamespace( - name="eth9", - type="1000base-t", - label="", - description="", - mgmt_only=False, - enabled=True, - poe_mode=None, - poe_type=None, - rf_role=None, - ) - iface_a = SimpleNamespace( - name="eth0", - type="1000base-t", - label="", - description="", - mgmt_only=False, - enabled=True, - poe_mode=None, - poe_type=None, - rf_role=None, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"interface_templates": [iface_z, iface_a]}} - result = serialize_device_type(record, components) - names = [i["name"] for i in result["interfaces"]] - assert names == sorted(names) + fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_front_port_with_no_mapping_adds_no_stanza(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) + + assert "port-mappings" not in result + + def test_a_type_without_front_ports_adds_no_stanza(self): + assert "port-mappings" not in serialize_device_type(self._device(), {1: {}}) + + def test_the_importer_reads_back_what_the_export_wrote(self): + """Serializer to normalizer, both real: the stanza is the seam between them.""" + from core.repo import normalize_port_mappings + + ports = [ + self._front_port("1", [self._mapping("MPO1", 1)]), + self._front_port("2", [self._mapping("MPO1", 2)]), + ] + exported = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert normalize_port_mappings(exported) is None + assert [fp["_mappings"] for fp in exported["front-ports"]] == [ + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 1}], + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 2}], + ] + assert "port-mappings" not in exported, "the normalizer consumes the stanza" diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 3f7b3315..a2f61af4 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -2,22 +2,23 @@ import threading from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest +from helpers import paginate_dispatch, recording_handle from requests.exceptions import ConnectionError as RequestsConnectionError -from unittest.mock import MagicMock, patch + from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES -from core.outcomes import EntityKind, Outcome from core.netbox_api import ( + DeviceTypes, NetBox, NetBoxError, - DeviceTypes, SSLVerificationError, _delete_image_attachment, _FrontPortRecordWithMappings, ) -from helpers import paginate_dispatch, recording_handle - +from core.outcomes import EntityKind, Outcome # All component list keys used by the GraphQL client for empty-response fallback. _ALL_COMPONENT_KEYS = [component.list_key for component in COMPONENT_TYPES] @@ -80,7 +81,6 @@ def _factory(nb_api=None, handle=None, counter=None, **kwargs): handle if handle is not None else mock_handle, counter if counter is not None else MagicMock(), False, - False, graphql=kwargs.pop("graphql", graphql_client), repo_path=kwargs.pop("repo_path", mock_settings.repo_path), **kwargs, @@ -112,13 +112,12 @@ def _mark_cache_ready(device_types): def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): # Mock api call - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.url == "http://mock-netbox" assert nb.token == "mock-token" # Verify module support detection - assert nb.modules def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, mock_handle): @@ -128,7 +127,7 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc remove_unmanaged_types=True, verify_images=True, ) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" netbox = NetBox(config, mock_handle) @@ -138,24 +137,30 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): - # Test 5.0 - mock_pynetbox.api.return_value.version = "5.0" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters + """Every supported release uses the new filter names; 4.7 adds module bay types.""" + for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): + mock_pynetbox.api.return_value.version = version + nb = NetBox(mock_settings, mock_handle) + assert nb.module_bay_types is module_bay_types, version - # Test 4.0 - mock_pynetbox.api.return_value.version = "4.0" - nb = NetBox(mock_settings, mock_handle) - assert not nb.new_filters - # Test 4.1 - mock_pynetbox.api.return_value.version = "4.1" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters +def test_the_netbox_version_is_fetched_once(mock_settings, mock_pynetbox, mock_handle): + """Each pynetbox `version` access is another HTTP request, and only the first is error-mapped.""" + from unittest.mock import PropertyMock + + api_type = type(mock_pynetbox.api.return_value) + version = PropertyMock(return_value="4.7.0") + api_type.version = version + try: + NetBox(mock_settings, mock_handle) + + assert version.call_count == 1, "the log lines must reuse the version already fetched" + finally: + del api_type.version def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] nb = NetBox(mock_settings, mock_handle) @@ -168,7 +173,7 @@ def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): def test_create_manufacturers_no_new_is_verbose_only(mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -233,7 +238,7 @@ def test_create_generic_counts_the_created_list_at_the_caller(mock_pynetbox, mak def test_redundant_image_upload(mock_settings, mock_pynetbox, mock_handle): # Setup # Ensure modules check doesn't fail - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = MagicMock() @@ -514,6 +519,41 @@ def test_update_components_legacy_mapping_two_tuple_warns_and_skips( assert any("NetBox < 4.5" in str(c) for c in mock_handle.log.call_args_list) +def test_update_components_legacy_truncation_is_reported( + mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle +): + """The create path says "only first mapping applied"; the update path said nothing at all.""" + from core.change_detector import ChangeType, ComponentChange, PropertyChange + + mock_nb_api = MagicMock() + dt = make_device_types(nb_api=mock_nb_api) + dt.m2m_front_ports = False + + existing_fp = MagicMock(id=10, name="FP1") + rp1 = MagicMock(id=21, name="RP1") + rp2 = MagicMock(id=22, name="RP2") + dt.components.record("front_port_templates", "device", 1, {"FP1": existing_fp}) + dt.components.record("rear_port_templates", "device", 1, {"RP1": rp1, "RP2": rp2}) + + # Two mappings: the legacy model can hold only one of them. + new_mappings_set = frozenset({("RP1", 1, 1), ("RP2", 2, 1)}) + changes = [ + ComponentChange( + component_type="front-ports", + component_name="FP1", + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange("_mappings", frozenset(), new_mappings_set)], + ), + ] + + mock_handle.log.reset_mock() + dt.update_components({}, 1, changes, parent_type="device") + + logged = " ".join(str(c) for c in mock_handle.log.call_args_list) + assert "FP1" in logged, f"truncation must name the port, got: {logged}" + assert "4.5" in logged, f"truncation must say why, got: {logged}" + + def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( mock_settings, mock_pynetbox, graphql_client, make_device_types ): @@ -611,7 +651,7 @@ class TestNetBoxConnectApi: def test_ssl_ignore_sets_verify_false(self, mock_pynetbox, mock_handle, make_config): mock_settings = make_config(ignore_ssl_errors=True) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.netbox.http_session.verify is False @@ -622,7 +662,7 @@ class TestCreateManufacturersError: def test_request_error_logged(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" # Make pynetbox.RequestError in the module under test be the real exception class mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -766,8 +806,10 @@ def test_per_item_error_reports_every_failing_item( messages = [call.args[0] for call in mock_handle.log.call_args_list] assert len(messages) == 2 - assert "eth0" in messages[0] and "This field must be unique." in messages[0] - assert "eth2" in messages[1] and "Invalid choice." in messages[1] + assert "eth0" in messages[0] + assert "This field must be unique." in messages[0] + assert "eth2" in messages[1] + assert "Invalid choice." in messages[1] def test_non_list_error_logs_failed_items( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle @@ -930,7 +972,7 @@ def test_creates_new_device_type_with_components( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) @@ -1013,7 +1055,7 @@ def test_empty_src_returns_none(self): assert _image_dir_for_yaml("", "device-types", "elevation-images") is None def test_unknown_src_returns_none(self): - from core.netbox_api import _image_dir_for_yaml, _UNKNOWN_SRC + from core.netbox_api import _UNKNOWN_SRC, _image_dir_for_yaml assert _image_dir_for_yaml(_UNKNOWN_SRC, "device-types", "elevation-images") is None @@ -1118,14 +1160,14 @@ class TestCreateModuleTypes: """Tests for TestCreateModuleTypes.""" def test_empty_module_types_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) # Should not raise and should not call create nb.create_module_types([]) nb.netbox.dcim.module_types.create.assert_not_called() def test_creates_new_module_type(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1250,7 +1292,7 @@ class TestCreateModuleTypesBody: """Tests for TestCreateModuleTypesBody.""" def test_cached_module_type_skips_creation(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1285,7 +1327,7 @@ def test_create_module_type_request_error_logged( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1310,7 +1352,7 @@ def test_create_module_type_request_error_logged( def test_creates_module_type_with_components( self, mock_settings, mock_pynetbox, mock_graphql_requests, graphql_client, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1606,7 +1648,7 @@ class TestCreateManufacturersSuccessLog: def test_verbose_log_per_created_manufacturer(self, mock_settings, mock_pynetbox, mock_handle): """verbose_log should be called for each created manufacturer.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_m = MagicMock() @@ -1881,6 +1923,7 @@ def test_update_property_change_request_error_logged( "Device Type Updated" log MUST NOT be emitted. """ import pynetbox as real_pynb2 + from core.change_detector import ChangeReport, DeviceTypeChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -1951,6 +1994,7 @@ def _build_subdevice_role_flip_setup( report zero dependent devices and one blocking device-bay template. """ import pynetbox as real_pynb2 + from core.change_detector import ChangeReport, DeviceTypeChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -1984,8 +2028,10 @@ def _build_subdevice_role_flip_setup( # Force the .error property to return the parsed dict (pynetbox usually does this). err.error = { "subdevice_role": [ - "Must delete all device bay templates associated with this device " - "before declassifying it as a parent device." + ( + "Must delete all device bay templates associated with this device " + "before declassifying it as a parent device." + ) ] } @@ -2009,7 +2055,7 @@ def test_constraint_failure_logs_hint_when_flag_off( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """Without --force-resolve-conflicts, classifier hint is logged but no auto-resolve runs.""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=False ) mock_nb_api.dcim.device_types.update.side_effect = err @@ -2036,13 +2082,14 @@ def test_constraint_failure_logs_hint_when_flag_off( assert failures[0].outcome == Outcome.FAILED assert "SuperServer" in failures[0].identity assert "module-bay-1" in failures[0].blocking_objects - assert failures[0].hint and "--force-resolve-conflicts" in failures[0].hint + assert failures[0].hint + assert "--force-resolve-conflicts" in failures[0].hint def test_constraint_failure_auto_resolves_when_flag_on_and_safe( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """With flag on + zero dependents, blocking templates are deleted and PATCH retried.""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) # First update call fails; second (after auto-resolve) succeeds. @@ -2063,7 +2110,7 @@ def test_constraint_failure_auto_resolve_retry_still_fails( """If the retried PATCH after auto-resolve still fails, count it as a failure exactly once.""" import pynetbox as real_pynb2 - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) err2 = real_pynb2.RequestError(MagicMock(status_code=500, content=b'{"detail":"still bad"}')) @@ -2084,7 +2131,7 @@ def test_constraint_failure_blocked_when_devices_in_use( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """With flag on but live devices reference the type, no remediation runs (safety gate).""" - nb, dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( + nb, _dt, mock_nb_api, report, device_type, blocking_template, err = self._build_subdevice_role_flip_setup( mock_settings, mock_handle, mock_pynetbox, make_device_types, force=True ) live_device = MagicMock() @@ -2333,7 +2380,7 @@ def test_create_failure_reaches_the_end_of_run_failure_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b"{\"manufacturer\":[\"Related object not found using the provided attributes: {'slug': 'ribbon'}\"]}" ) @@ -2487,7 +2534,6 @@ def test_creates_all_component_types( nb = NetBox(mock_settings, mock_handle) nb.device_types = dt - nb.modules = True created_dt = MagicMock() created_dt.id = 1 @@ -2574,7 +2620,7 @@ class TestFilterActionableModuleTypesEdge: def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """Empty module_types list returns [], {} immediately.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) result, images, _ = nb.filter_actionable_module_types([], {}, only_new=False) assert result == [] @@ -2582,7 +2628,7 @@ def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mo def test_only_new_delegates_to_filter_new(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """only_new=True returns only genuinely new module types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() all_mts = {"cisco": {"LC": existing_mt}} @@ -2599,7 +2645,7 @@ def test_new_module_type_added_to_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """Module type not in all_module_types is added to actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2622,7 +2668,7 @@ def test_existing_module_with_new_image_is_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """Existing module type with an image not yet in NetBox is actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2655,14 +2701,14 @@ def test_existing_module_with_new_image_is_actionable( } # existing images are empty → the image is "new" → actionable - with patch.object(nb, "_fetch_module_type_existing_images", return_value={42: set()}): - with patch( + with ( + patch.object(nb, "_fetch_module_type_existing_images", return_value={42: set()}), + patch( "core.netbox_api.NetBox._discover_module_image_files", return_value=[str(img)], - ): - result, existing_images_map, _ = nb.filter_actionable_module_types( - [module_type], all_mts, only_new=False - ) + ), + ): + result, existing_images_map, _ = nb.filter_actionable_module_types([module_type], all_mts, only_new=False) assert result == [module_type] # Verify the upload worklist is propagated so create_module_types can upload the image. assert existing_images_map == {42: set()} @@ -2673,7 +2719,7 @@ def test_existing_module_with_changed_property_is_actionable( """Existing module type with a changed scalar property (e.g. part_number) is actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2726,7 +2772,7 @@ def test_existing_module_with_missing_image_and_property_change_logs_both( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2782,7 +2828,7 @@ def test_existing_module_with_only_missing_image_is_actionable_but_not_logged( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2827,7 +2873,7 @@ def test_existing_module_with_unchanged_property_is_not_actionable( """Existing module type whose properties all match NetBox is not actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2868,7 +2914,7 @@ class TestCreateModuleTypesEdge: def test_existing_module_type_verbose_logged(self, mock_settings, mock_pynetbox, mock_handle): """When a module type already exists, verbose_log is called with 'Cached'.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() @@ -2894,7 +2940,7 @@ def test_only_new_skips_existing_module_component_creation( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """only_new=True + existing module → skip component creation.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("interface_templates", "module", 5, {}) @@ -2923,7 +2969,7 @@ def test_creates_module_type_with_power_outlets_console_server_ports_front_ports self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """power-outlets, console-server-ports, front-ports branches in create_module_types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) for endpoint_name in ("power_outlet_templates", "console_server_port_templates", "front_port_templates"): @@ -2955,7 +3001,7 @@ def test_a_new_module_type_gets_its_module_bays( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The DTL module-type schema allows module-bays, so creation must not skip them.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("module_bay_templates", "module", 5, {}) @@ -2981,7 +3027,7 @@ def test_existing_module_type_property_update_calls_api(self, mock_settings, moc """Existing module type with changed part_number calls module_types.update and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3017,7 +3063,7 @@ def test_existing_module_type_property_unchanged_no_api_call(self, mock_settings """Existing module type with matching part_number does not call module_types.update.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3048,7 +3094,7 @@ def test_existing_module_type_only_new_skips_property_update(self, mock_settings """only_new=True skips property update even when part_number differs.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3082,7 +3128,7 @@ def test_existing_module_type_component_update_calls_update_components( """Existing module type with changed component property calls update_components and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3146,7 +3192,7 @@ def test_existing_module_type_property_and_component_update_increments_once( """Both property and component change → module_updated incremented only once.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3214,7 +3260,7 @@ def test_existing_module_type_removal_only_no_counter_increment( """COMPONENT_REMOVED-only changes call update_components but do NOT increment module_updated.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3275,7 +3321,7 @@ def test_property_update_plus_removal_only_remove_false_counts_as_updated( """Properties changed + removal-only diff with remove_components=False → module_updated incremented.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.update_components = MagicMock() @@ -3358,7 +3404,7 @@ def test_existing_image_is_skipped( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """If the image name is already in module_type_existing_images, upload is skipped.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -3388,7 +3434,7 @@ def test_new_image_is_uploaded_and_tracked( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """When image is not yet in existing_images, upload_image_attachment is called.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -3495,6 +3541,7 @@ def test_property_update_success_counter_incremented( ): """Successful property update increments components_updated counter.""" from collections import Counter as _Counter + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_nb_api = mock_pynetbox.api.return_value @@ -3523,6 +3570,7 @@ def test_property_update_request_error_logged( ): """RequestError during property update is caught and logged.""" import pynetbox as real_pynb2 + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -3656,6 +3704,7 @@ def test_request_error_on_delete_is_logged( ): """RequestError during component deletion is caught and logged.""" import pynetbox as real_pynb2 + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb2.RequestError @@ -4061,12 +4110,11 @@ def test_file_close_exception_swallowed( fake_fh = MagicMock() fake_fh.close.side_effect = OSError("cannot close") - with patch("builtins.open", return_value=fake_fh): - with patch("core.netbox_api.requests") as mock_req: - mock_req.RequestException = _req2.RequestException - mock_req.patch.side_effect = _req2.RequestException("server error") - # Should NOT raise despite close() raising - dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) + with patch("builtins.open", return_value=fake_fh), patch("core.netbox_api.requests") as mock_req: + mock_req.RequestException = _req2.RequestException + mock_req.patch.side_effect = _req2.RequestException("server error") + # Should NOT raise despite close() raising + dt.upload_images("http://nb", "token", {"front_image": str(img)}, 1) # The RequestException log should still have been called assert mock_handle.log.called @@ -4182,43 +4230,9 @@ def test_image_file_not_found_logs_error( nb.create_device_types([device_type]) assert any("Error locating image file" in str(c) for c in mock_handle.log.call_args_list) - def test_module_bays_not_created_when_modules_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle - ): - """module-bays are only created when self.modules is True.""" - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - dt.existing_device_types = {} - dt.existing_device_types_by_slug = {} - dt.components.record("module_bay_templates", "device", 1, {}) - - nb = NetBox(mock_settings, mock_handle) - nb.device_types = dt - nb.modules = False # explicitly disabled - - created_dt = MagicMock() - created_dt.id = 1 - created_dt.manufacturer.name = "Cisco" - created_dt.model = "TestSwitch" - mock_nb_api.dcim.device_types.create.return_value = created_dt - - device_type = { - "manufacturer": {"slug": "cisco"}, - "model": "TestSwitch", - "slug": "testswitch", - "module-bays": [{"name": "MB1"}], - "src": "/tmp/device-types/cisco/testswitch.yaml", - } - nb.create_device_types([device_type]) - mock_nb_api.dcim.module_bay_templates.create.assert_not_called() - - -class TestCreateModuleTypesCornerCases: - """Corner-case tests for create_module_types (cognitive complexity 16).""" - def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_mt = MagicMock() @@ -4252,7 +4266,7 @@ def tracking_iter(): def test_all_module_types_fetched_when_none(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """all_module_types is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4277,7 +4291,7 @@ def test_module_type_existing_images_fetched_when_none( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """module_type_existing_images is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4421,7 +4435,7 @@ class TestGetExistingRackTypes: def test_delegates_to_graphql(self, mock_settings, mock_pynetbox, graphql_client, mock_handle): """get_existing_rack_types() returns whatever graphql.get_rack_types() returns.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.graphql = graphql_client expected = {"apc": {"AR1300": MagicMock()}} @@ -4442,7 +4456,7 @@ class TestCreateRackTypes: """Tests for NetBox.create_rack_types().""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_empty_list_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): @@ -4510,7 +4524,7 @@ def test_existing_rack_type_fields_differ_calls_update(self, mock_settings, mock def test_new_rack_type_calls_create(self, mock_settings, mock_pynetbox, mock_handle): """Non-existing rack type: create called, counter incremented, added to cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 99 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4532,7 +4546,7 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet """RequestError during create is logged; processing continues.""" import pynetbox - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4551,9 +4565,10 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynetbox, mock_handle): """RequestError during update is logged; processing continues.""" import pynetbox + from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4573,7 +4588,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, mock_handle): """When all_rack_types=None, get_existing_rack_types() is called to populate the cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) nb.get_existing_rack_types = MagicMock(return_value={}) rack_type = { @@ -4587,7 +4602,7 @@ def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When a progress wrapper is provided, it is used as the iterator.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 1 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4613,49 +4628,47 @@ class TestVerifyCompatibility: """Tests for NetBox.verify_compatibility() version thresholds.""" @pytest.mark.parametrize( - "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", + ("version_str", "expected_m2m", "expected_module_bay_types"), [ - ("3.1", False, False, False, False), - ("3.2", True, False, False, False), - ("4.0", True, False, False, False), - ("4.1", True, True, True, False), - ("4.4", True, True, True, False), - ("4.5", True, True, True, True), - ("4.6", True, True, True, True), + ("4.3", False, False), + ("4.4", False, False), + ("4.5", True, False), + ("4.6", True, False), + ("4.7", True, True), + ("5.0", True, True), # Version strings with non-numeric suffixes - ("4.5-beta", True, True, True, True), - ("4.1.0", True, True, True, False), + ("4.5-beta", True, False), + ("4.3.0", False, False), ], ) def test_version_thresholds( self, version_str, - expected_modules, - expected_new_filters, - expected_rack_types, expected_m2m, + expected_module_bay_types, mock_settings, mock_pynetbox, mock_handle, ): + """Only the flags that still vary above the 4.3 floor are set.""" mock_pynetbox.api.return_value.version = version_str nb = NetBox(mock_settings, mock_handle) - assert nb.modules == expected_modules, f"modules mismatch for {version_str}" - assert nb.new_filters == expected_new_filters, f"new_filters mismatch for {version_str}" - assert nb.rack_types == expected_rack_types, f"rack_types mismatch for {version_str}" assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" + assert nb.module_bay_types == expected_module_bay_types, f"module_bay_types mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): - """Version string with only major component (e.g. '4') does not crash.""" - mock_pynetbox.api.return_value.version = "4" + """A version string with only a major component (e.g. '5') does not crash.""" + mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is False # 4.0 → no new filters + assert nb.m2m_front_ports is True - def test_version_42_enables_new_filters_not_m2m(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "4.2" + def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( + self, mock_settings, mock_pynetbox, mock_handle + ): + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True assert nb.m2m_front_ports is False + assert nb.module_bay_types is False # ============================================================ @@ -4855,7 +4868,7 @@ class TestLogModuleTypeChanges: def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, mock_handle): """A non-empty changed_property_log triggers verbose logging.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4868,7 +4881,7 @@ def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, def test_empty_log_emits_nothing(self, mock_settings, mock_pynetbox, mock_handle): """An empty changed_property_log does not trigger any logging calls.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4887,7 +4900,7 @@ class TestTryUpdateModuleTypeErrors: """Tests for RequestError and retryable-exception handlers in _try_update_module_type.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def _make_module_type_res(self): @@ -4901,7 +4914,7 @@ def test_request_error_returns_false_and_logs(self, mock_settings, mock_pynetbox """pynetbox.RequestError during update causes (False, False) return and log.""" import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) @@ -4926,7 +4939,7 @@ def test_retryable_exception_returns_false_and_logs(self, mock_settings, mock_py import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) mock_handle.log.reset_mock() @@ -4969,7 +4982,7 @@ def test_retryable_exception_on_create_returns_false( import requests mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -5020,6 +5033,7 @@ def test_load_for_vendor_populates_existing_device_types( ): """load_for_vendor populates existing_device_types and existing_device_types_by_slug.""" from unittest.mock import patch as _patch + from core.graphql_client import DotDict mock_nb_api = mock_pynetbox.api.return_value @@ -5052,6 +5066,7 @@ def test_load_for_vendor_replaces_prior_data( ): """A second call to load_for_vendor replaces data from the first call.""" from unittest.mock import patch as _patch + from core.graphql_client import DotDict mock_nb_api = mock_pynetbox.api.return_value @@ -5083,6 +5098,7 @@ def test_load_for_vendor_resets_state_before_fetch_on_failure( ): """State is reset before the fetch so a raised exception leaves a clean slate.""" from unittest.mock import patch as _patch + import pytest mock_nb_api = mock_pynetbox.api.return_value @@ -5304,6 +5320,7 @@ def test_returns_false_when_no_cache_entry(self, tmp_path): def test_returns_false_when_hash_matches(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed data = b"unchanged_content" @@ -5314,6 +5331,7 @@ def test_returns_false_when_hash_matches(self, tmp_path): def test_returns_true_when_hash_differs(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed img = tmp_path / "front.png" @@ -5323,6 +5341,7 @@ def test_returns_true_when_hash_differs(self, tmp_path): def test_returns_false_on_file_read_error(self, tmp_path): import hashlib + from core.netbox_api import _is_image_hash_changed path = str(tmp_path / "missing.png") @@ -5378,9 +5397,8 @@ def test_verify_images_reuploads_missing_image( mock_resp = MagicMock() mock_resp.ok = False # image missing on server → "missing" - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + nb.create_device_types([device_type]) nb.device_types.upload_images.assert_called_once() @@ -5463,9 +5481,8 @@ def test_verify_images_skips_ok_image( mock_resp = MagicMock() mock_resp.ok = True # image accessible on server mock_resp.headers = {"Content-Type": "image/png"} - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + nb.create_device_types([device_type]) nb.device_types.upload_images.assert_not_called() @@ -5500,9 +5517,8 @@ def test_default_mode_still_skips_without_http( "src": str(dev_types_dir / "ap.yaml"), } - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get") as mock_get: - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get") as mock_get: + nb.create_device_types([device_type]) mock_get.assert_not_called() nb.device_types.upload_images.assert_not_called() @@ -5546,10 +5562,9 @@ def test_verify_ok_seeds_hash_cache( mock_resp = MagicMock() mock_resp.ok = True mock_resp.headers = {"Content-Type": "image/png"} - with patch("glob.glob", return_value=[str(img)]): - with patch("requests.get", return_value=mock_resp): - with patch("core.netbox_api._save_image_hash_cache") as mock_save: - nb.create_device_types([device_type]) + with patch("glob.glob", return_value=[str(img)]), patch("requests.get", return_value=mock_resp): + with patch("core.netbox_api._save_image_hash_cache") as mock_save: + nb.create_device_types([device_type]) # Hash cache must have been seeded with the local file's hash assert str(img) in nb._image_hash_cache @@ -5595,6 +5610,7 @@ def test_delete_image_attachment_success(self, mock_settings, mock_handle): def test_delete_image_attachment_logs_request_errors(self, mock_settings, mock_handle): import requests + from core.netbox_api import _delete_image_attachment with patch("core.netbox_api.requests.delete", side_effect=requests.RequestException("boom")): @@ -5613,6 +5629,7 @@ def test_load_module_type_properties_falls_back_without_a_schema(self, tmp_path) def test_fmt_connection_error_contains_url_and_hint(self): """_fmt_connection_error returns a message with the URL and a reachability hint.""" import requests as _requests + from core.netbox_api import _fmt_connection_error url = "http://netbox.example.com" @@ -5624,9 +5641,10 @@ def test_fmt_connection_error_contains_url_and_hint(self): def test_fmt_connection_error_verify_compatibility_uses_it(self, mock_settings, mock_pynetbox, mock_handle): """verify_compatibility uses _fmt_connection_error for ConnectionError.""" - import requests as _requests from unittest.mock import PropertyMock + import requests as _requests + type(mock_pynetbox.api.return_value).version = PropertyMock( side_effect=_requests.exceptions.ConnectionError("drop") ) @@ -5635,7 +5653,8 @@ def test_fmt_connection_error_verify_compatibility_uses_it(self, mock_settings, NetBox(mock_settings, mock_handle) exc_msg = str(exc_info.value.args[0]) if exc_info.value.args else "" - assert mock_settings.netbox_url in exc_msg and "Connection" in exc_msg + assert mock_settings.netbox_url in exc_msg + assert "Connection" in exc_msg def test_check_image_url_reports_the_transport_error_when_log_fn_provided(self): """The wire detail goes to log_fn; the verdict goes to the return value.""" @@ -5699,7 +5718,8 @@ def test_a_corrupt_cache_file_is_reported_by_path(self, mock_settings, mock_pyne assert nb._image_hash_cache == {} logged = " ".join(str(call) for call in mock_handle.log.call_args_list) - assert str(path) in logged and "unreadable image hash cache" in logged + assert str(path) in logged + assert "unreadable image hash cache" in logged def test_a_cache_file_holding_the_wrong_shape_is_reported( self, mock_settings, mock_pynetbox, mock_handle, tmp_path @@ -5777,7 +5797,7 @@ class TestUploadModuleTypeImagesVerify: """Tests for _upload_module_type_images with verify_images=True.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types.upload_image_attachment = MagicMock(return_value=True) nb.verify_images = True @@ -5936,7 +5956,7 @@ def test_load_image_hash_cache_returns_empty_dict_on_bad_json(self, tmp_path): def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_settings, mock_pynetbox, mock_handle): from core.graphql_client import GraphQLError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with patch.object(NetBox, "get_manufacturers", side_effect=GraphQLError("bad query")): with pytest.raises(NetBoxError, match="GraphQL error: bad query"): @@ -5945,7 +5965,7 @@ def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_setti def test_init_raises_typed_error_when_device_types_initialization_fails( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with ( patch.object(NetBox, "get_manufacturers", return_value=[]), @@ -6022,7 +6042,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.manufacturers.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6032,7 +6052,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock assert any("Connection error creating manufacturers" in str(c) for c in mock_handle.log.call_args_list) def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") @@ -6047,9 +6067,10 @@ def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pynetbox, mock_handle): from types import SimpleNamespace + from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") resolution = SimpleNamespace( @@ -6070,9 +6091,10 @@ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pyn def test_try_resolve_update_logs_auto_resolve_failure(self, mock_settings, mock_pynetbox, mock_handle): from types import SimpleNamespace + from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6099,13 +6121,15 @@ def boom(): def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( self, mock_settings, mock_pynetbox, mock_handle ): + from types import SimpleNamespace + import pynetbox as real_pynb import requests - from types import SimpleNamespace + from core.update_failure_resolver import FailureKind mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6132,7 +6156,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( def test_log_device_type_change_outcome_partial_success_mentions_property_failure( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6151,7 +6175,7 @@ def test_log_device_type_change_outcome_partial_success_mentions_property_failur def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6168,7 +6192,7 @@ def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( assert any("Device Type Cached" in str(c) for c in mock_handle.verbose_log.call_args_list) def test_filter_images_for_upload_keeps_changed_image(self, mock_settings, mock_pynetbox, tmp_path, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True @@ -6191,10 +6215,11 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( ): import pynetbox as real_pynb import requests + from core.change_detector import PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.update.side_effect = requests.exceptions.ConnectionError("offline") nb._log_device_type_change_outcome = MagicMock() @@ -6217,7 +6242,7 @@ def test_create_new_device_type_logs_retryable_error(self, mock_settings, mock_p import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6238,7 +6263,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( ): from core.change_detector import ChangeType, ComponentChange, PropertyChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) changes = [ ComponentChange("interfaces", "xe-0", ChangeType.COMPONENT_ADDED), @@ -6261,7 +6286,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True details = {7: {"linecard.front": {"att_id": 5, "url": "/media/linecard.front.jpg"}}} @@ -6273,7 +6298,7 @@ def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( assert nb._module_image_details == details def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(spec=["id", "manufacturer", "model"]) module_type_res.id = 1 @@ -6290,7 +6315,7 @@ def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, def test_filter_actionable_module_types_marks_verify_images_module_actionable( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True _mark_cache_ready(nb.device_types) @@ -6314,7 +6339,7 @@ class TestAdditionalModuleTypeCoverage: def test_apply_module_type_component_updates_records_failed_no_actionable_changes( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6334,7 +6359,7 @@ def test_apply_module_type_component_updates_marks_partial_on_partial_component_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6362,7 +6387,7 @@ def test_apply_module_type_component_updates_marks_partial_when_properties_only_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6383,7 +6408,7 @@ def test_apply_module_type_component_updates_records_failed_when_no_changes_appl ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6431,6 +6456,7 @@ def test_apply_updates_for_type_logs_retryable_exception( ): import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange, PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -6457,6 +6483,7 @@ def test_remove_components_logs_retryable_exception( ): import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb.RequestError @@ -6507,10 +6534,11 @@ class TestRemainingCoverageBranches: def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb import requests + from core.graphql_client import DotDict mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6530,7 +6558,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6547,7 +6575,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock def test_upload_module_type_images_discards_missing_attachment_before_failed_upload( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=False) @@ -6575,7 +6603,7 @@ def test_upload_module_type_images_discards_missing_attachment_before_failed_upl def test_upload_module_type_images_skips_changed_image_when_delete_fails( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=True) @@ -6814,7 +6842,6 @@ def _device_types(url, handle): handle, MagicMock(), False, - False, graphql=NetBoxGraphQLClient(url, "token", page_size=10), repo_path="/tmp/repo", max_threads=2, @@ -6825,7 +6852,7 @@ def test_filter_actionable_module_types_skips_unchanged_existing_module( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6874,7 +6901,7 @@ def test_filter_actionable_module_types_includes_module_with_missing_component( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6922,7 +6949,7 @@ def test_missing_netbox_field_is_not_treated_as_change( ): """When existing module lacks an attribute, it's skipped — no false positive change.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6979,7 +7006,7 @@ def test_remove_components_is_called_when_flag_set( ): """When remove_components=True and there are component changes, remove_components is called.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7037,7 +7064,7 @@ def test_component_reconciliation_continues_when_scalar_patch_fails( skipped entirely. """ mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7106,7 +7133,7 @@ def _netbox(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_rack_type_create_failure_is_reported(self, mock_settings, mock_pynetbox, mock_handle): @@ -7172,13 +7199,13 @@ class TestSummaryWordingMatchesTheFailedOperation: def _summary_text(self, nb): """Render the real end-of-run summary for *nb* and return it as one string.""" - from datetime import datetime + from datetime import UTC, datetime from types import SimpleNamespace from core.import_run import RunSummary, _log_run_summary handle, console = recording_handle() - summary = RunSummary.capture(nb, SimpleNamespace(duplicate_definitions=[]), datetime.now()) + summary = RunSummary.capture(nb, SimpleNamespace(duplicate_definitions=[]), datetime.now(UTC)) _log_run_summary(handle, summary) return "\n".join(console.lines) @@ -7190,7 +7217,7 @@ def test_device_type_create_failure_is_not_called_an_update_failure( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b'{"manufacturer":["Related object not found using the provided attributes: slug ribbon"]}' ) @@ -7223,12 +7250,11 @@ def test_module_type_create_failure_is_not_called_an_update_failure( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.module_types.create.side_effect = _request_error( b'{"model":["This field may not be blank."]}' ) nb = NetBox(mock_settings, mock_handle) - nb.modules = True nb._process_single_module_type( {"manufacturer": {"slug": "panduit"}, "model": "FAP6WBUSC", "slug": "fap6wbusc"}, "/repo/module-types/Panduit/FAP6WBUSC.yaml", @@ -7247,20 +7273,20 @@ class TestSkippedComponentReasonReachesTheReport: def _dt(self, mock_pynetbox, make_device_types, parent_id=1): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" - dt = make_device_types(nb_api=mock_nb_api) - return dt + mock_nb_api.version = "4.3" + return make_device_types(nb_api=mock_nb_api) def test_unresolvable_power_port_reaches_the_report( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The Powerman case: an outlet dropped for a bad power_port must name the reason.""" import pynetbox as real_pynb + from core.change_detector import ChangeReport, ChangeType, ComponentChange, DeviceTypeChange mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) inlet = MagicMock() @@ -7341,6 +7367,7 @@ def test_component_update_transport_failure_reports_the_transport_error( """A retry-exhausted update must name the transport error, not the generic label.""" import pynetbox as real_pynb import requests + from core.change_detector import ( ChangeReport, ChangeType, @@ -7351,7 +7378,7 @@ def test_component_update_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) existing_iface = MagicMock() @@ -7406,11 +7433,12 @@ def test_component_removal_transport_failure_reports_the_transport_error( """A retry-exhausted removal must name the transport error too.""" import pynetbox as real_pynb import requests + from core.change_detector import ChangeType, ComponentChange mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) stale = MagicMock() @@ -7435,11 +7463,12 @@ def test_failed_component_create_reports_the_netbox_message( ): """The report must name the constraint, not the generic failure label.""" import pynetbox as real_pynb + from core.change_detector import ChangeReport, ChangeType, ComponentChange, DeviceTypeChange mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 6119, {}) @@ -7484,7 +7513,7 @@ def _nb(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_new_device_type_with_failing_component_is_partial( @@ -7495,7 +7524,7 @@ def test_new_device_type_with_failing_component_is_partial( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 900, {}) mock_nb_api.dcim.interface_templates.create.side_effect = _request_error( @@ -7532,7 +7561,7 @@ def test_new_device_type_with_failing_component_is_partial( def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, graphql_client, make_device_types): """Errors buffered for one entity must not surface in the next entity's reason.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt._log_component_error("stale error from an earlier entity") @@ -7547,7 +7576,7 @@ def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, grap def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_client, make_device_types): """The buffer is empty after a scope closes, so the next scope starts clean.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) with dt.collect_component_errors() as first: @@ -7557,3 +7586,99 @@ def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_cl with dt.collect_component_errors() as second: pass assert second == [] + + +def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_settings, mock_pynetbox, mock_handle): + """A GraphQL schema error is a poor way to learn the server is too old.""" + from core.netbox_api import NetBoxError + + for version in ("4.1", "4.2", "3.5"): + mock_pynetbox.api.return_value.version = version + with pytest.raises(NetBoxError) as exc: + NetBox(mock_settings, mock_handle) + message = str(exc.value) + assert "4.3" in message, f"{version}: the message must name the minimum" + assert version in message, f"{version}: the message must name what was found" + + +class TestAMappingClearNeedsTheRemovalFlag: + """Clearing a front-port mapping removes data, so it obeys --remove-components. + + The tool tells the user "will not remove components from existing models" when the flag + is off. A mapping clear reaches NetBox as a COMPONENT_CHANGED property change, so it + bypassed that promise and cleared the linkage under a plain --update. + """ + + def _module_type_losing_a_mapping(self, nb): + """Record an existing FP1 mapped to RP1, and return YAML whose stanza omits it.""" + existing_module = MagicMock() + existing_module.id = 55 + existing_module.manufacturer.name = "Cisco" + existing_module.model = "CM-Map" + + existing_fp = SimpleNamespace( + name="FP1", + _mappings_canonical=[{"rear_port_name": "RP1", "front_port_position": 1, "rear_port_position": 1}], + _mappings_m2m=True, + ) + nb.device_types.components.record("front_port_templates", "module", 55, {"FP1": existing_fp}) + _mark_cache_ready(nb.device_types) + + curr_mt = { + "manufacturer": {"slug": "cisco"}, + "model": "CM-Map", + "slug": "cm-map", + # normalize_port_mappings assigns [] to a front port the stanza omits + "front-ports": [{"name": "FP1", "type": "8p8c", "_mappings": []}], + } + return {"cisco": {"CM-Map": existing_module}}, curr_mt + + def _mapping_clears_sent(self, nb): + """Return the _mappings property changes that reached update_components.""" + from core.change_detector import ChangeType + + sent = [] + for call_args in nb.device_types.update_components.call_args_list: + for change in call_args.args[2]: + if change.change_type is not ChangeType.COMPONENT_CHANGED: + continue + sent += [pc for pc in change.property_changes if pc.property_name == "_mappings"] + return sent + + @pytest.mark.parametrize(("remove_components", "expected"), [(False, 0), (True, 1)]) + def test_the_flag_decides_whether_a_clear_reaches_netbox( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle, remove_components, expected + ): + """Without the flag the clear must never be sent; with it, it must.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=remove_components + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset()] * expected + + def test_a_changed_mapping_still_applies_without_the_flag( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle + ): + """Only removal is gated. Repointing FP1 to another rear port is an ordinary update.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + curr_mt["front-ports"][0]["_mappings"] = [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=False + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset({("RP2", 1, 1)})] diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 6d759ea1..1aba6fe8 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -8,7 +8,7 @@ class TestNormalizeValues: def test_netbox_choice_object_reads_value(self): choice = SimpleNamespace(value="1000base-t") - y, n = normalize_values("1000base-t", choice) + _y, n = normalize_values("1000base-t", choice) assert n == "1000base-t" def test_empty_string_normalized_to_none(self): @@ -32,26 +32,26 @@ def test_trailing_spaces_stripped(self): assert n == "def" def test_numeric_yaml_coerces_string_netbox(self): - y, n = normalize_values(1.0, "1.0") + _y, n = normalize_values(1.0, "1.0") assert n == 1.0 def test_numeric_netbox_coerces_string_yaml(self): - y, n = normalize_values("2.5", 2.5) + y, _n = normalize_values("2.5", 2.5) assert y == 2.5 def test_int_yaml_preserves_int_type(self): """YAML int 166 vs NetBox '166.00' should normalize nb to int 166, not float 166.0.""" - y, n = normalize_values(166, "166.00") + _y, n = normalize_values(166, "166.00") assert n == 166 assert isinstance(n, int) def test_int_netbox_preserves_int_type(self): - y, n = normalize_values("166", 166) + y, _n = normalize_values("166", 166) assert y == 166 assert isinstance(y, int) def test_float_yaml_stays_float(self): - y, n = normalize_values(26.1, "26.10") + _y, n = normalize_values(26.1, "26.10") assert n == 26.1 def test_bool_not_coerced(self): @@ -66,11 +66,11 @@ def test_bool_as_int_trap(self): assert n == "1.0" def test_non_numeric_string_netbox_stays_string(self): - y, n = normalize_values(1.0, "notanumber") + _y, n = normalize_values(1.0, "notanumber") assert n == "notanumber" def test_non_numeric_string_yaml_stays_string(self): - y, n = normalize_values("notanumber", 1.0) + y, _n = normalize_values("notanumber", 1.0) assert y == "notanumber" diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py new file mode 100644 index 00000000..caee3dd0 --- /dev/null +++ b/tests/test_relation_scope.py @@ -0,0 +1,197 @@ +"""A relation assigned from the wrong manufacturer scope must be detected as a change. + +The catalog resolves a reference name in the owning manufacturer's scope first and only +then in Generic. If NetBox already holds the Generic object where the owner has one of +its own, comparing names alone reports equality and the wrong object survives. +""" + +import pytest + +from core.change_detector import ChangeDetector +from core.module_bay_types import ModuleBayTypeCatalog + + +class Handle: + """Capture what the detector logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +class Related: + """A module bay type as NetBox returns it inside a relation.""" + + def __init__(self, name, slug, manufacturer_slug): + """Store the identity fields the comparison needs.""" + self.name = name + self.slug = slug + self.manufacturer = type("M", (), {"slug": manufacturer_slug})() + + +class NetBoxBay: + """A module bay template as the cache hands it to the detector.""" + + def __init__(self, name, module_bay_types): + """Store the bay name and its assigned classes.""" + self.name = name + self.module_bay_types = module_bay_types + + +@pytest.fixture +def two_scope_catalog(tmp_path): + """Build a catalog where the same class name exists under Acme and under Generic.""" + for manufacturer, slug in (("Acme", "acme-x"), ("Generic", "generic-x")): + directory = tmp_path / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{slug}.yaml").write_text( + f"name: X\nslug: {slug}\nmanufacturer: {manufacturer}\n", encoding="utf-8" + ) + return str(tmp_path) + + +def _detector(catalog, handle=None): + """Build a detector whose device_types exposes the catalog, as the real one does.""" + device_types = type("DeviceTypes", (), {"module_bay_type_catalog": catalog, "module_bay_types_supported": True})() + return ChangeDetector(device_types, handle or Handle()) + + +def test_wrong_scope_assignment_is_detected(two_scope_catalog): + """Acme owns X, but NetBox assigned Generic's X. The names match; the objects do not.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert [c.property_name for c in changes] == ["module_bay_types"], ( + "a bay owned by Acme holding Generic's X must be corrected to Acme's X" + ) + + +def test_right_scope_assignment_is_left_alone(two_scope_catalog): + """The same bay already holding Acme's X is correct and must not be rewritten.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert changes == [] + + +def test_generic_fallback_is_correct_when_the_owner_has_no_such_class(two_scope_catalog): + """A Nokia bay resolves X to Generic's X, so holding Generic's X is right.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="nokia" + ) + assert changes == [] + + +def _changes(detector, yaml_comp, netbox_comp, manufacturer="acme"): + """Run the real property comparison for the relation and return what it found.""" + return detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer=manufacturer + ) + + +class TestUnmanagedRelations: + """A relation is only rewritten when both sides say something about it.""" + + def test_an_omitted_key_leaves_the_relation_alone(self, two_scope_catalog): + """A definition that never mentions the relation is not asking for it to be cleared.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0"}, netbox_comp) == [] + + def test_a_bare_key_leaves_the_relation_alone_but_says_so(self, two_scope_catalog): + """Parses as None, so nothing is cleared; a typo must still not be invisible.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": None}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + + def test_a_whitespace_only_entry_leaves_the_relation_alone(self, two_scope_catalog): + """The catalog rejects it later, which would skip the whole component's update.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [" "]}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): + """Reading an absent field as empty would report a change on every run.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = type("Bay", (), {"name": "Slot 0"})() + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_an_unresolvable_reference_reaches_the_write_path(self, two_scope_catalog): + """Reporting "no change" leaves the bay unrestricted in silence. + + The write path is the only thing that logs and records an unresolvable name, and it + only ever sees a component the detector reported as changed. + """ + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["NO-SUCH-CLASS"]}, netbox_comp) + + assert [(c.property_name, c.new_value) for c in changes] == [("module_bay_types", ["NO-SUCH-CLASS"])] + + +class TestNameComparisonFallback: + """Where an identity cannot be had, comparing names is still better than doing nothing.""" + + def test_names_are_compared_when_netbox_returned_no_slug(self, two_scope_catalog): + """A read path returning only id and name cannot answer the scope question.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", None, None)]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.property_name, c.old_value, c.new_value) for c in changes] == [("module_bay_types", ["Y"], ["X"])] + + def test_matching_names_are_left_alone_when_netbox_returned_no_slug(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", None, None)]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_names_are_compared_without_a_catalog(self): + """A caller that has not wired a catalog still gets the name comparison.""" + detector = _detector(None) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", "generic-y", "generic")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["Y"], ["X"])] + + def test_an_empty_list_clears_the_relation(self, two_scope_catalog): + """`module_bay_types: []` is an explicit instruction to hold no classes.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": []}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["X"], [])] diff --git a/tests/test_repo.py b/tests/test_repo.py index ab4e695a..c33af6b2 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,8 +1,15 @@ import os +import re +from unittest.mock import MagicMock, call, mock_open, patch import pytest -from unittest.mock import MagicMock, call, mock_open, patch -from git import Actor, Repo as GitRepo, exc as git_exc +import yaml +from git import Actor +from git import Repo as GitRepo +from git import exc as git_exc + +from core.errors import UnknownError +from core.log_handler import LogHandler from core.repo import ( DTLRepo, GitBranchNotFoundError, @@ -14,11 +21,9 @@ _safe_index_load, _safe_json_load, _safe_pickle_load, - validate_git_url, normalize_port_mappings, + validate_git_url, ) -from core.errors import UnknownError -from core.log_handler import LogHandler def _dtl_repo(config, repo_path, handle): @@ -45,31 +50,31 @@ def test_https_valid(self): assert err is None def test_https_no_hostname_invalid(self): - ok, err = validate_git_url("https://") + ok, _err = validate_git_url("https://") assert ok is False def test_git_at_scp_valid(self): - ok, err = validate_git_url("git@github.com:org/repo.git") + ok, _err = validate_git_url("git@github.com:org/repo.git") assert ok is True def test_git_at_no_colon_invalid(self): - ok, err = validate_git_url("git@github.com/org/repo.git") + ok, _err = validate_git_url("git@github.com/org/repo.git") assert ok is False def test_ssh_valid(self): - ok, err = validate_git_url("ssh://git@github.com/org/repo.git") + ok, _err = validate_git_url("ssh://git@github.com/org/repo.git") assert ok is True def test_ssh_no_hostname_invalid(self): - ok, err = validate_git_url("ssh://") + ok, _err = validate_git_url("ssh://") assert ok is False def test_file_valid(self): - ok, err = validate_git_url("file:///tmp/repo") + ok, _err = validate_git_url("file:///tmp/repo") assert ok is True def test_file_empty_path_invalid(self): - ok, err = validate_git_url("file://") + ok, _err = validate_git_url("file://") assert ok is False def test_empty_url_invalid(self): @@ -78,11 +83,11 @@ def test_empty_url_invalid(self): assert "Empty" in err def test_ftp_invalid(self): - ok, err = validate_git_url("ftp://example.com/repo.git") + ok, _err = validate_git_url("ftp://example.com/repo.git") assert ok is False def test_none_invalid(self): - ok, err = validate_git_url(None) + ok, _err = validate_git_url(None) assert ok is False @@ -142,7 +147,7 @@ def test_invalid_url_raises_before_the_clone_path(self): mock_args.repo_url = "ftp://bad.url" mock_args.repo_branch = "master" with _clone_present(False), patch("core.repo.Repo"): - with pytest.raises(InvalidGitURLError, match="Invalid Git URL: ftp://bad.url"): + with pytest.raises(InvalidGitURLError, match=re.escape("Invalid Git URL: ftp://bad.url")): _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) def test_invalid_path_raises_before_repository_access(self, tmp_path): @@ -254,7 +259,7 @@ class TestDTLRepoRealGit: @pytest.fixture(autouse=True) def mock_git_repo(self): """Override the global autouse git mock so these tests exercise real git.""" - yield None + return @pytest.fixture(autouse=True) def clear_ambient_git_env(self, monkeypatch): @@ -357,8 +362,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_get_relative_path(self): repo = self._make_repo() @@ -472,7 +476,7 @@ def test_repo_open_git_error_uses_the_configured_url_without_reading_self_repo(s failure = git_exc.GitCommandError("status", 1) with _clone_present(), patch("core.repo.Repo", side_effect=failure): - with pytest.raises(GitCommandError, match="https://example.invalid/repo.git") as exc_info: + with pytest.raises(GitCommandError, match=re.escape("https://example.invalid/repo.git")) as exc_info: _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) assert "cmdline: status" in exc_info.value.formatted_traceback @@ -498,7 +502,9 @@ def test_invalid_git_repository_raises_its_catalogue_error(self): invalid = git_exc.InvalidGitRepositoryError("/tmp/repo") with _clone_present(), patch("core.repo.Repo", side_effect=invalid): - with pytest.raises(GitInvalidRepositoryError, match='The repo "/tmp/repo" is not a valid git repo.'): + with pytest.raises( + GitInvalidRepositoryError, match=re.escape('The repo "/tmp/repo" is not a valid git repo.') + ): _dtl_repo(mock_args, "/tmp/repo", LogHandler(False)) @@ -546,8 +552,7 @@ def _make_repo(self, tmp_path): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) - return repo + return _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) def test_get_devices_all_vendors(self, tmp_path): repo = self._make_repo(tmp_path) @@ -555,7 +560,7 @@ def test_get_devices_all_vendors(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "Juniper").mkdir() - files, vendors = repo.get_devices(str(devices)) + _files, vendors = repo.get_devices(str(devices)) assert len(vendors) == 2 assert any(v["name"] == "Cisco" for v in vendors) @@ -565,7 +570,7 @@ def test_get_devices_filters_vendors(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "Juniper").mkdir() - files, vendors = repo.get_devices(str(devices), vendors=["cisco"]) + _files, vendors = repo.get_devices(str(devices), vendors=["cisco"]) assert len(vendors) == 1 assert vendors[0]["name"] == "Cisco" @@ -575,7 +580,7 @@ def test_get_devices_skips_testing_folder(self, tmp_path): devices.mkdir() (devices / "Cisco").mkdir() (devices / "testing").mkdir() - files, vendors = repo.get_devices(str(devices)) + _files, vendors = repo.get_devices(str(devices)) assert not any(v["name"] == "testing" for v in vendors) @@ -595,8 +600,7 @@ def _make_repo(self, tmp_path): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) - return repo + return _dtl_repo(mock_args, str(tmp_path / "repo"), mock_handle) def test_discovers_vendors_from_single_path(self, tmp_path): """Test discovery from a single existing path.""" @@ -676,7 +680,7 @@ def test_handles_os_errors_gracefully(self, tmp_path): def mock_listdir(path): if "devices" in path: raise OSError("Permission denied") - elif "modules" in path: + if "modules" in path: return ["Cisco"] return [] @@ -1104,6 +1108,44 @@ def test_null_stanza_is_deleted(self): assert err is None assert "port-mappings" not in data + @pytest.mark.parametrize("stanza", ["", "port-mappings:", "port-mappings: []"]) + def test_only_an_explicit_list_manages_front_port_mappings(self, stanza): + data = yaml.safe_load(f""" +front-ports: + - name: FP1 + type: 8p8c + - name: FP2 + type: 8p8c +{stanza} +""") + + assert normalize_port_mappings(data) is None + assert "port-mappings" not in data + for port in data["front-ports"]: + if stanza == "port-mappings: []": + assert port["_mappings"] == [] + else: + assert "_mappings" not in port + + @pytest.mark.parametrize( + ("stanza", "expected"), + [ + ("RP1", "Error: port-mappings must be a list: 'RP1'"), + ("{FP1: RP1}", "Error: port-mappings must be a list: {'FP1': 'RP1'}"), + ("[RP1]", "Error: port-mappings entry must be a mapping: 'RP1'"), + ], + ) + def test_malformed_stanza_returns_an_error(self, stanza, expected): + data = yaml.safe_load(f""" +front-ports: + - {{name: FP1, type: 8p8c}} +rear-ports: + - {{name: RP1, type: 8p8c}} +port-mappings: {stanza} +""") + + assert normalize_port_mappings(data) == expected + def test_empty_stanza_no_front_ports_still_deleted(self): """Empty port-mappings stanza with no front-ports is cleaned up (not silently skipped).""" data = { @@ -1214,7 +1256,7 @@ def test_valid_existing_dir_returns_true(self, tmp_path): """Existing writable directory returns True.""" from core.repo import validate_repo_path - ok, msg = validate_repo_path(str(tmp_path)) + ok, _msg = validate_repo_path(str(tmp_path)) assert ok is True @@ -1229,6 +1271,7 @@ def test_parse_device_type_returns_error_when_normalize_fails(tmp_path): Covers repo.py lines 225-226: 'if err: return err'. """ from unittest.mock import patch + from core.repo import parse_single_file yaml_file = tmp_path / "test.yaml" @@ -1306,8 +1349,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_get_racks_path_ends_with_rack_types(self): repo = self._make_repo() @@ -1359,8 +1401,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_keyboard_interrupt_is_reraised(self): import pytest @@ -1395,8 +1436,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_item_without_manufacturer_is_included_without_dedup(self): """Item missing 'manufacturer' key skips dedup and is appended as-is.""" @@ -1497,8 +1537,7 @@ def _make_repo(self): ref.name = "origin/master" mock_git_repo.remotes.origin.refs = [ref] MockRepo.return_value = mock_git_repo - repo = _dtl_repo(mock_args, "/tmp/repo", mock_handle) - return repo + return _dtl_repo(mock_args, "/tmp/repo", mock_handle) def test_returns_none_when_pickle_missing(self, tmp_path): """Returns None gracefully when the device pickle doesn't exist.""" @@ -1796,3 +1835,171 @@ def test_corrupted_device_json_returns_none(self, tmp_path): repo.cwd = "" assert repo.resolve_slug_files(["nokia"]) is None + + +class TestAnExplicitlyEmptyStanza: + """`port-mappings: []` is an author saying "none", which is not the same as saying nothing.""" + + def test_an_empty_stanza_clears_every_front_port_mapping(self): + """Without _mappings: [] the change detector cannot express removing a mapping.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c"}, {"name": "FP2", "type": "8p8c"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 2}], + "port-mappings": [], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [] + assert data["front-ports"][1]["_mappings"] == [] + + def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): + """Silently preferring the inline linkage ignores the newer, explicit statement.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 1}], + "port-mappings": [], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an empty stanza beside an inline linkage must not pass silently" + assert result.startswith("Error:"), result + + def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): + """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" + from core.repo import normalize_port_mappings + + data = {"front-ports": [{"name": "FP1", "type": "8p8c"}], "rear-ports": []} + + assert normalize_port_mappings(data) is None + assert "_mappings" not in data["front-ports"][0] + + +class TestAStanzaThatDoesNotListAFrontPort: + """A stanza speaks for the whole file, so a port it omits has no mapping.""" + + def test_a_nonempty_stanza_clears_an_omitted_front_port_mapping(self): + from types import SimpleNamespace + + from core.change_detector import ChangeDetector + + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP1, rear_port: RP1} +""") + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [] + + existing = SimpleNamespace( + name="FP2", + _mappings_canonical=[{"rear_port_name": "RP2", "front_port_position": 1, "rear_port_position": 1}], + ) + detector = ChangeDetector(SimpleNamespace(), LogHandler(False)) + changes = detector._compare_component_properties( + data["front-ports"][1], existing, ["_mappings"], comp_type="front-ports" + ) + assert len(changes) == 1 + assert changes[0].property_name == "_mappings" + assert changes[0].old_value == {("RP2", 1, 1)} + assert changes[0].new_value == set() + + def test_a_half_migrated_file_errors_before_assigning_mappings(self): + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c, rear_port: RP1} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP2, rear_port: RP2} +""") + + assert normalize_port_mappings(data) == ( + "Error: front port 'FP1' declares an inline rear_port but the port-mappings " + "stanza does not list it; the stanza is authoritative, so add 'FP1' to it " + "or remove the inline rear_port keys" + ) + assert all("_mappings" not in port for port in data["front-ports"]) + + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): + """The old wording blamed a conflict against a stanza that never mentioned the port.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP2", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an inline linkage the stanza omits must not pass silently" + assert "conflicting mapping definitions" not in result, result + assert "FP1" in result, result + assert "does not list it" in result, result + + def test_a_disagreement_on_a_shared_front_port_still_reads_as_a_conflict(self): + """Both formats naming one port differently is a real conflict, not an omission.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP1", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None + assert "conflicting mapping definitions" in result, result + + def test_a_stanza_may_add_a_port_the_inline_format_never_linked(self): + """The half-finished migration the two formats exist to allow: both are kept.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [ + {"front_port": "FP1", "rear_port": "RP1"}, + {"front_port": "FP2", "rear_port": "RP2"}, + ], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index dd8a46bc..c456b12b 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -39,9 +39,12 @@ def _orphaned_docstrings(tree): if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue for statement in node.body[1:]: - if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): - if isinstance(statement.value.value, str): - yield node.name, statement.lineno + if ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, str) + ): + yield node.name, statement.lineno def test_the_scan_follows_pytest_discovery(tmp_path): @@ -82,6 +85,25 @@ def test_integration_collection_reads_credentials_from_a_local_env_file(tmp_path assert "1 passed" in result.stdout, result.stdout + result.stderr +def test_the_fake_netbox_releases_its_listening_socket(): + """shutdown() only stops the serve loop. One server per test leaks a descriptor each.""" + import socket + + from helpers import FakeNetBox + + server = FakeNetBox(manufacturers=[{"id": 1, "name": "Juniper", "slug": "juniper"}]) + port = server._server.server_port + server.close() + + probe = socket.socket() + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: # pragma: no cover - only reached when close() leaks + raise AssertionError(f"FakeNetBox.close() left port {port} bound: {exc}") from exc + finally: + probe.close() + + def test_no_test_function_contains_an_orphaned_docstring(): """A dropped ``def`` header silently merges two tests into one. diff --git a/tests/test_update_failure_resolver.py b/tests/test_update_failure_resolver.py index a6135626..1185c099 100644 --- a/tests/test_update_failure_resolver.py +++ b/tests/test_update_failure_resolver.py @@ -9,11 +9,10 @@ from core.update_failure_resolver import ( FailureKind, - extract_error_payload, classify_device_type_update_failure, + extract_error_payload, ) - SUBDEVICE_ROLE_ERROR_DICT = { "subdevice_role": [ "Must delete all device bay templates associated with this device before declassifying it as a parent device." @@ -314,57 +313,18 @@ def test_classifier_count_fallback_when_count_query_fails(): assert res.dependent_devices_count == 5 -def test_new_filters_uses_device_type_id_key(): - """new_filters=True must call filter(device_type_id=...) not devicetype_id=... +def test_the_device_bay_template_lookup_uses_the_supported_filter_key(): + """The 4.1 rename is below the supported floor, so only device_type_id is ever sent. - This matters because NetBox >= 4.1 changed the query param name. - Passing the wrong key causes pynetbox to silently return ALL templates. + The wrong key does not raise: pynetbox silently returns every template, so the + classifier would report an unrelated device type as the blocker. """ nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=99, device_type_yaml={}) nb.dcim.device_bay_templates.filter.assert_called_once_with(device_type_id=99) -def test_old_filters_uses_devicetype_id_key(): - """new_filters=False (default) must call filter(devicetype_id=...) for NetBox < 4.1.""" - nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.device_bay_templates.filter.assert_called_once_with(devicetype_id=99) - - -def test_count_dependent_devices_uses_new_filter_key(): - """When new_filters=True, dcim.devices must be queried with device_type_id= not devicetype_id=.""" +def test_the_dependent_device_count_uses_the_supported_filter_key(): nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=77, device_type_yaml={}) nb.dcim.devices.filter.assert_called_once_with(device_type_id=77, limit=5) - - -def test_count_dependent_devices_uses_legacy_filter_key(): - """When new_filters=False (default), dcim.devices must be queried with devicetype_id=.""" - nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.devices.filter.assert_called_once_with(devicetype_id=77, limit=5)