Add the SONiC golden-test harness

tests/e2e/compare.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Golden-file comparator for the SONiC config-generation E2E test.
+
+Compares exported SONiC ``config_db.json`` files against committed golden
+files. config_db is a shallow TABLE -> entry -> attribute tree, so
+structural difference paths are rendered as ``TABLE|entry.attribute``.
+
+Canonicalization sorts dictionary keys only; array order is preserved and
+compared, since list order can be part of the generated contract.
+"""
+
+import argparse
+import difflib
+import json
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+def _walk(golden, actual, path, depth, out):
+    """Collect difference paths between two parsed JSON values.
+
+    ``depth`` counts dict levels already entered: children of the top level
+    (tables) attach with ``|``, everything deeper with ``.``. List indices
+    attach as ``[i]`` and stay at their parent's depth.
+    """
+    if isinstance(golden, dict) and isinstance(actual, dict):
+        for key in sorted(golden.keys() | actual.keys()):
+            sep = "|" if depth == 1 else "."
+            child = f"{path}{sep}{key}" if path else str(key)
+            if key not in actual:
+                out.append(f"{child}: only in golden")
+            elif key not in golden:
+                out.append(f"{child}: only in actual")
+            else:
+                _walk(golden[key], actual[key], child, depth + 1, out)
+    elif isinstance(golden, list) and isinstance(actual, list):
+        for i in range(max(len(golden), len(actual))):
+            child = f"{path}[{i}]"
+            if i >= len(actual):
+                out.append(f"{child}: only in golden")
+            elif i >= len(golden):
+                out.append(f"{child}: only in actual")
+            else:
+                _walk(golden[i], actual[i], child, depth, out)
+    elif golden != actual:
+        out.append(f"{path}: golden {golden!r}, actual {actual!r}")
+
+
+def diff_paths(golden, actual):
+    """Return sorted structural paths of the differences between two configs."""
+    out = []
+    _walk(golden, actual, "", 0, out)
+    return sorted(out)
+
+
+def _canonical(config):
+    """Canonical JSON text: dict keys sorted, arrays untouched."""
+    return json.dumps(config, indent=2, sort_keys=True) + "\n"
+
+
+def unified_diff(golden, actual, name):
+    """Unified diff of the canonicalized golden and actual configs."""
+    return "".join(
+        difflib.unified_diff(
+            _canonical(golden).splitlines(keepends=True),
+            _canonical(actual).splitlines(keepends=True),
+            fromfile=f"golden/{name}",
+            tofile=f"export/{name}",
+        )
+    )
+
+
+@dataclass
+class Mismatch:
+    """Content difference of one exported file against its golden file."""
+
+    paths: list
+    diff: str
+
+
+@dataclass
+class CompareResult:
+    """Outcome of comparing an export directory against the golden directory."""
+
+    missing: list = field(default_factory=list)
+    extra: list = field(default_factory=list)
+    mismatched: dict = field(default_factory=dict)
+
+    @property
+    def ok(self):
+        return not (self.missing or self.extra or self.mismatched)
+
+    def report(self):
+        lines = []
+        for name in self.missing:
+            lines.append(
+                f"MISSING {name}: not exported "
+                "(generation or export failed - see generate log)"
+            )
+        for name in self.extra:
+            lines.append(f"EXTRA {name}: unexpected extra export (no golden file)")
+        for name, mismatch in self.mismatched.items():
+            lines.append(f"MISMATCH {name}:")
+            lines.extend(f"  {path}" for path in mismatch.paths)
+            lines.append(mismatch.diff)
+        if not lines:
+            lines.append("OK: exports match the golden files")
+        return "\n".join(lines)
+
+
+def compare_dirs(golden_dir, export_dir):
+    """Compare all golden ``*.json`` files against the exported ones.
+
+    Requires exact file-set equality: a golden file without an export is
+    ``missing`` (generation or export failed), an export without a golden
+    file is ``extra``, and differing content is ``mismatched``. Non-JSON
+    files (e.g. firmware symlinks in the export directory) are ignored.
+    """
+    golden_dir = Path(golden_dir)
+    export_dir = Path(export_dir)
+    golden_names = {p.name for p in golden_dir.glob("*.json")}
+    export_names = {p.name for p in export_dir.glob("*.json")}
+
+    result = CompareResult(
+        missing=sorted(golden_names - export_names),
+        extra=sorted(export_names - golden_names),
+    )
+    for name in sorted(golden_names & export_names):
+        golden = json.loads((golden_dir / name).read_text())
+        actual = json.loads((export_dir / name).read_text())
+        paths = diff_paths(golden, actual)
+        if paths:
+            result.mismatched[name] = Mismatch(
+                paths=paths, diff=unified_diff(golden, actual, name)
+            )
+    return result
+
+
+def regenerate(golden_dir, export_dir):
+    """Rewrite the golden directory from the export directory.
+
+    Every exported ``*.json`` file is stored in canonical form (sorted dict
+    keys) so regenerated goldens produce minimal, reviewable git diffs;
+    stale golden files without a matching export are removed. Never run in
+    CI - regeneration is a deliberate local step after an intentional
+    generator change.
+
+    Returns a list of entries describing coverage that was lost: either a
+    table that was populated in the previous golden and would become empty,
+    or a whole golden file that had no matching export at all and is being
+    removed. Regeneration is the only path by which coverage can silently
+    drop -- in the normal path a table going empty (or a whole device
+    disappearing) changes the golden and fails the comparison -- and either
+    is invisible inside a several-hundred-line JSON diff. The files are
+    still written (or removed); reporting is the caller's decision to act
+    on.
+    """
+    golden_dir = Path(golden_dir)
+    export_dir = Path(export_dir)
+    golden_dir.mkdir(parents=True, exist_ok=True)
+
+    def _populated(config):
+        return {t for t, v in config.items() if v}
+
+    lost = []
+    export_names = {p.name for p in export_dir.glob("*.json")}
+    for stale in golden_dir.glob("*.json"):
+        if stale.name not in export_names:
+            n = len(_populated(json.loads(stale.read_text())))
+            if n:
+                lost.append(f"{stale.name}: file removed, had {n} populated tables")
+            stale.unlink()
+
+    for name in sorted(export_names):
+        config = json.loads((export_dir / name).read_text())
+        previous = golden_dir / name
+        if previous.exists():
+            before = _populated(json.loads(previous.read_text()))
+            for table in sorted(before - _populated(config)):
+                lost.append(f"{name}: {table}")
+        previous.write_text(_canonical(config))
+    return lost
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--golden", required=True, help="golden file directory")
+    parser.add_argument("--export", required=True, help="export directory to check")
+    parser.add_argument(
+        "--regenerate",
+        action="store_true",
+        help="rewrite the golden files from the export directory",
+    )
+    parser.add_argument(
+        "--allow-coverage-loss",
+        action="store_true",
+        help="regenerate even when a previously populated table becomes empty",
+    )
+    args = parser.parse_args(argv)
+
+    if args.regenerate:
+        lost = regenerate(args.golden, args.export)
+        if lost and not args.allow_coverage_loss:
+            print("COVERAGE LOSS -- coverage was lost:")
+            for entry in lost:
+                print(f"  {entry}")
+            print(
+                "Goldens were written. Review the diff, then either fix the "
+                "fixtures or re-run with --allow-coverage-loss to accept it."
+            )
+            return 1
+        return 0
+
+    result = compare_dirs(args.golden, args.export)
+    print(result.report())
+    return 0 if result.ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/e2e/generate.py

@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Generation driver for the SONiC config E2E test.
+
+Runs ``sync_sonic()`` against the seeded NetBox and asserts that generation
+itself succeeded. ``sync_sonic`` returns only a ``{device: config}`` dict:
+its internal rc is reported solely through the task layer when a ``task_id``
+is set, and per-device failures are logged and swallowed. Success is
+therefore asserted here by
+
+- comparing the returned device set against the expectation derived from
+  the golden files (``--golden``), and
+- failing on any ERROR-level loguru record captured during the run, which
+  covers every swallowed-exception path in ``sync.py``.
+
+All environment (NETBOX_API/NETBOX_TOKEN, SONIC_EXPORT_DIR,
+SONIC_PORT_CONFIG_PATH, SONIC_EXPORT_IDENTIFIER=hostname) must be set
+before this module is imported, since ``osism.settings`` reads it at import
+time. The exported files are checked against the goldens by ``compare.py``.
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from loguru import logger
+
+
+def expected_devices(golden_dir, prefix, suffix):
+    """Derive the expected device names from the golden file names."""
+    devices = set()
+    for path in Path(golden_dir).glob("*.json"):
+        if path.name.startswith(prefix) and path.name.endswith(suffix):
+            devices.add(path.name[len(prefix) : -len(suffix)])
+    return devices
+
+
+def run_generation(sync):
+    """Call the sync function, capturing ERROR-level loguru records.
+
+    Returns ``(device_configs, errors)`` where ``errors`` is the list of
+    error messages logged during the run.
+    """
+    errors = []
+    sink_id = logger.add(
+        lambda message: errors.append(str(message).strip()),
+        level="ERROR",
+        format="{message}",
+    )
+    try:
+        configs = sync()
+    finally:
+        logger.remove(sink_id)
+    return configs, errors
+
+
+def main(argv=None, sync=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--golden",
+        help="golden file directory the expected device set is derived from",
+    )
+    parser.add_argument(
+        "--no-expect",
+        action="store_true",
+        help="skip the device-set check (golden bootstrap / regeneration)",
+    )
+    args = parser.parse_args(argv)
+    if not args.no_expect and not args.golden:
+        parser.error("either --golden or --no-expect is required")
+
+    if sync is None:
+        missing_env = [
+            name for name in ("NETBOX_API", "NETBOX_TOKEN") if not os.environ.get(name)
+        ]
+        if missing_env:
+            print(f"Missing required environment variables: {', '.join(missing_env)}")
+            return 2
+        from osism.tasks.conductor.sonic.sync import sync_sonic
+
+        sync = sync_sonic
+
+    configs, errors = run_generation(sync)
+
+    failed = False
+    for error in errors:
+        print(f"ERROR during generation: {error}")
+        failed = True
+
+    empty = sorted(name for name, config in configs.items() if not config)
+    for name in empty:
+        print(f"EMPTY config generated for device: {name}")
+        failed = True
+
+    if args.golden:
+        from osism import settings
+
+        expected = expected_devices(
+            args.golden, settings.SONIC_EXPORT_PREFIX, settings.SONIC_EXPORT_SUFFIX
+        )
+        for name in sorted(expected - set(configs)):
+            print(f"MISSING device (expected from goldens, not generated): {name}")
+            failed = True
+        for name in sorted(set(configs) - expected):
+            print(f"UNEXPECTED device (generated, no golden file): {name}")
+            failed = True
+
+    print(f"Generated {len(configs)} SONiC configurations")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())