diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..25d5e76 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +# Coverage for CI's floor (pytest --cov --cov-fail-under=N in ci.yml). +[run] +source = + systemgate +omit = + */tests/* + +[report] +skip_empty = true diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5924afb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +# Editor settings shared by every Conker repository; .gitattributes keeps line endings LF. +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{js,jsx,mjs,cjs,ts,tsx,json,yml,yaml,css,html}] +indent_size = 2 + +[*.md] +# A trailing double space is a Markdown line break. +trim_trailing_whitespace = false + +[*.{patch,diff}] +# Unified diffs need their single-space prefix on empty context lines. +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..6cdd4fd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Every change needs the owner's review. +* @alexeybe1kin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec2a004..69318eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: run: | pip install --quiet ruff==0.16.6 python -m ruff check . + python -m ruff format --check . - name: The module contract requires these files to exist run: | @@ -63,8 +64,15 @@ jobs: # 600 lines per source file; files that were already larger may only shrink. run: python scripts/check_file_sizes.py + - name: Types + # pyright basic mode (pyrightconfig.json). Errors that predate the check are counted + # per file in type-baseline.txt; a count may only go down. + run: python scripts/check_types.py + - name: Tests - run: python -m pytest tests/ -q + # The coverage floor is the coverage measured when it arrived, rounded down, so it + # only stops coverage from falling (.coveragerc names what is measured). + run: python -m pytest tests/ -q --cov --cov-report=term --cov-fail-under=93 contract: # The health contract is an HTTP contract, so it is asserted against a diff --git a/.gitignore b/.gitignore index 14d0d74..da4df3d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ __pycache__/ .mypy_cache/ data/ +.coverage +.coverage.* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3dd593e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +# The same checks CI runs, with the same pinned tools (requirements-dev.txt), so a commit +# that passes here passes the lint stage there. Install once: pip install pre-commit && +# pre-commit install --install-hooks -t pre-commit -t pre-push +default_install_hook_types: [pre-commit, pre-push] +repos: +- repo: local + hooks: + - id: ruff-check + name: ruff check + entry: python -m ruff check . + language: system + pass_filenames: false + types: [python] + - id: ruff-format + name: ruff format --check + entry: python -m ruff format --check . + language: system + pass_filenames: false + types: [python] + - id: file-sizes + name: file sizes (600 lines) + entry: python scripts/check_file_sizes.py + language: system + pass_filenames: false + always_run: true + - id: types + name: pyright basic, baseline only shrinks + entry: python scripts/check_types.py + language: system + pass_filenames: false + types: [python] + stages: [pre-push] diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..56e7c13 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,7 @@ +{ + "typeCheckingMode": "basic", + "pythonVersion": "3.12", + "pythonPlatform": "Linux", + "include": ["systemgate"], + "exclude": ["**/tests", "**/__pycache__", "**/node_modules", "**/.venv"] +} diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..351531b --- /dev/null +++ b/renovate.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended"], + "timezone": "UTC", + "schedule": ["before 6am on monday"], + "minimumReleaseAge": "7 days", + "automerge": false, + "dependencyDashboard": true, + "prConcurrentLimit": 5, + "packageRules": [ + { + "description": "One weekly pull request for every minor, patch and digest update.", + "matchUpdateTypes": ["minor", "patch", "pin", "digest"], + "groupName": "weekly minor and patch updates", + "groupSlug": "weekly-minor-patch" + }, + { + "description": "Major updates stay one pull request per package, reviewed one at a time.", + "matchUpdateTypes": ["major"], + "automerge": false + } + ] +} diff --git a/requirements-dev.txt b/requirements-dev.txt index 2248b3d..c950747 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ -r requirements.txt pytest==9.1.1 ruff==0.16.6 +pytest-cov==7.1.0 +pyright==1.1.414 diff --git a/ruff.toml b/ruff.toml index 9aa6ac6..cb05548 100644 --- a/ruff.toml +++ b/ruff.toml @@ -24,7 +24,9 @@ ignore = [ # that raised instead would take down the very endpoint the owner uses to # find out what is wrong. See the truthful-status rule in CLAUDE.md. "BLE001", - # Line length is left to review until this module adopts `ruff format`. + # `ruff format` enforces line length for code (CI runs `ruff format --check`). + # What it cannot wrap are long string literals, whose exact text matters more + # than a column limit. "E501", ] diff --git a/scripts/check_file_sizes.py b/scripts/check_file_sizes.py index 543fb9c..3c5599d 100644 --- a/scripts/check_file_sizes.py +++ b/scripts/check_file_sizes.py @@ -45,12 +45,20 @@ def main() -> int: if ceiling is None and lines > LIMIT: problems.append(f"{path}: {lines} lines, limit {LIMIT}. Split it by concern.") elif ceiling is not None and lines > ceiling: - problems.append(f"{path}: grew to {lines} lines; it is allowlisted at {ceiling} and may only shrink.") + problems.append( + f"{path}: grew to {lines} lines; it is allowlisted at {ceiling} and may only shrink." + ) elif ceiling is not None and lines <= LIMIT: - problems.append(f"{path}: now {lines} lines, under the limit; delete its line from {ALLOWLIST}.") + problems.append( + f"{path}: now {lines} lines, under the limit; delete its line from {ALLOWLIST}." + ) elif ceiling is not None and lines < ceiling: - problems.append(f"{path}: shrank to {lines} lines; lower its entry in {ALLOWLIST} from {ceiling}.") - problems += [f"{path}: listed in {ALLOWLIST} but no longer tracked; delete the line." for path in allowed] + problems.append( + f"{path}: shrank to {lines} lines; lower its entry in {ALLOWLIST} from {ceiling}." + ) + problems += [ + f"{path}: listed in {ALLOWLIST} but no longer tracked; delete the line." for path in allowed + ] if problems: print("\n".join(problems)) return 1 diff --git a/scripts/check_types.py b/scripts/check_types.py new file mode 100644 index 0000000..281a8e0 --- /dev/null +++ b/scripts/check_types.py @@ -0,0 +1,96 @@ +"""Fail when pyright (basic mode) finds a type error that is not in the baseline. + +Run from the repository root: python scripts/check_types.py +pyrightconfig.json sets the mode and the checked folders. Errors that already existed when the +check arrived are counted per file in type-baseline.txt; a file's count may shrink but never +grow, and a line must be lowered or deleted once its file has fewer errors, so the list only +ever gets shorter. `--write` rewrites the baseline from the current errors: use it only to +lower counts, never to accept new errors. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from collections import Counter +from pathlib import Path + +BASELINE = Path("type-baseline.txt") +HEADER = ( + "# pyright basic-mode errors per file when the type check arrived; counts may only shrink.\n" + "# Lower or delete a line when its file improves (scripts/check_types.py says which).\n" +) + + +def current_errors() -> Counter[str]: + run = subprocess.run( + [sys.executable, "-m", "pyright", "--outputjson"], + capture_output=True, + text=True, + check=False, + ) + try: + report = json.loads(run.stdout) + except json.JSONDecodeError: + print(run.stdout + run.stderr) + print("pyright did not produce a JSON report.") + sys.exit(2) + root = Path.cwd().resolve() + counts: Counter[str] = Counter() + for item in report.get("generalDiagnostics", []): + if item.get("severity") == "error": + path = Path(item["file"]).resolve().relative_to(root).as_posix() + counts[path] += 1 + return counts + + +def baseline() -> dict[str, int]: + if not BASELINE.exists(): + return {} + entries = {} + for line in BASELINE.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + parts = line.rsplit(maxsplit=1) + if len(parts) != 2 or not parts[1].isdigit(): + print(f"{BASELINE}: malformed entry: {line!r}") + sys.exit(1) + entries[parts[0]] = int(parts[1]) + return entries + + +def write(counts: Counter[str]) -> None: + lines = [f"{path} {count}" for path, count in sorted(counts.items())] + text = HEADER + "".join(f"{line}\n" for line in lines) + BASELINE.write_text(text, encoding="utf-8", newline="\n") + print(f"wrote {BASELINE}: {sum(counts.values())} errors in {len(counts)} files") + + +def main() -> int: + counts = current_errors() + if "--write" in sys.argv[1:]: + write(counts) + return 0 + allowed = baseline() + problems = [] + for path in sorted(set(counts) | set(allowed)): + found, ceiling = counts.get(path, 0), allowed.get(path, 0) + if found > ceiling: + problems.append( + f"{path}: {found} type errors, baseline {ceiling}. Fix the new ones " + "(python -m pyright " + path + ")." + ) + elif found < ceiling: + target = "delete its line" if found == 0 else f"lower it to {found}" + problems.append(f"{path}: down to {found} type errors; {target} in {BASELINE}.") + if problems: + print("\n".join(problems)) + return 1 + print(f"types ok ({sum(counts.values())} baselined errors in {len(counts)} files)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/systemgate/__init__.py b/systemgate/__init__.py index 13b67e7..8d74a01 100644 --- a/systemgate/__init__.py +++ b/systemgate/__init__.py @@ -1,2 +1 @@ """SystemGate package.""" - diff --git a/systemgate/auth.py b/systemgate/auth.py index c7c634f..825264c 100644 --- a/systemgate/auth.py +++ b/systemgate/auth.py @@ -39,4 +39,3 @@ def verify_key(key: str | None, encoded: str) -> bool: _matched.clear() _matched[encoded] = fast return True - diff --git a/systemgate/backups.py b/systemgate/backups.py index 9490200..68d439a 100644 --- a/systemgate/backups.py +++ b/systemgate/backups.py @@ -4,6 +4,7 @@ or manifest claiming success is not evidence that the snapshot is intact. Verification does not establish deletion replay, vault recovery or restorability. """ + from __future__ import annotations import hashlib @@ -14,14 +15,32 @@ import time from datetime import datetime from pathlib import Path, PurePosixPath +from typing import NoReturn FORMAT = "conker-snapshot-1" -SERVICES = {"pi", "toolgate", "memorygate", "systemgate", "embeddings", "postgres", - "qdrant", "ollama", "searxng"} +SERVICES = { + "pi", + "toolgate", + "memorygate", + "systemgate", + "embeddings", + "postgres", + "qdrant", + "ollama", + "searxng", +} STORES = {"pi", "toolgate", "systemgate", "qdrant", "ollama", "memorygate-backups"} -REQUIRED = {*(name + ".tar" for name in STORES), "memorygate.dump", "postgres-globals.sql", - "config/env", "config/versions.env", "config/docker-compose.yml", "config/compose.json", - "config/runtime.json", "memorygate/runtime-token"} +REQUIRED = { + *(name + ".tar" for name in STORES), + "memorygate.dump", + "postgres-globals.sql", + "config/env", + "config/versions.env", + "config/docker-compose.yml", + "config/compose.json", + "config/runtime.json", + "memorygate/runtime-token", +} class InvalidSnapshot(ValueError): @@ -31,14 +50,20 @@ def __init__(self, status: str, reason: str) -> None: super().__init__(reason) -def _invalid(reason: str, status: str = "invalid") -> None: +def _invalid(reason: str, status: str = "invalid") -> NoReturn: raise InvalidSnapshot(status, reason) def _safe_name(name: str) -> bool: path = PurePosixPath(name) - return (bool(name) and not path.is_absolute() and ".." not in path.parts - and "\\" not in name and ":" not in name and path.as_posix() == name) + return ( + bool(name) + and not path.is_absolute() + and ".." not in path.parts + and "\\" not in name + and ":" not in name + and path.as_posix() == name + ) def _files(directory: Path) -> dict[str, Path]: @@ -70,8 +95,11 @@ def _archive(path: Path, required: str | None) -> None: seen = set() with tarfile.open(path, "r:") as archive: for member in archive: - if (not _safe_name(member.name) or member.name in seen - or not (member.isfile() or member.isdir())): + if ( + not _safe_name(member.name) + or member.name in seen + or not (member.isfile() or member.isdir()) + ): _invalid("Archive has unsafe or duplicate members; obtain an intact snapshot.") seen.add(member.name) if member.name == required and not member.isfile(): @@ -94,7 +122,9 @@ def verify_snapshot(directory: Path) -> dict: if manifest_path.is_symlink(): _invalid("Manifest must not be a link; obtain an intact snapshot.") if not manifest_path.is_file(): - _invalid("Manifest missing: incomplete or legacy backup. Create a new backup.", "incomplete") + _invalid( + "Manifest missing: incomplete or legacy backup. Create a new backup.", "incomplete" + ) manifest_before = _signature(manifest_path) if manifest_path.stat().st_size > 4 * 1024 * 1024: _invalid("Manifest exceeds the supported size; verify it with the recovery command.") @@ -121,25 +151,39 @@ def verify_snapshot(directory: Path) -> dict: if files.keys() != actual.keys(): _invalid("Files are missing or unexpected; obtain an intact snapshot.") images = manifest.get("images") - if (not isinstance(images, dict) or images.keys() != SERVICES - or any(not isinstance(image, dict) - or not re.fullmatch(r"sha256:[0-9a-f]{64}", str(image.get("id", ""))) - for image in images.values())): + if ( + not isinstance(images, dict) + or images.keys() != SERVICES + or any( + not isinstance(image, dict) + or not re.fullmatch(r"sha256:[0-9a-f]{64}", str(image.get("id", ""))) + for image in images.values() + ) + ): _invalid("Image identities are invalid; obtain an intact version manifest.") stores = manifest.get("stores") - if (not isinstance(stores, dict) or not stores.keys() >= STORES - or any(name not in STORES and not re.fullmatch(r"extra-[a-z]+-[0-9]+", name) - for name in stores) - or {name + ".tar" for name in stores} != {n for n in files if n.endswith(".tar")}): + if ( + not isinstance(stores, dict) + or not stores.keys() >= STORES + or any( + name not in STORES and not re.fullmatch(r"extra-[a-z]+-[0-9]+", name) for name in stores + ) + or {name + ".tar" for name in stores} != {n for n in files if n.endswith(".tar")} + ): _invalid("Storage inventory and archives disagree; obtain an intact snapshot.") before = {name: _signature(path) for name, path in actual.items()} for name, path in actual.items(): record = files[name] - if (not _safe_name(name) or not isinstance(record, dict) - or record.keys() != {"size", "sha256"} - or type(record.get("size")) is not int or record["size"] < 0 - or not re.fullmatch(r"[0-9a-f]{64}", str(record.get("sha256", ""))) - or before[name][2] != record["size"] or _hash(path) != record["sha256"]): + if ( + not _safe_name(name) + or not isinstance(record, dict) + or record.keys() != {"size", "sha256"} + or type(record.get("size")) is not int + or record["size"] < 0 + or not re.fullmatch(r"[0-9a-f]{64}", str(record.get("sha256", ""))) + or before[name][2] != record["size"] + or _hash(path) != record["sha256"] + ): _invalid("File size or integrity hash differs; obtain an intact snapshot.") with (directory / "memorygate.dump").open("rb") as stream: if stream.read(5) != b"PGDMP": @@ -147,25 +191,43 @@ def verify_snapshot(directory: Path) -> dict: for name, path in actual.items(): if name.endswith(".tar"): _archive(path, {"pi.tar": "pi.db", "toolgate.tar": "toolgate.db"}.get(name)) - if (actual.keys() != _files(directory).keys() or _signature(manifest_path) != manifest_before - or any(_signature(path) != before[name] for name, path in actual.items())): + if ( + actual.keys() != _files(directory).keys() + or _signature(manifest_path) != manifest_before + or any(_signature(path) != before[name] for name, path in actual.items()) + ): _invalid("Snapshot changed during verification; wait for backup to finish and retry.") - return {"created_at": created_at, "status": "verified", "verified_at": time.time(), - "verification": "manifest-files-and-archives", "restore_status": "not_tested", - "deletion_ledger": manifest.get("deletion_ledger", "unknown"), - "execution_journal": manifest.get("execution_journal", "unknown")} + return { + "created_at": created_at, + "status": "verified", + "verified_at": time.time(), + "verification": "manifest-files-and-archives", + "restore_status": "not_tested", + "deletion_ledger": manifest.get("deletion_ledger", "unknown"), + "execution_journal": manifest.get("execution_journal", "unknown"), + } def backup_status(root: Path) -> dict: - result = {"root": str(root), "status": "unknown", "latest": None, "results": [], - "rejected": [], "checked_at": time.time(), "verified_count": 0, - "rejected_count": 0} + result = { + "root": str(root), + "status": "unknown", + "latest": None, + "results": [], + "rejected": [], + "checked_at": time.time(), + "verified_count": 0, + "rejected_count": 0, + } try: entries = sorted(root.iterdir(), key=lambda item: item.name) except OSError: - return {**result, "status": "unavailable", - "reason": "Backup root cannot be read. Mount backups at /backups and set " - "SYSTEMGATE_BACKUP_ROOT=/backups inside the container."} + return { + **result, + "status": "unavailable", + "reason": "Backup root cannot be read. Mount backups at /backups and set " + "SYSTEMGATE_BACKUP_ROOT=/backups inside the container.", + } verified, rejected = [], [] for directory in entries: if not directory.is_dir() and not directory.is_symlink(): @@ -177,11 +239,22 @@ def backup_status(root: Path) -> dict: rejected.append({**row, "status": exc.status, "reason": exc.reason}) except (OSError, ValueError, tarfile.TarError): # Parser and OS errors can contain credentials or recovered text. - rejected.append({**row, "status": "unverifiable", - "reason": "Snapshot could not be verified. Check permissions " - "and run the recovery verifier on the original."}) + rejected.append( + { + **row, + "status": "unverifiable", + "reason": "Snapshot could not be verified. Check permissions " + "and run the recovery verifier on the original.", + } + ) verified.sort(key=lambda row: (row["created_at"], row["name"]), reverse=True) - return {**result, "status": "degraded" if rejected else ("ok" if verified else "empty"), - "latest": verified[0] if verified else None, "results": verified[:20], - "rejected": rejected[:20], "verified_count": len(verified), - "rejected_count": len(rejected), "checked_at": time.time()} + return { + **result, + "status": "degraded" if rejected else ("ok" if verified else "empty"), + "latest": verified[0] if verified else None, + "results": verified[:20], + "rejected": rejected[:20], + "verified_count": len(verified), + "rejected_count": len(rejected), + "checked_at": time.time(), + } diff --git a/systemgate/main.py b/systemgate/main.py index 97ab242..a7e6c34 100644 --- a/systemgate/main.py +++ b/systemgate/main.py @@ -63,7 +63,9 @@ def _ensure_key_hash(settings: Settings) -> None: path.chmod(0o600) -def require_admin(request: Request, x_systemgate_key: str | None = Header(None, alias="X-SystemGate-Key")) -> str: +def require_admin( + request: Request, x_systemgate_key: str | None = Header(None, alias="X-SystemGate-Key") +) -> str: path = _secret_path(request.app.state.settings) encoded = path.read_text(encoding="utf-8").strip() if path.exists() else "" if not verify_key(x_systemgate_key, encoded): @@ -141,7 +143,10 @@ def vitals(): temps = {} try: sensors = psutil.sensors_temperatures - temps = {name: [entry._asdict() for entry in values] for name, values in sensors(fahrenheit=False).items()} + temps = { + name: [entry._asdict() for entry in values] + for name, values in sensors(fahrenheit=False).items() + } except (AttributeError, OSError): temps = {} # Which filesystem is measured is a deployment detail, so report it rather @@ -203,7 +208,28 @@ def _safe_addr(addr: Any) -> str: scope = "loopback" elif ip.startswith("100."): scope = "tailscale" - elif ip.startswith(("10.", "192.168.", "172.16.", "172.17.", "172.18.", "172.19.", "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", "172.25.", "172.26.", "172.27.", "172.28.", "172.29.", "172.30.", "172.31.")): + elif ip.startswith( + ( + "10.", + "192.168.", + "172.16.", + "172.17.", + "172.18.", + "172.19.", + "172.20.", + "172.21.", + "172.22.", + "172.23.", + "172.24.", + "172.25.", + "172.26.", + "172.27.", + "172.28.", + "172.29.", + "172.30.", + "172.31.", + ) + ): scope = "private-lan" else: scope = "public-or-host" @@ -275,7 +301,13 @@ def processes(limit: int = 20): rows.append({field: proc.info.get(field) for field in _PROCESS_FIELDS}) except (psutil.NoSuchProcess, psutil.AccessDenied): continue - rows.sort(key=lambda item: (float(item.get("cpu_percent") or 0), float(item.get("memory_percent") or 0)), reverse=True) + rows.sort( + key=lambda item: ( + float(item.get("cpu_percent") or 0), + float(item.get("memory_percent") or 0), + ), + reverse=True, + ) return {"results": rows[:limit]} @@ -299,7 +331,10 @@ def logs_errors(): def _run_summary(command: list[str]) -> dict[str, Any]: try: result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=12) - return {"ok": result.returncode == 0, "output": _bounded(result.stdout or result.stderr, 2000)} + return { + "ok": result.returncode == 0, + "output": _bounded(result.stdout or result.stderr, 2000), + } except Exception as exc: return {"ok": False, "output": type(exc).__name__} diff --git a/systemgate/runtime.py b/systemgate/runtime.py index bff848f..d42bc81 100644 --- a/systemgate/runtime.py +++ b/systemgate/runtime.py @@ -23,11 +23,7 @@ def _text(value, limit=160): def _number(value): - return ( - value - if type(value) in (int, float) and math.isfinite(value) and value >= 0 - else None - ) + return value if type(value) in (int, float) and math.isfinite(value) and value >= 0 else None def _identity(pid, created): @@ -48,9 +44,7 @@ def error(section, code, unavailable=False): value = sections[section] if code not in value["errors"]: value["errors"].append(code) - value["status"] = ( - "unavailable" if unavailable and not value["results"] else "partial" - ) + value["status"] = "unavailable" if unavailable and not value["results"] else "partial" def add(section, row): value = sections[section] @@ -89,9 +83,7 @@ def add(section, row): "createdAt": created, "name": _text(info.get("name")), "status": _text(info.get("status"), 40) or "unknown", - "memoryBytes": _number( - getattr(info.get("memory_info"), "rss", None) - ), + "memoryBytes": _number(getattr(info.get("memory_info"), "rss", None)), "cpuPercent": None, "command": None, "user": None, @@ -113,14 +105,8 @@ def add(section, row): if index >= MAX_SCAN: sections["ports"]["truncated"] = True break - protocol = {socket.SOCK_STREAM: "tcp", socket.SOCK_DGRAM: "udp"}.get( - conn.type - ) - if ( - not protocol - or (protocol == "tcp" and conn.status != "LISTEN") - or not conn.laddr - ): + protocol = {socket.SOCK_STREAM: "tcp", socket.SOCK_DGRAM: "udp"}.get(conn.type) + if not protocol or (protocol == "tcp" and conn.status != "LISTEN") or not conn.laddr: continue address, port = _text(conn.laddr.ip, 64), conn.laddr.port process_id = None @@ -194,10 +180,7 @@ def add(section, row): binding.get("PublicPort"), binding.get("PrivatePort"), ) - if any( - type(p) is not int or not 1 <= p <= 65535 - for p in (public, private) - ): + if any(type(p) is not int or not 1 <= p <= 65535 for p in (public, private)): error("ports", "invalid_container_binding") continue key = f"{identity}:{address}:{public}:{private}:{protocol}" @@ -240,14 +223,10 @@ def add(section, row): "collectionSeconds": max(0, finished - started), "source": { "procfs": procfs, - "processScope": "configured-procfs" - if procfs != "/proc" - else "collector-namespace", + "processScope": "configured-procfs" if procfs != "/proc" else "collector-namespace", "networkScope": "collector-namespace", "containerScope": ( - "configured-container-source" - if container_source_configured - else "unavailable" + "configured-container-source" if container_source_configured else "unavailable" ), }, "status": "partial" diff --git a/tests/test_auth.py b/tests/test_auth.py index 0ffdd6a..bcceb63 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -11,7 +11,9 @@ def slow(monkeypatch): monkeypatch.setattr(auth, "_matched", {}) calls = [] real = auth.hashlib.pbkdf2_hmac - monkeypatch.setattr(auth.hashlib, "pbkdf2_hmac", lambda *args: calls.append(args) or real(*args)) + monkeypatch.setattr( + auth.hashlib, "pbkdf2_hmac", lambda *args: calls.append(args) or real(*args) + ) return calls diff --git a/tests/test_backups.py b/tests/test_backups.py index b945036..8c3ba1d 100644 --- a/tests/test_backups.py +++ b/tests/test_backups.py @@ -14,11 +14,28 @@ # Independently specified producer contract, so changing the consumer's required # inventory cannot silently change the fixtures to agree with the bug. STORES = {"pi", "toolgate", "systemgate", "qdrant", "ollama", "memorygate-backups"} -SERVICES = {"pi", "toolgate", "memorygate", "systemgate", "embeddings", "postgres", - "qdrant", "ollama", "searxng"} -FILES = {*(name + ".tar" for name in STORES), "memorygate.dump", "postgres-globals.sql", - "config/env", "config/versions.env", "config/docker-compose.yml", "config/compose.json", - "config/runtime.json", "memorygate/runtime-token"} +SERVICES = { + "pi", + "toolgate", + "memorygate", + "systemgate", + "embeddings", + "postgres", + "qdrant", + "ollama", + "searxng", +} +FILES = { + *(name + ".tar" for name in STORES), + "memorygate.dump", + "postgres-globals.sql", + "config/env", + "config/versions.env", + "config/docker-compose.yml", + "config/compose.json", + "config/runtime.json", + "memorygate/runtime-token", +} def snapshot(root, name="snapshot-a", created="2026-01-01T00:00:00+00:00"): @@ -35,11 +52,18 @@ def snapshot(root, name="snapshot-a", created="2026-01-01T00:00:00+00:00"): member.size = len(value) archive.addfile(member, io.BytesIO(value)) else: - path.write_bytes(b"PGDMPfixture" if filename == "memorygate.dump" else b"secret-placeholder") - manifest = {"format": "conker-snapshot-1", "status": "complete", "created_at": created, - "images": {name: {"id": "sha256:" + "a" * 64} for name in SERVICES}, - "stores": {name: {} for name in STORES}, "deletion_ledger": "unavailable", - "execution_journal": "unavailable"} + path.write_bytes( + b"PGDMPfixture" if filename == "memorygate.dump" else b"secret-placeholder" + ) + manifest = { + "format": "conker-snapshot-1", + "status": "complete", + "created_at": created, + "images": {name: {"id": "sha256:" + "a" * 64} for name in SERVICES}, + "stores": {name: {} for name in STORES}, + "deletion_ledger": "unavailable", + "execution_journal": "unavailable", + } save_manifest(directory, manifest) return directory @@ -49,8 +73,11 @@ def save_manifest(directory, manifest=None): manifest = json.loads((directory / "manifest.json").read_text()) manifest["files"] = { path.relative_to(directory).as_posix(): { - "size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), - } for path in directory.rglob("*") if path.is_file() and path.name != "manifest.json" + "size": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for path in directory.rglob("*") + if path.is_file() and path.name != "manifest.json" } (directory / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") @@ -83,8 +110,21 @@ def test_tampering_same_size_and_mtime_cannot_remain_verified(tmp_path): assert result["rejected"][0]["status"] == "invalid" -@pytest.mark.parametrize("change", ["missing", "extra", "partial", "format", "images", - "stores", "timestamp", "malformed", "postgres", "archive"]) +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "partial", + "format", + "images", + "stores", + "timestamp", + "malformed", + "postgres", + "archive", + ], +) def test_invalid_snapshots_are_never_recovery_points(tmp_path, change): directory = snapshot(tmp_path) path = directory / "manifest.json" @@ -102,9 +142,13 @@ def test_invalid_snapshots_are_never_recovery_points(tmp_path, change): elif change == "malformed": path.write_text('{"secret-do-not-print":') else: - field, value = {"partial": ("status", "writing"), "format": ("format", "future"), - "images": ("images", {}), "stores": ("stores", {}), - "timestamp": ("created_at", "invalid")}[change] + field, value = { + "partial": ("status", "writing"), + "format": ("format", "future"), + "images": ("images", {}), + "stores": ("stores", {}), + "timestamp": ("created_at", "invalid"), + }[change] manifest[field] = value path.write_text(json.dumps(manifest)) result = backups.backup_status(tmp_path) @@ -113,8 +157,10 @@ def test_invalid_snapshots_are_never_recovery_points(tmp_path, change): assert "secret-do-not-print" not in json.dumps(result) -@pytest.mark.parametrize("member_name,kind", [("../escape", "file"), ("pi.db", "link"), - ("pi.db", "dir"), ("missing.db", "file")]) +@pytest.mark.parametrize( + "member_name,kind", + [("../escape", "file"), ("pi.db", "link"), ("pi.db", "dir"), ("missing.db", "file")], +) def test_even_hash_matching_unsafe_archives_fail(tmp_path, member_name, kind): directory = snapshot(tmp_path) with tarfile.open(directory / "pi.tar", "w") as archive: @@ -145,7 +191,7 @@ def test_a_truncated_archive_is_never_a_recovery_point(tmp_path): data = buffer.getvalue() # pi.db's 5016 bytes occupy 5120 padded, so the cut lands 100 bytes into the # second member's payload. - (directory / "pi.tar").write_bytes(data[:512 + 5120 + 512 + 100]) + (directory / "pi.tar").write_bytes(data[: 512 + 5120 + 512 + 100]) save_manifest(directory) result = backups.backup_status(tmp_path) assert result["latest"] is None @@ -181,8 +227,9 @@ def record(archive, member): def test_read_errors_are_unverifiable_without_leaking_payload(tmp_path, monkeypatch): snapshot(tmp_path) - monkeypatch.setattr(backups, "_hash", lambda path: (_ for _ in ()).throw( - PermissionError("SECRET-IN-ERROR"))) + monkeypatch.setattr( + backups, "_hash", lambda path: (_ for _ in ()).throw(PermissionError("SECRET-IN-ERROR")) + ) result = backups.backup_status(tmp_path) assert result["latest"] is None assert result["rejected"][0]["status"] == "unverifiable" @@ -225,6 +272,7 @@ def test_endpoint_reads_the_configured_mount_and_requires_auth(tmp_path, monkeyp monkeypatch.setenv("SYSTEMGATE_BACKUP_ROOT", str(tmp_path / "backups")) snapshot(tmp_path / "backups") from systemgate.main import app + with TestClient(app) as client: assert client.get("/backups").status_code == 401 body = client.get("/backups", headers={"X-SystemGate-Key": "backup-test-key-1234"}).json() diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py index 9bdf58c..89799dd 100644 --- a/tests/test_endpoints.py +++ b/tests/test_endpoints.py @@ -13,6 +13,7 @@ def make_client(tmp_path, monkeypatch): monkeypatch.setenv("SYSTEMGATE_DATA_DIR", str(tmp_path / "data")) monkeypatch.setenv("SYSTEMGATE_BACKUP_ROOT", str(tmp_path / "backups")) from systemgate.main import app + return TestClient(app) @@ -49,11 +50,14 @@ def test_health_leaks_nothing_to_an_unauthenticated_caller(tmp_path, monkeypatch def test_vitals_declares_which_machine_it_measured(tmp_path, monkeypatch): """Host or container is a deployment detail; the caller must not guess.""" import systemgate.main as main + monkeypatch.setattr(main.psutil, "cpu_percent", lambda interval=0: 1.0) monkeypatch.setattr(main.psutil, "cpu_count", lambda: 1) monkeypatch.setattr(main.psutil, "virtual_memory", lambda: Mock(_asdict=lambda: {})) monkeypatch.setattr(main.psutil, "disk_usage", lambda path: Mock(_asdict=lambda: {})) - monkeypatch.setattr(main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False) + monkeypatch.setattr( + main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False + ) monkeypatch.setattr(main.psutil, "PROCFS_PATH", "/proc", raising=False) with make_client(tmp_path, monkeypatch) as client: @@ -111,11 +115,18 @@ def test_redirecting_procfs_actually_changes_what_is_read(tmp_path): def test_vitals_with_mocked_psutil(tmp_path, monkeypatch): import systemgate.main as main + monkeypatch.setattr(main.psutil, "cpu_percent", lambda interval=0: 12.5) monkeypatch.setattr(main.psutil, "cpu_count", lambda: 8) - monkeypatch.setattr(main.psutil, "virtual_memory", lambda: Mock(_asdict=lambda: {"total": 100, "available": 50})) - monkeypatch.setattr(main.psutil, "disk_usage", lambda path: Mock(_asdict=lambda: {"total": 200, "free": 100})) - monkeypatch.setattr(main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False) + monkeypatch.setattr( + main.psutil, "virtual_memory", lambda: Mock(_asdict=lambda: {"total": 100, "available": 50}) + ) + monkeypatch.setattr( + main.psutil, "disk_usage", lambda path: Mock(_asdict=lambda: {"total": 200, "free": 100}) + ) + monkeypatch.setattr( + main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False + ) with make_client(tmp_path, monkeypatch) as client: data = client.get("/vitals", headers=headers()).json() @@ -127,7 +138,9 @@ def test_vitals_survive_an_unreadable_disk_path(tmp_path, monkeypatch): import systemgate.main as main monkeypatch.setenv("SYSTEMGATE_DISK_PATH", str(tmp_path / "missing")) - monkeypatch.setattr(main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False) + monkeypatch.setattr( + main.psutil, "sensors_temperatures", lambda fahrenheit=False: {}, raising=False + ) with make_client(tmp_path, monkeypatch) as client: response = client.get("/vitals", headers=headers()) @@ -187,7 +200,17 @@ def test_processes_logs_packages_and_backups_are_bounded(tmp_path, monkeypatch): backup_root.mkdir() (backup_root / "20260101T000000Z").mkdir() import systemgate.main as main - proc = Mock(info={"pid": 1, "name": "python", "username": "me", "cpu_percent": 1.0, "memory_percent": 2.0, "cmdline": ["python", "app.py"]}) + + proc = Mock( + info={ + "pid": 1, + "name": "python", + "username": "me", + "cpu_percent": 1.0, + "memory_percent": 2.0, + "cmdline": ["python", "app.py"], + } + ) monkeypatch.setattr(main.psutil, "process_iter", lambda fields: [proc]) monkeypatch.setattr(main, "_journal_errors", lambda: "bounded errors") monkeypatch.setattr(main, "_run_summary", lambda command: {"ok": True, "output": "[]"}) @@ -204,9 +227,17 @@ def test_processes_logs_packages_and_backups_are_bounded(tmp_path, monkeypatch): def test_processes_never_return_command_lines(tmp_path, monkeypatch): import systemgate.main as main + asked = [] - info = {"pid": 7, "name": "app", "username": "me", "cpu_percent": 0.0, "memory_percent": 0.0, - "cmdline": ["app", "--password", "hunter2"], "environ": {"TOKEN": "hunter2"}} + info = { + "pid": 7, + "name": "app", + "username": "me", + "cpu_percent": 0.0, + "memory_percent": 0.0, + "cmdline": ["app", "--password", "hunter2"], + "environ": {"TOKEN": "hunter2"}, + } def process_iter(fields): asked.extend(fields) @@ -249,12 +280,17 @@ def run(*args, **kwargs): def test_startup_primes_the_key_check_so_wrong_keys_stay_cheap(tmp_path, monkeypatch): from systemgate import auth + monkeypatch.setattr(auth, "_matched", {}) with make_client(tmp_path, monkeypatch) as client: slow = [] real = auth.hashlib.pbkdf2_hmac - monkeypatch.setattr(auth.hashlib, "pbkdf2_hmac", lambda *args: slow.append(args) or real(*args)) - assert client.get("/vitals", headers={"X-SystemGate-Key": "wrong-key-0000"}).status_code == 401 + monkeypatch.setattr( + auth.hashlib, "pbkdf2_hmac", lambda *args: slow.append(args) or real(*args) + ) + assert ( + client.get("/vitals", headers={"X-SystemGate-Key": "wrong-key-0000"}).status_code == 401 + ) assert client.get("/processes", headers=headers()).status_code == 200 assert slow == [] diff --git a/tests/test_module_contract.py b/tests/test_module_contract.py index 4be8b0f..2624534 100644 --- a/tests/test_module_contract.py +++ b/tests/test_module_contract.py @@ -15,6 +15,7 @@ Skips when nothing is listening. CI brings the stack up first. Contract: docs/module-contract.md in the conker repository. """ + from __future__ import annotations import os @@ -92,7 +93,9 @@ def test_reason_never_leaks_to_an_unauthenticated_caller(health): reason = str(check.get("reason", "")) assert not reason.startswith("/"), f"{name} reason is an absolute path: {reason!r}" assert " /" not in reason, f"{name} reason embeds an absolute path: {reason!r}" - assert not re.search(r"[A-Za-z]:\\\\", reason), f"{name} reason embeds a Windows path: {reason!r}" + assert not re.search(r"[A-Za-z]:\\\\", reason), ( + f"{name} reason embeds a Windows path: {reason!r}" + ) assert "://" not in reason, f"{name} reason embeds a URL: {reason!r}" assert "Traceback" not in reason assert len(reason) <= 120, f"{name} reason is too long to be coarse: {reason!r}" diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 0b0b6b3..3866884 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -117,9 +117,7 @@ def test_bounds_and_pid_reuse(): ps.process_iter.return_value = [NS(info={**original, "pid": pid}) for pid in range(42, 47)] value = collect(ps, lambda: client, procfs="/proc", limit=1) assert value["processes"]["truncated"] and value["ports"]["truncated"] - assert all( - len(value[s]["results"]) <= 1 for s in ("processes", "ports", "containers") - ) + assert all(len(value[s]["results"]) <= 1 for s in ("processes", "ports", "containers")) ps.Process.return_value.create_time.return_value = 999 value = collect(ps, lambda: client, procfs="/proc") assert value["processes"]["results"] == [] @@ -133,10 +131,7 @@ def test_total_failure_is_not_empty_success(): ps.net_connections.side_effect = PermissionError("private") client.containers.list.side_effect = PermissionError("private") value = collect(ps, lambda: client, procfs="/proc") - assert all( - value[s]["status"] == "unavailable" - for s in ("processes", "containers", "ports") - ) + assert all(value[s]["status"] == "unavailable" for s in ("processes", "containers", "ports")) assert value["status"] == "partial" and "private" not in str(value) @@ -159,9 +154,7 @@ def test_route_owner_auth_and_no_mutations(tmp_path, monkeypatch): assert response.status_code == 200 and response.json()["mode"] == "observed" assert response.json()["containers"]["status"] == "unavailable" assert ( - api.post( - "/runtime", headers={"X-SystemGate-Key": "test-runtime-key"} - ).status_code + api.post("/runtime", headers={"X-SystemGate-Key": "test-runtime-key"}).status_code == 405 ) diff --git a/type-baseline.txt b/type-baseline.txt new file mode 100644 index 0000000..5217211 --- /dev/null +++ b/type-baseline.txt @@ -0,0 +1,3 @@ +# pyright basic-mode errors per file when the type check arrived; counts may only shrink. +# Lower or delete a line when its file improves (scripts/check_types.py says which). +systemgate/backups.py 1