From e80007543e50c514404d783ffab317a94ca95b4d Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 19:11:07 -0700 Subject: [PATCH 1/5] topology: Name CPUs by MPIDR on arm64 Multikernel names a CPU by its physical ID, the APIC id on x86, and kerf reads those from /proc/cpuinfo. On arm64 the ID is the MPIDR affinity value and /proc/cpuinfo does not have it, so "kerf init" stopped at "Could not read APIC IDs" before doing anything. Read the MPIDR from the CPU's device tree node where the host booted from one, and from the MADT's GICC entries, matched through the processor UID of the CPU's firmware node, on a UEFI/ACPI host. logical_to_physical() picks whichever the machine has, and both the CPU validation of init and the NUMA placement use it. Unlike the /proc/cpuinfo entry on x86, the sysfs files stay around for a CPU that the pool has taken. Loading and starting an instance already had the arm64 syscall numbers, and an Image file goes to kexec_file_load() as it is. The rdtsc module only exists on x86, where it remains a dependency; exec merely prints a timestamp with it. Checked against QEMU virt guests booted from a device tree and from UEFI/ACPI, before and after CPUs were moved into the pool. kerf itself has not been run on arm64 yet. Signed-off-by: Cong Wang --- src/kerf/init/main.py | 33 +++--------- src/kerf/topology.py | 116 +++++++++++++++++++++++++++++++++++++++-- tests/test_topology.py | 72 ++++++++++++++++++++++++- 3 files changed, 188 insertions(+), 33 deletions(-) diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index d3b343e..fae71fc 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -54,7 +54,7 @@ from ..devices import default_alias, is_partition, pci_node_name, split_alias from ..resources import get_busy_chunks_from_iomem from ..runtime import DeviceTreeManager -from ..topology import cpu_numa_nodes, node_for_cpus +from ..topology import cpu_numa_nodes, logical_to_physical, node_for_cpus MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel" @@ -333,31 +333,10 @@ def get_total_cpus_from_system() -> Optional[int]: def get_valid_apic_ids_from_system() -> Optional[set]: """ - Get set of valid APIC IDs from the system via /proc/cpuinfo. - Returns set of valid APIC IDs or None if not available. + Get the physical IDs of the system's CPUs: APIC ids on x86, MPIDRs on arm64. + Returns the set of valid IDs or None if not available. """ - try: - cpuinfo_path = Path('/proc/cpuinfo') - if not cpuinfo_path.exists(): - return None - - apic_ids = set() - with open(cpuinfo_path, 'r', encoding='utf-8') as f: - for line in f: - if line.startswith('apicid'): - parts = line.split(':') - if len(parts) == 2: - try: - apic_id = int(parts[1].strip()) - apic_ids.add(apic_id) - except ValueError: - pass - - return apic_ids if apic_ids else None - except (OSError, IOError): - pass - - return None + return set(logical_to_physical().values()) or None _NODE_SPEC = re.compile(r"^(.+)@(.*)$") @@ -631,8 +610,8 @@ def build_baseline_from_cmdline( valid_apic_ids = get_valid_apic_ids_from_system() if valid_apic_ids is None: raise KernelInterfaceError( - "Could not read APIC IDs from /proc/cpuinfo. " - "Ensure the system exposes CPU topology information." + "Could not read the physical CPU IDs (APIC ids from /proc/cpuinfo, " + "or MPIDRs from /sys/devices/system/cpu on arm64)." ) # The host stops listing a CPU in /proc/cpuinfo once the pool takes it, diff --git a/src/kerf/topology.py b/src/kerf/topology.py index 1537b16..eccba20 100644 --- a/src/kerf/topology.py +++ b/src/kerf/topology.py @@ -12,14 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Host NUMA topology, as sysfs and /proc/cpuinfo describe it.""" +""" +Host CPU topology, as sysfs and /proc/cpuinfo describe it. +Multikernel names CPUs by their physical IDs, the one thing all kernels on +a machine agree on: APIC ids on x86, MPIDR affinity values on arm64. +""" + +import os import re +import struct from pathlib import Path from typing import Dict, Iterable, List, Optional NODE_ROOT = "/sys/devices/system/node" CPUINFO = "/proc/cpuinfo" +CPU_ROOT = "/sys/devices/system/cpu" +MADT = "/sys/firmware/acpi/tables/APIC" + +# ACPI MADT: the table header, and the GICC entry that describes an arm64 CPU +MADT_HEADER_SIZE = 44 +MADT_GICC = 0x0B +MADT_GICC_UID = 8 +MADT_GICC_MPIDR = 68 +MADT_GICC_ENABLED = 1 _NODE_DIR = re.compile(r"^node(\d+)$") @@ -73,6 +89,89 @@ def logical_to_apic(cpuinfo_path: str = CPUINFO) -> Dict[int, int]: return mapping +def _madt_mpidrs(madt_path: str) -> Dict[int, int]: + """Map ACPI processor UID to MPIDR from the MADT's GICC entries.""" + try: + with open(madt_path, "rb") as f: + madt = f.read() + except OSError: + return {} + + mpidr_of: Dict[int, int] = {} + offset = MADT_HEADER_SIZE + while offset + 2 <= len(madt): + entry_type, length = madt[offset], madt[offset + 1] + if length < 2: + break + if entry_type == MADT_GICC and length >= MADT_GICC_MPIDR + 8: + uid, flags = struct.unpack_from(" Dict[int, int]: + """ + Map logical CPU number to MPIDR affinity on arm64. + + /proc/cpuinfo does not carry it. A CPU's device tree node does, in its + reg, and on an ACPI host the MADT does, keyed by the processor UID that + the CPU's firmware node reports. Both stay in sysfs for a CPU that was + given to the pool, unlike its /proc/cpuinfo entry on x86. + + Args: + cpu_root: Directory to read instead of /sys/devices/system/cpu + madt_path: File to read instead of /sys/firmware/acpi/tables/APIC + + Returns: + Mapping of logical CPU number to MPIDR, empty if unreadable + """ + mapping: Dict[int, int] = {} + madt: Optional[Dict[int, int]] = None + try: + entries = os.listdir(cpu_root) + except OSError: + return {} + + for entry in entries: + match = re.fullmatch(r"cpu(\d+)", entry) + if not match: + continue + cpu, cpu_dir = int(match.group(1)), os.path.join(cpu_root, entry) + try: + with open(os.path.join(cpu_dir, "of_node", "reg"), "rb") as f: + reg = f.read() + if len(reg) in (4, 8): + mapping[cpu] = int.from_bytes(reg, "big") + continue + except OSError: + pass + try: + with open(os.path.join(cpu_dir, "firmware_node", "uid"), "r", encoding="utf-8") as f: + uid = int(f.read().strip()) + except (OSError, ValueError): + continue + if madt is None: + madt = _madt_mpidrs(madt_path) + if uid in madt: + mapping[cpu] = madt[uid] + return mapping + + +def logical_to_physical( + cpuinfo_path: str = CPUINFO, cpu_root: str = CPU_ROOT, madt_path: str = MADT +) -> Dict[int, int]: + """ + Map logical CPU number to the physical ID multikernel names the CPU by. + + Returns: + APIC ids where /proc/cpuinfo has them, MPIDRs otherwise; empty if + neither can be read + """ + return logical_to_apic(cpuinfo_path) or logical_to_mpidr(cpu_root, madt_path) + + def _int_or_none(value: str) -> Optional[int]: try: return int(value) @@ -80,18 +179,25 @@ def _int_or_none(value: str) -> Optional[int]: return None -def cpu_numa_nodes(node_root: str = NODE_ROOT, cpuinfo_path: str = CPUINFO) -> Dict[int, int]: +def cpu_numa_nodes( + node_root: str = NODE_ROOT, + cpuinfo_path: str = CPUINFO, + cpu_root: str = CPU_ROOT, + madt_path: str = MADT, +) -> Dict[int, int]: """ - Map APIC id to NUMA node for every CPU the host still reports. + Map physical CPU id to NUMA node for every CPU the host still reports. Args: node_root: Path to read instead of /sys/devices/system/node cpuinfo_path: Path to read instead of /proc/cpuinfo + cpu_root: Path to read instead of /sys/devices/system/cpu + madt_path: Path to read instead of /sys/firmware/acpi/tables/APIC Returns: - Mapping of APIC id to NUMA node, empty if the topology is unreadable + Mapping of physical CPU id to NUMA node, empty if the topology is unreadable """ - apic_of = logical_to_apic(cpuinfo_path) + apic_of = logical_to_physical(cpuinfo_path, cpu_root, madt_path) if not apic_of: return {} diff --git a/tests/test_topology.py b/tests/test_topology.py index 413d8fd..4259820 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -14,7 +14,13 @@ """Reading the host NUMA topology out of sysfs and /proc/cpuinfo.""" -from kerf.topology import cpu_numa_nodes, logical_to_apic, node_for_cpus +from kerf.topology import ( + cpu_numa_nodes, + logical_to_apic, + logical_to_mpidr, + logical_to_physical, + node_for_cpus, +) def _cpuinfo(tmp_path, apic_of_cpu): @@ -104,3 +110,67 @@ def test_memoryless_node_with_no_cpus_is_skipped(tmp_path): node_root = _node_root(tmp_path, {0: "0-1", 1: ""}) assert cpu_numa_nodes(node_root, cpuinfo) == {0: 0, 2: 0} + + +def _arm64_cpu_root(tmp_path, cpus): + """cpus: logical number -> ("dt", reg bytes) or ("acpi", uid)""" + root = tmp_path / "cpu" + root.mkdir() + (root / "cpufreq").mkdir() + for cpu, (kind, value) in cpus.items(): + node = root / f"cpu{cpu}" / ("of_node" if kind == "dt" else "firmware_node") + node.mkdir(parents=True) + if kind == "dt": + (node / "reg").write_bytes(value) + else: + (node / "uid").write_text(f"{value}\n", encoding="utf-8") + return str(root) + + +def _madt(tmp_path, gicc): + """gicc: list of (uid, mpidr, enabled)""" + table = bytearray(44) + table[0:4] = b"APIC" + for uid, mpidr, enabled in gicc: + entry = bytearray(80) + entry[0], entry[1] = 0x0B, 80 + entry[8:12] = uid.to_bytes(4, "little") + entry[12:16] = (1 if enabled else 0).to_bytes(4, "little") + entry[68:76] = mpidr.to_bytes(8, "little") + table += entry + table += bytes([0x0C, 24]) + bytes(22) # a GIC distributor entry to skip + path = tmp_path / "APIC" + path.write_bytes(bytes(table)) + return str(path) + + +def test_arm64_dt_host_reads_mpidr_from_cpu_nodes(tmp_path): + cpu_root = _arm64_cpu_root( + tmp_path, + {0: ("dt", (0).to_bytes(4, "big")), 1: ("dt", (0x100).to_bytes(8, "big"))}, + ) + + assert logical_to_mpidr(cpu_root, str(tmp_path / "none")) == {0: 0, 1: 0x100} + + +def test_arm64_acpi_host_reads_mpidr_from_madt(tmp_path): + cpu_root = _arm64_cpu_root(tmp_path, {0: ("acpi", 7), 1: ("acpi", 9), 2: ("acpi", 11)}) + madt = _madt(tmp_path, [(9, 0x101, True), (7, 0x100, True), (11, 0x200, False)]) + + assert logical_to_mpidr(cpu_root, madt) == {0: 0x100, 1: 0x101} + + +def test_physical_ids_are_mpidrs_where_cpuinfo_has_no_apic_ids(tmp_path): + cpuinfo = tmp_path / "cpuinfo" + cpuinfo.write_text("processor\t: 0\nBogoMIPS\t: 125.00\n", encoding="utf-8") + cpu_root = _arm64_cpu_root(tmp_path, {0: ("dt", (2).to_bytes(8, "big"))}) + node_root = _node_root(tmp_path, {0: "0"}) + + assert logical_to_physical(str(cpuinfo), cpu_root, "") == {0: 2} + assert cpu_numa_nodes(node_root, str(cpuinfo), cpu_root, "") == {2: 0} + + +def test_physical_ids_are_apic_ids_on_x86(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2}) + + assert logical_to_physical(cpuinfo, str(tmp_path / "none"), "") == {0: 0, 1: 2} From 49146921bb8f3d07adc161a67b34626d94f580b2 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sun, 20 Sep 2026 20:25:06 -0700 Subject: [PATCH 2/5] load: Always boot an arm64 spawn with pseudo-NMIs On arm64 the host has one way to force down a spawn that no longer answers: a stop interrupt. A CPU that spins with interrupts off only takes it when its kernel runs with pseudo-NMIs, which is a boot parameter and off by default. Without it such a CPU can never be stopped, and the host refuses to reuse it until the machine reboots. That makes irqchip.gicv3_pseudo_nmi=1 a requirement of multikernel rather than a choice, so kerf load appends it to every spawn's command line on arm64. It goes last, where it wins over a command line that turns it off, and is not repeated when it is already in effect. The spawn kernel has to be built with CONFIG_ARM64_PSEUDO_NMI for it to mean anything. Signed-off-by: Cong Wang --- src/kerf/load/main.py | 45 +++++++++++++++++++++++++++++++++++++- tests/test_load_cmdline.py | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/test_load_cmdline.py diff --git a/src/kerf/load/main.py b/src/kerf/load/main.py index 7ce53cf..ada47c1 100644 --- a/src/kerf/load/main.py +++ b/src/kerf/load/main.py @@ -23,7 +23,7 @@ import sys from datetime import datetime, timezone from pathlib import Path -from typing import Optional +from typing import List, Optional import click @@ -102,6 +102,42 @@ def build_ip_param( return "ip=" + ":".join(parts) +# What a spawn kernel cannot do without, per architecture. On arm64 the host +# can only force a wedged spawn down with a stop interrupt, and that reaches +# a CPU that has interrupts off only when the spawn runs with pseudo-NMIs +# (it must also be built with CONFIG_ARM64_PSEUDO_NMI). +REQUIRED_KERNEL_PARAMS = { + "aarch64": ("irqchip.gicv3_pseudo_nmi=1",), + "arm64": ("irqchip.gicv3_pseudo_nmi=1",), +} + + +def required_kernel_params(cmdline_parts: List[str], arch: Optional[str] = None) -> List[str]: + """ + The parameters that must still be appended to a spawn's command line. + + The kernel honors the last setting of a parameter, so one that the + command line sets differently is appended again and wins; one whose + last setting already is the required one is left alone. + + Args: + cmdline_parts: The command line so far, as it will be joined + arch: Architecture to decide for instead of this machine's + + Returns: + Parameters to append, in order + """ + arch = (arch or platform.machine()).lower() + words = " ".join(cmdline_parts).split() + missing = [] + for param in REQUIRED_KERNEL_PARAMS.get(arch, ()): + key = param.split("=", 1)[0] + "=" + settings = [word for word in words if word.startswith(key)] + if not settings or settings[-1] != param: + missing.append(param) + return missing + + def get_kexec_file_load_syscall(): """Get the kexec_file_load syscall number for current architecture.""" arch = platform.machine().lower() @@ -510,6 +546,13 @@ def load( # pylint: disable=too-many-arguments,too-many-positional-arguments,to if verbose: click.echo(f"Console: console={console_device}") + # Last, so that it wins over anything given above + required = required_kernel_params(cmdline_parts) + if required: + cmdline_parts.extend(required) + if verbose: + click.echo(f"Required on this architecture: {' '.join(required)}") + cmdline_str = " ".join(cmdline_parts) if verbose: diff --git a/tests/test_load_cmdline.py b/tests/test_load_cmdline.py new file mode 100644 index 0000000..da84295 --- /dev/null +++ b/tests/test_load_cmdline.py @@ -0,0 +1,44 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kernel parameters kerf load adds to every spawn's command line.""" + +from kerf.load.main import required_kernel_params + +PSEUDO_NMI = "irqchip.gicv3_pseudo_nmi=1" + + +def test_arm64_spawn_always_gets_pseudo_nmis(): + assert required_kernel_params([], "aarch64") == [PSEUDO_NMI] + assert required_kernel_params(["root=/dev/vda console=mktty0"], "arm64") == [PSEUDO_NMI] + + +def test_x86_needs_nothing(): + assert not required_kernel_params(["root=/dev/sda1"], "x86_64") + + +def test_not_repeated_when_already_in_effect(): + assert not required_kernel_params([f"quiet {PSEUDO_NMI}", "console=mktty0"], "aarch64") + + +def test_wins_over_a_command_line_that_turns_it_off(): + parts = ["irqchip.gicv3_pseudo_nmi=0 quiet"] + + assert required_kernel_params(parts, "aarch64") == [PSEUDO_NMI] + + +def test_appended_again_when_turned_off_after_being_on(): + parts = [PSEUDO_NMI, "irqchip.gicv3_pseudo_nmi=0"] + + assert required_kernel_params(parts, "aarch64") == [PSEUDO_NMI] From 1641add80984557c1de194aedb65c11730ccc181 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 25 Sep 2026 11:59:28 -0700 Subject: [PATCH 3/5] Name a CPU's physical ID after the architecture in messages Every message about a CPU ID called it an APIC ID, which is the x86 name. On arm64 the ID is the MPIDR, so an arm64 user was told that their CPUs had "APIC IDs" such as 65536. Add cpu_id_name(), "MPIDR" on arm64 and "APIC ID" elsewhere, and use it in the errors, warnings and verbose output of init and create and in resource validation. The --cpus help texts are fixed strings, so they name both. Signed-off-by: Cong Wang --- src/kerf/create/main.py | 17 +++++++++-------- src/kerf/init/main.py | 18 +++++++++--------- src/kerf/resources.py | 9 +++++---- src/kerf/topology.py | 8 ++++++++ tests/test_topology.py | 9 +++++++++ 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/kerf/create/main.py b/src/kerf/create/main.py index 26a807c..26a2b85 100644 --- a/src/kerf/create/main.py +++ b/src/kerf/create/main.py @@ -35,6 +35,7 @@ get_available_cpus, ) from ..exceptions import ValidationError, KernelInterfaceError, ResourceError, ParseError +from ..topology import cpu_id_name def parse_cpu_spec(cpu_spec: str) -> List[int]: @@ -137,7 +138,7 @@ def _allocate_spread( numa_cpu_lists[numa_node_id] = sorted(numa_cpus) if not numa_cpu_lists: - raise ResourceError(f"No available APIC IDs in specified NUMA nodes: {numa_nodes}") + raise ResourceError(f"No available {cpu_id_name()}s in specified NUMA nodes: {numa_nodes}") allocated = [] numa_indices = {node_id: 0 for node_id in numa_cpu_lists} @@ -187,7 +188,7 @@ def _allocate_local( if len(numa_cpus) >= count: return sorted(numa_cpus[:count]) raise ResourceError( - f"Not enough APIC IDs in NUMA node {numa_node_id}: " + f"Not enough {cpu_id_name()}s in NUMA node {numa_node_id}: " f"requested {count}, but only {len(numa_cpus)} available" ) @@ -200,7 +201,7 @@ def _allocate_local( if len(numa_cpus) >= count: return sorted(numa_cpus[:count]) - raise ResourceError(f"No single NUMA node has {count} available APIC IDs for 'local' affinity") + raise ResourceError(f"No single NUMA node has {count} available {cpu_id_name()}s for 'local' affinity") def allocate_cpus_from_pool( @@ -236,11 +237,11 @@ def allocate_cpus_from_pool( if len(available) < count: if numa_nodes: raise ResourceError( - f"Not enough APIC IDs available in NUMA nodes {numa_nodes}: " + f"Not enough {cpu_id_name()}s available in NUMA nodes {numa_nodes}: " f"requested {count}, but only {len(available)} available" ) raise ResourceError( - f"Not enough APIC IDs available: requested {count}, " + f"Not enough {cpu_id_name()}s available: requested {count}, " f"but only {len(available)} available in pool" ) @@ -364,8 +365,8 @@ def dump_overlay_for_debug( @click.option( "--cpus", "-c", - help='Explicit APIC ID allocation (e.g., "128" for APIC ID 128, "128-134" for range, ' - '"128,130,132" for list). Use physical APIC IDs, not logical CPU numbers. ' + help='Explicit CPU allocation by physical ID, the APIC ID on x86 or the MPIDR on arm64 ' + '(e.g., "128", "128-134" for range, "128,130,132" for list), not logical CPU numbers. ' 'Mutually exclusive with --cpu-count', ) @click.option( @@ -561,7 +562,7 @@ def create( # pylint: disable=too-many-arguments,too-many-positional-arguments try: cpu_spec_value = parse_cpu_spec(cpus) except ValueError as e: - click.echo(f"Error: Invalid APIC ID specification '{cpus}': {e}", err=True) + click.echo(f"Error: Invalid {cpu_id_name()} specification '{cpus}': {e}", err=True) sys.exit(2) # Parse memory specification diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index fae71fc..ce11681 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -54,7 +54,7 @@ from ..devices import default_alias, is_partition, pci_node_name, split_alias from ..resources import get_busy_chunks_from_iomem from ..runtime import DeviceTreeManager -from ..topology import cpu_numa_nodes, logical_to_physical, node_for_cpus +from ..topology import cpu_id_name, cpu_numa_nodes, logical_to_physical, node_for_cpus MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel" @@ -621,8 +621,8 @@ def build_baseline_from_cmdline( invalid_cpus = set(cpu_list) - valid_apic_ids if invalid_cpus: raise ValueError( - f"Invalid APIC ID(s) specified: {sorted(invalid_cpus)}. " - f"Valid APIC IDs on this system: {sorted(valid_apic_ids)}" + f"Invalid {cpu_id_name()}(s) specified: {sorted(invalid_cpus)}. " + f"Valid {cpu_id_name()}s on this system: {sorted(valid_apic_ids)}" ) # Total CPUs is based on the max APIC ID + 1 for sizing purposes @@ -633,7 +633,7 @@ def build_baseline_from_cmdline( if 0 in available_cpus and len(host_reserved_cpus) == 0: if verbose: - click.echo("Warning: APIC ID 0 is in available list but no host-reserved CPUs. Moving APIC ID 0 to host-reserved.", err=True) + click.echo(f"Warning: {cpu_id_name()} 0 is in available list but no host-reserved CPUs. Moving {cpu_id_name()} 0 to host-reserved.", err=True) available_cpus.discard(0) host_reserved_cpus = [0] cpu_list = sorted(list(available_cpus)) @@ -644,10 +644,10 @@ def build_baseline_from_cmdline( total_bytes = sum(requested.values()) if verbose: - click.echo(f"Parsed APIC ID specification: {cpus}") - click.echo(f" Valid APIC IDs on system: {sorted(valid_apic_ids)}") - click.echo(f" Host-reserved APIC IDs: {host_reserved_cpus}") - click.echo(f" Available APIC IDs: {cpu_list}") + click.echo(f"Parsed {cpu_id_name()} specification: {cpus}") + click.echo(f" Valid {cpu_id_name()}s on system: {sorted(valid_apic_ids)}") + click.echo(f" Host-reserved {cpu_id_name()}s: {host_reserved_cpus}") + click.echo(f" Available {cpu_id_name()}s: {cpu_list}") click.echo("Requested pool memory:") for node, size in sorted(requested.items()): click.echo(f" node {node}: {size} bytes ({size / (1024**3):.2f} GB)") @@ -949,7 +949,7 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> @click.command() @click.pass_context @click.option('--input', '-i', help='Baseline DTB to replay, as written by "kerf dump". Mutually exclusive with --cpus, --memory and --devices.') -@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') +@click.option('--cpus', '-c', help='Physical CPU IDs for baseline, the APIC ID on x86 or the MPIDR on arm64 (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Not logical CPU numbers. Mutually exclusive with --input.') @click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, per-node "8GB@0,8GB@1", or "none" for no pool memory. Required with --cpus, mutually exclusive with --input.') @click.option('--devices', '-d', help='Host device names or PCI addresses to pool, each with an optional "=alias" the spawned kernels name the device by (comma-separated, e.g. "nvme0n1,enp9s0,0000:4f:01.0=nvme1"), or "none" for no devices. By default an NVMe keeps its controller name and a network device keeps its interface name. Mutually exclusive with --input.') @click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') diff --git a/src/kerf/resources.py b/src/kerf/resources.py index 81ab87b..7c1a212 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -28,6 +28,7 @@ from .models import GlobalDeviceTree, PoolMemoryRegion from .exceptions import ResourceError +from .topology import cpu_id_name IOMEM_PATH = "/proc/iomem" @@ -358,8 +359,8 @@ def validate_cpu_allocation( invalid_cpus = requested_set - hardware_cpus if invalid_cpus: raise ResourceError( - f"Invalid APIC IDs requested: {sorted(invalid_cpus)}. " - f"Available APIC IDs: {sorted(hardware_cpus)}" + f"Invalid {cpu_id_name()}s requested: {sorted(invalid_cpus)}. " + f"Available {cpu_id_name()}s: {sorted(hardware_cpus)}" ) # Check APIC IDs are available @@ -372,10 +373,10 @@ def validate_cpu_allocation( continue conflict_cpus = set(instance.resources.cpus) & unavailable if conflict_cpus: - conflicts.append(f"{instance.name} uses APIC IDs {sorted(conflict_cpus)}") + conflicts.append(f"{instance.name} uses {cpu_id_name()}s {sorted(conflict_cpus)}") conflict_msg = ", ".join(conflicts) if conflicts else "allocated to other instances" - raise ResourceError(f"APIC IDs {sorted(unavailable)} are not available ({conflict_msg})") + raise ResourceError(f"{cpu_id_name()}s {sorted(unavailable)} are not available ({conflict_msg})") def find_next_instance_id(tree: GlobalDeviceTree) -> int: diff --git a/src/kerf/topology.py b/src/kerf/topology.py index eccba20..304b9b0 100644 --- a/src/kerf/topology.py +++ b/src/kerf/topology.py @@ -20,6 +20,7 @@ """ import os +import platform import re import struct from pathlib import Path @@ -40,6 +41,13 @@ _NODE_DIR = re.compile(r"^node(\d+)$") +def cpu_id_name() -> str: + """What this machine calls a physical CPU ID, for messages.""" + if platform.machine().lower() in ("aarch64", "arm64"): + return "MPIDR" + return "APIC ID" + + def _parse_cpulist(text: str) -> List[int]: """Expand a sysfs cpulist ("0-3,8") into logical CPU numbers.""" cpus: List[int] = [] diff --git a/tests/test_topology.py b/tests/test_topology.py index 4259820..b586a1f 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -14,7 +14,10 @@ """Reading the host NUMA topology out of sysfs and /proc/cpuinfo.""" +import pytest + from kerf.topology import ( + cpu_id_name, cpu_numa_nodes, logical_to_apic, logical_to_mpidr, @@ -174,3 +177,9 @@ def test_physical_ids_are_apic_ids_on_x86(tmp_path): cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2}) assert logical_to_physical(cpuinfo, str(tmp_path / "none"), "") == {0: 0, 1: 2} + + +@pytest.mark.parametrize("machine, name", [("x86_64", "APIC ID"), ("aarch64", "MPIDR")]) +def test_messages_name_the_physical_id_of_the_architecture(monkeypatch, machine, name): + monkeypatch.setattr("platform.machine", lambda: machine) + assert cpu_id_name() == name From da11a276d11440125d00fd79b5796c6c19eff4a9 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 25 Sep 2026 12:00:49 -0700 Subject: [PATCH 4/5] init: Keep the host's boot CPU by its own physical ID init assumed that the host's boot CPU had physical ID 0. Where the pool request covered every CPU, it moved ID 0 back to the host. In every other case it passed the boot CPU to the kernel, which cannot take it into the pool. Physical ID 0 need not be the boot CPU, either: an arm64 server can boot Linux on MPIDR 0x10000 and have no CPU 0 at all, and x86 does not promise the BSP APIC ID 0. Read the boot CPU's physical ID from logical CPU 0 and always keep it with the host, with a warning whenever the request named it. The kernel now rejects a baseline that includes the boot CPU, so the request should not reach it. Signed-off-by: Cong Wang --- src/kerf/init/main.py | 19 ++++++++++--------- tests/conftest.py | 7 +++++++ tests/test_init_reconcile.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index ce11681..367b934 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -339,6 +339,11 @@ def get_valid_apic_ids_from_system() -> Optional[set]: return set(logical_to_physical().values()) or None +def get_boot_cpu_from_system() -> Optional[int]: + """The physical ID of the host's boot CPU, which never joins the pool.""" + return logical_to_physical().get(0) + + _NODE_SPEC = re.compile(r"^(.+)@(.*)$") PAGE_SIZE = 4096 @@ -628,15 +633,11 @@ def build_baseline_from_cmdline( # Total CPUs is based on the max APIC ID + 1 for sizing purposes total_cpus = max(valid_apic_ids) + 1 # Host reserved are all valid APIC IDs not in the available list - available_cpus = set(cpu_list) - host_reserved_cpus = sorted(list(valid_apic_ids - available_cpus)) - - if 0 in available_cpus and len(host_reserved_cpus) == 0: - if verbose: - click.echo(f"Warning: {cpu_id_name()} 0 is in available list but no host-reserved CPUs. Moving {cpu_id_name()} 0 to host-reserved.", err=True) - available_cpus.discard(0) - host_reserved_cpus = [0] - cpu_list = sorted(list(available_cpus)) + boot_cpu = get_boot_cpu_from_system() + if boot_cpu in cpu_list: + click.echo(f"Warning: {cpu_id_name()} {boot_cpu} is the host's boot CPU and stays with the host.", err=True) + cpu_list = [cpu for cpu in cpu_list if cpu != boot_cpu] + host_reserved_cpus = sorted(valid_apic_ids - set(cpu_list)) requested, note = resolve_memory_nodes(requested, cpu_list, pool_cpus, pool_regions) if note: diff --git a/tests/conftest.py b/tests/conftest.py index e754ad6..72d7a23 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,13 @@ InstanceConfig, WorkloadType, ) +from kerf.init import main as init_main + + +@pytest.fixture(autouse=True) +def boot_cpu_is_zero(monkeypatch): + """Keep init from reading the real host's boot CPU.""" + monkeypatch.setattr(init_main, "get_boot_cpu_from_system", lambda: 0) @pytest.fixture diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index 9c4b528..5e7f3d1 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -374,3 +374,24 @@ def read_instance_dtb(self, name): assert set(lent) == {"pci_0000_09_00_0", "pci_0000_00_1f_2"} assert lent["pci_0000_09_00_0"].alias == "enp9s0" assert lent["pci_0000_09_00_0"].pci_id == "0000:09:00.0" + + +def test_the_boot_cpu_stays_with_the_host(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 2, 3}) + + tree = main.build_baseline_from_cmdline("0-3", memory="512MB") + + assert tree.hardware.cpus.available == [1, 2, 3] + assert tree.hardware.cpus.host_reserved == [0] + + +def test_the_boot_cpu_is_found_by_its_own_id(monkeypatch): + # An arm64 server can boot on MPIDR 0x10000 and have no CPU 0 at all. + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", + lambda: {0x10000, 0x20000, 0x30000}) + monkeypatch.setattr(main, "get_boot_cpu_from_system", lambda: 0x10000) + + tree = main.build_baseline_from_cmdline("65536,131072,196608", memory="512MB") + + assert tree.hardware.cpus.available == [0x20000, 0x30000] + assert tree.hardware.cpus.host_reserved == [0x10000] From 0f36002a232a93959f92de7e7cb9186bd6a7928e Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 26 Sep 2026 15:59:31 -0700 Subject: [PATCH 5/5] release: Build an arm64 wheel alongside the x86_64 one The wheel carries a native kerf-init, so an arm64 host could only install from the sdist, which needs musl-gcc. Build the wheel on native x86_64 and arm64 runners and retag each for its own architecture. The sdist is built once, on x86_64, and publishing collects every job's distributions. Signed-off-by: Cong Wang --- .github/workflows/release.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cb44e90..f4db639 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,14 @@ on: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + include: + - runner: ubuntu-latest + arch: x86_64 + - runner: ubuntu-24.04-arm + arch: aarch64 + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -35,30 +42,34 @@ jobs: sudo apt-get update sudo apt-get install -y musl-tools - - name: Build sdist and wheel + - name: Build wheel run: | - poetry build + poetry build --format wheel # The build script makes the wheel interpreter and platform specific - # (cp311-cp311-linux_x86_64), which PyPI rejects. The only native code + # (cp311-cp311-linux_), which PyPI rejects. The only native code # is the static kerf-init, so the wheel runs on any Python 3 and libc. - name: Retag wheel for PyPI + env: + ARCH: ${{ matrix.arch }} run: | file src/kerf/data/kerf-init | grep -q "statically linked" pip install wheel auditwheel python -m wheel tags --remove --python-tag py3 --abi-tag none \ - --platform-tag manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64 \ + --platform-tag "manylinux_2_17_$ARCH.manylinux2014_$ARCH.musllinux_1_1_$ARCH" \ dist/*.whl auditwheel show dist/*.whl - - name: Check the sdist builds a wheel + - name: Build sdist + if: matrix.arch == 'x86_64' run: | + poetry build --format sdist pip wheel --no-deps -w /tmp/sdist-wheel dist/*.tar.gz - name: Upload distributions uses: actions/upload-artifact@v4 with: - name: dist + name: dist-${{ matrix.arch }} path: dist/ publish: @@ -73,7 +84,8 @@ jobs: - name: Download distributions uses: actions/download-artifact@v4 with: - name: dist + pattern: dist-* + merge-multiple: true path: dist/ - name: Publish to PyPI @@ -88,7 +100,8 @@ jobs: - name: Download distributions uses: actions/download-artifact@v4 with: - name: dist + pattern: dist-* + merge-multiple: true path: dist/ - name: Create GitHub release