kerf: Run on arm64 hosts and publish an arm64 wheel

.github/workflows/release.yml

@@ -7,7 +7,14 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            arch: x86_64
+          - runner: ubuntu-24.04-arm
+            arch: aarch64
+    runs-on: ${{ matrix.runner }}
     steps:
     - uses: actions/checkout@v4
 
@@ -35,30 +42,34 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y musl-tools
 
-    - name: Build sdist and wheel
+    - name: Build wheel
       run: |
-        poetry build
+        poetry build --format wheel
 
     # The build script makes the wheel interpreter and platform specific
-    # (cp311-cp311-linux_x86_64), which PyPI rejects. The only native code
+    # (cp311-cp311-linux_<arch>), which PyPI rejects. The only native code
     # is the static kerf-init, so the wheel runs on any Python 3 and libc.
     - name: Retag wheel for PyPI
+      env:
+        ARCH: ${{ matrix.arch }}
       run: |
         file src/kerf/data/kerf-init | grep -q "statically linked"
         pip install wheel auditwheel
         python -m wheel tags --remove --python-tag py3 --abi-tag none \
-          --platform-tag manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64 \
+          --platform-tag "manylinux_2_17_$ARCH.manylinux2014_$ARCH.musllinux_1_1_$ARCH" \
           dist/*.whl
         auditwheel show dist/*.whl
 
-    - name: Check the sdist builds a wheel
+    - name: Build sdist
+      if: matrix.arch == 'x86_64'
       run: |
+        poetry build --format sdist
         pip wheel --no-deps -w /tmp/sdist-wheel dist/*.tar.gz
 
     - name: Upload distributions
       uses: actions/upload-artifact@v4
       with:
-        name: dist
+        name: dist-${{ matrix.arch }}
         path: dist/
 
   publish:
@@ -73,7 +84,8 @@ jobs:
     - name: Download distributions
       uses: actions/download-artifact@v4
       with:
-        name: dist
+        pattern: dist-*
+        merge-multiple: true
         path: dist/
 
     - name: Publish to PyPI
@@ -88,7 +100,8 @@ jobs:
     - name: Download distributions
       uses: actions/download-artifact@v4
       with:
-        name: dist
+        pattern: dist-*
+        merge-multiple: true
         path: dist/
 
     - name: Create GitHub release

src/kerf/create/main.py

@@ -35,6 +35,7 @@
     get_available_cpus,
 )
 from ..exceptions import ValidationError, KernelInterfaceError, ResourceError, ParseError
+from ..topology import cpu_id_name
 
 
 def parse_cpu_spec(cpu_spec: str) -> List[int]:
@@ -137,7 +138,7 @@ def _allocate_spread(
                 numa_cpu_lists[numa_node_id] = sorted(numa_cpus)
 
         if not numa_cpu_lists:
-            raise ResourceError(f"No available APIC IDs in specified NUMA nodes: {numa_nodes}")
+            raise ResourceError(f"No available {cpu_id_name()}s in specified NUMA nodes: {numa_nodes}")
 
         allocated = []
         numa_indices = {node_id: 0 for node_id in numa_cpu_lists}
@@ -187,7 +188,7 @@ def _allocate_local(
         if len(numa_cpus) >= count:
             return sorted(numa_cpus[:count])
         raise ResourceError(
-            f"Not enough APIC IDs in NUMA node {numa_node_id}: "
+            f"Not enough {cpu_id_name()}s in NUMA node {numa_node_id}: "
             f"requested {count}, but only {len(numa_cpus)} available"
         )
 
@@ -200,7 +201,7 @@ def _allocate_local(
         if len(numa_cpus) >= count:
             return sorted(numa_cpus[:count])
 
-    raise ResourceError(f"No single NUMA node has {count} available APIC IDs for 'local' affinity")
+    raise ResourceError(f"No single NUMA node has {count} available {cpu_id_name()}s for 'local' affinity")
 
 
 def allocate_cpus_from_pool(
@@ -236,11 +237,11 @@ def allocate_cpus_from_pool(
     if len(available) < count:
         if numa_nodes:
             raise ResourceError(
-                f"Not enough APIC IDs available in NUMA nodes {numa_nodes}: "
+                f"Not enough {cpu_id_name()}s available in NUMA nodes {numa_nodes}: "
                 f"requested {count}, but only {len(available)} available"
             )
         raise ResourceError(
-            f"Not enough APIC IDs available: requested {count}, "
+            f"Not enough {cpu_id_name()}s available: requested {count}, "
             f"but only {len(available)} available in pool"
         )
 
@@ -364,8 +365,8 @@ def dump_overlay_for_debug(
 @click.option(
     "--cpus",
     "-c",
-    help='Explicit APIC ID allocation (e.g., "128" for APIC ID 128, "128-134" for range, '
-    '"128,130,132" for list). Use physical APIC IDs, not logical CPU numbers. '
+    help='Explicit CPU allocation by physical ID, the APIC ID on x86 or the MPIDR on arm64 '
+    '(e.g., "128", "128-134" for range, "128,130,132" for list), not logical CPU numbers. '
     'Mutually exclusive with --cpu-count',
 )
 @click.option(
@@ -561,7 +562,7 @@ def create(  # pylint: disable=too-many-arguments,too-many-positional-arguments
             try:
                 cpu_spec_value = parse_cpu_spec(cpus)
             except ValueError as e:
-                click.echo(f"Error: Invalid APIC ID specification '{cpus}': {e}", err=True)
+                click.echo(f"Error: Invalid {cpu_id_name()} specification '{cpus}': {e}", err=True)
                 sys.exit(2)
 
         # Parse memory specification

src/kerf/init/main.py

@@ -54,7 +54,7 @@
 from ..devices import default_alias, is_partition, pci_node_name, split_alias
 from ..resources import get_busy_chunks_from_iomem
 from ..runtime import DeviceTreeManager
-from ..topology import cpu_numa_nodes, node_for_cpus
+from ..topology import cpu_id_name, cpu_numa_nodes, logical_to_physical, node_for_cpus
 
 
 MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel"
@@ -333,31 +333,15 @@ def get_total_cpus_from_system() -> Optional[int]:
 
 def get_valid_apic_ids_from_system() -> Optional[set]:
     """
-    Get set of valid APIC IDs from the system via /proc/cpuinfo.
-    Returns set of valid APIC IDs or None if not available.
+    Get the physical IDs of the system's CPUs: APIC ids on x86, MPIDRs on arm64.
+    Returns the set of valid IDs or None if not available.
     """
-    try:
-        cpuinfo_path = Path('/proc/cpuinfo')
-        if not cpuinfo_path.exists():
-            return None
+    return set(logical_to_physical().values()) or None
 
-        apic_ids = set()
-        with open(cpuinfo_path, 'r', encoding='utf-8') as f:
-            for line in f:
-                if line.startswith('apicid'):
-                    parts = line.split(':')
-                    if len(parts) == 2:
-                        try:
-                            apic_id = int(parts[1].strip())
-                            apic_ids.add(apic_id)
-                        except ValueError:
-                            pass
-
-        return apic_ids if apic_ids else None
-    except (OSError, IOError):
-        pass
 
-    return None
+def get_boot_cpu_from_system() -> Optional[int]:
+    """The physical ID of the host's boot CPU, which never joins the pool."""
+    return logical_to_physical().get(0)
 
 
 _NODE_SPEC = re.compile(r"^(.+)@(.*)$")
@@ -631,8 +615,8 @@ def build_baseline_from_cmdline(
     valid_apic_ids = get_valid_apic_ids_from_system()
     if valid_apic_ids is None:
         raise KernelInterfaceError(
-            "Could not read APIC IDs from /proc/cpuinfo. "
-            "Ensure the system exposes CPU topology information."
+            "Could not read the physical CPU IDs (APIC ids from /proc/cpuinfo, "
+            "or MPIDRs from /sys/devices/system/cpu on arm64)."
         )
 
     # The host stops listing a CPU in /proc/cpuinfo once the pool takes it,
@@ -642,33 +626,29 @@ def build_baseline_from_cmdline(
     invalid_cpus = set(cpu_list) - valid_apic_ids
     if invalid_cpus:
         raise ValueError(
-            f"Invalid APIC ID(s) specified: {sorted(invalid_cpus)}. "
-            f"Valid APIC IDs on this system: {sorted(valid_apic_ids)}"
+            f"Invalid {cpu_id_name()}(s) specified: {sorted(invalid_cpus)}. "
+            f"Valid {cpu_id_name()}s on this system: {sorted(valid_apic_ids)}"
         )
 
     # Total CPUs is based on the max APIC ID + 1 for sizing purposes
     total_cpus = max(valid_apic_ids) + 1
     # Host reserved are all valid APIC IDs not in the available list
-    available_cpus = set(cpu_list)
-    host_reserved_cpus = sorted(list(valid_apic_ids - available_cpus))
-
-    if 0 in available_cpus and len(host_reserved_cpus) == 0:
-        if verbose:
-            click.echo("Warning: APIC ID 0 is in available list but no host-reserved CPUs. Moving APIC ID 0 to host-reserved.", err=True)
-        available_cpus.discard(0)
-        host_reserved_cpus = [0]
-        cpu_list = sorted(list(available_cpus))
+    boot_cpu = get_boot_cpu_from_system()
+    if boot_cpu in cpu_list:
+        click.echo(f"Warning: {cpu_id_name()} {boot_cpu} is the host's boot CPU and stays with the host.", err=True)
+        cpu_list = [cpu for cpu in cpu_list if cpu != boot_cpu]
+    host_reserved_cpus = sorted(valid_apic_ids - set(cpu_list))
 
     requested, note = resolve_memory_nodes(requested, cpu_list, pool_cpus, pool_regions)
     if note:
         click.echo(note)
 
     total_bytes = sum(requested.values())
     if verbose:
-        click.echo(f"Parsed APIC ID specification: {cpus}")
-        click.echo(f"  Valid APIC IDs on system: {sorted(valid_apic_ids)}")
-        click.echo(f"  Host-reserved APIC IDs: {host_reserved_cpus}")
-        click.echo(f"  Available APIC IDs: {cpu_list}")
+        click.echo(f"Parsed {cpu_id_name()} specification: {cpus}")
+        click.echo(f"  Valid {cpu_id_name()}s on system: {sorted(valid_apic_ids)}")
+        click.echo(f"  Host-reserved {cpu_id_name()}s: {host_reserved_cpus}")
+        click.echo(f"  Available {cpu_id_name()}s: {cpu_list}")
         click.echo("Requested pool memory:")
         for node, size in sorted(requested.items()):
             click.echo(f"  node {node}: {size} bytes ({size / (1024**3):.2f} GB)")
@@ -970,7 +950,7 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) ->
 @click.command()
 @click.pass_context
 @click.option('--input', '-i', help='Baseline DTB to replay, as written by "kerf dump". Mutually exclusive with --cpus, --memory and --devices.')
-@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.')
+@click.option('--cpus', '-c', help='Physical CPU IDs for baseline, the APIC ID on x86 or the MPIDR on arm64 (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Not logical CPU numbers. Mutually exclusive with --input.')
 @click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, per-node "8GB@0,8GB@1", or "none" for no pool memory. Required with --cpus, mutually exclusive with --input.')
 @click.option('--devices', '-d', help='Host device names or PCI addresses to pool, each with an optional "=alias" the spawned kernels name the device by (comma-separated, e.g. "nvme0n1,enp9s0,0000:4f:01.0=nvme1"), or "none" for no devices. By default an NVMe keeps its controller name and a network device keeps its interface name. Mutually exclusive with --input.')
 @click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.')

src/kerf/load/main.py

@@ -23,7 +23,7 @@
 import sys
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 import click
 
@@ -102,6 +102,42 @@ def build_ip_param(
     return "ip=" + ":".join(parts)
 
 
+# What a spawn kernel cannot do without, per architecture. On arm64 the host
+# can only force a wedged spawn down with a stop interrupt, and that reaches
+# a CPU that has interrupts off only when the spawn runs with pseudo-NMIs
+# (it must also be built with CONFIG_ARM64_PSEUDO_NMI).
+REQUIRED_KERNEL_PARAMS = {
+    "aarch64": ("irqchip.gicv3_pseudo_nmi=1",),
+    "arm64": ("irqchip.gicv3_pseudo_nmi=1",),
+}
+
+
+def required_kernel_params(cmdline_parts: List[str], arch: Optional[str] = None) -> List[str]:
+    """
+    The parameters that must still be appended to a spawn's command line.
+
+    The kernel honors the last setting of a parameter, so one that the
+    command line sets differently is appended again and wins; one whose
+    last setting already is the required one is left alone.
+
+    Args:
+        cmdline_parts: The command line so far, as it will be joined
+        arch: Architecture to decide for instead of this machine's
+
+    Returns:
+        Parameters to append, in order
+    """
+    arch = (arch or platform.machine()).lower()
+    words = " ".join(cmdline_parts).split()
+    missing = []
+    for param in REQUIRED_KERNEL_PARAMS.get(arch, ()):
+        key = param.split("=", 1)[0] + "="
+        settings = [word for word in words if word.startswith(key)]
+        if not settings or settings[-1] != param:
+            missing.append(param)
+    return missing
+
+
 def get_kexec_file_load_syscall():
     """Get the kexec_file_load syscall number for current architecture."""
     arch = platform.machine().lower()
@@ -510,6 +546,13 @@ def load(  # pylint: disable=too-many-arguments,too-many-positional-arguments,to
             if verbose:
                 click.echo(f"Console: console={console_device}")
 
+        # Last, so that it wins over anything given above
+        required = required_kernel_params(cmdline_parts)
+        if required:
+            cmdline_parts.extend(required)
+            if verbose:
+                click.echo(f"Required on this architecture: {' '.join(required)}")
+
         cmdline_str = " ".join(cmdline_parts)
 
         if verbose:

src/kerf/resources.py

@@ -28,6 +28,7 @@
 
 from .models import GlobalDeviceTree, PoolMemoryRegion
 from .exceptions import ResourceError
+from .topology import cpu_id_name
 
 IOMEM_PATH = "/proc/iomem"
 
@@ -358,8 +359,8 @@ def validate_cpu_allocation(
     invalid_cpus = requested_set - hardware_cpus
     if invalid_cpus:
         raise ResourceError(
-            f"Invalid APIC IDs requested: {sorted(invalid_cpus)}. "
-            f"Available APIC IDs: {sorted(hardware_cpus)}"
+            f"Invalid {cpu_id_name()}s requested: {sorted(invalid_cpus)}. "
+            f"Available {cpu_id_name()}s: {sorted(hardware_cpus)}"
         )
 
     # Check APIC IDs are available
@@ -372,10 +373,10 @@ def validate_cpu_allocation(
                 continue
             conflict_cpus = set(instance.resources.cpus) & unavailable
             if conflict_cpus:
-                conflicts.append(f"{instance.name} uses APIC IDs {sorted(conflict_cpus)}")
+                conflicts.append(f"{instance.name} uses {cpu_id_name()}s {sorted(conflict_cpus)}")
 
         conflict_msg = ", ".join(conflicts) if conflicts else "allocated to other instances"
-        raise ResourceError(f"APIC IDs {sorted(unavailable)} are not available ({conflict_msg})")
+        raise ResourceError(f"{cpu_id_name()}s {sorted(unavailable)} are not available ({conflict_msg})")
 
 
 def find_next_instance_id(tree: GlobalDeviceTree) -> int: