From 223687fab5150204d23a90844a9f7201649399a3 Mon Sep 17 00:00:00 2001 From: Tariq Ibrahim Date: Wed, 5 Aug 2026 09:04:58 -0700 Subject: [PATCH] fail fast when FabricManager fails to come up Signed-off-by: Tariq Ibrahim --- rhel10/nvidia-driver | 4 ++-- rhel10/precompiled/nvidia-driver | 4 ++-- rhel8/nvidia-driver | 4 ++-- rhel8/precompiled/nvidia-driver | 2 +- rhel9/nvidia-driver | 4 ++-- rhel9/precompiled/nvidia-driver | 4 ++-- ubuntu22.04/nvidia-driver | 8 +++----- ubuntu22.04/precompiled/nvidia-driver | 8 +++----- ubuntu24.04/nvidia-driver | 8 +++----- ubuntu24.04/precompiled/nvidia-driver | 8 +++----- ubuntu26.04/nvidia-driver | 8 +++----- ubuntu26.04/precompiled/nvidia-driver | 8 +++----- 12 files changed, 29 insertions(+), 41 deletions(-) diff --git a/rhel10/nvidia-driver b/rhel10/nvidia-driver index 8e6560315..c24e40106 100755 --- a/rhel10/nvidia-driver +++ b/rhel10/nvidia-driver @@ -773,13 +773,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/rhel10/precompiled/nvidia-driver b/rhel10/precompiled/nvidia-driver index 5b22efecb..d0aa11b21 100755 --- a/rhel10/precompiled/nvidia-driver +++ b/rhel10/precompiled/nvidia-driver @@ -268,12 +268,12 @@ _load_driver() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/rhel8/nvidia-driver b/rhel8/nvidia-driver index 2de548c86..49db4628f 100755 --- a/rhel8/nvidia-driver +++ b/rhel8/nvidia-driver @@ -750,13 +750,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/rhel8/precompiled/nvidia-driver b/rhel8/precompiled/nvidia-driver index cbe7f4854..fd1133337 100755 --- a/rhel8/precompiled/nvidia-driver +++ b/rhel8/precompiled/nvidia-driver @@ -210,7 +210,7 @@ _load_driver() { if _assert_nvswitch_system; then echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/rhel9/nvidia-driver b/rhel9/nvidia-driver index 701360bdc..b362ab922 100755 --- a/rhel9/nvidia-driver +++ b/rhel9/nvidia-driver @@ -769,13 +769,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/rhel9/precompiled/nvidia-driver b/rhel9/precompiled/nvidia-driver index 62020c2d5..e89471bd7 100755 --- a/rhel9/precompiled/nvidia-driver +++ b/rhel9/precompiled/nvidia-driver @@ -294,12 +294,12 @@ _load_driver() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/ubuntu22.04/nvidia-driver b/ubuntu22.04/nvidia-driver index bf54dd5a8..e4945c82a 100755 --- a/ubuntu22.04/nvidia-driver +++ b/ubuntu22.04/nvidia-driver @@ -377,9 +377,7 @@ _load_driver() { set +o xtrace -o nounset fi - _start_daemons - - return 0 + _start_daemons || return 1 } # Stop persistenced and unload the kernel modules if they are currently loaded. @@ -714,13 +712,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/ubuntu22.04/precompiled/nvidia-driver b/ubuntu22.04/precompiled/nvidia-driver index bb63653a3..98b231541 100755 --- a/ubuntu22.04/precompiled/nvidia-driver +++ b/ubuntu22.04/precompiled/nvidia-driver @@ -269,7 +269,7 @@ _load_driver() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then @@ -277,12 +277,10 @@ _load_driver() { fabricmanager_install nscq_install imex_install - + echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi - - return 0 } # Stop persistenced and unload the kernel modules if they are currently loaded. diff --git a/ubuntu24.04/nvidia-driver b/ubuntu24.04/nvidia-driver index f02a6b1d5..90e544e74 100755 --- a/ubuntu24.04/nvidia-driver +++ b/ubuntu24.04/nvidia-driver @@ -318,9 +318,7 @@ _load_driver() { set +o xtrace -o nounset fi - _start_daemons - - return 0 + _start_daemons || return 1 } # Stop persistenced and unload the kernel modules if they are currently loaded. @@ -642,13 +640,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/ubuntu24.04/precompiled/nvidia-driver b/ubuntu24.04/precompiled/nvidia-driver index 6a5cb5804..43cf88f1b 100755 --- a/ubuntu24.04/precompiled/nvidia-driver +++ b/ubuntu24.04/precompiled/nvidia-driver @@ -270,7 +270,7 @@ _load_driver() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then @@ -278,12 +278,10 @@ _load_driver() { fabricmanager_install nscq_install imex_install - + echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi - - return 0 } # Stop persistenced and unload the kernel modules if they are currently loaded. diff --git a/ubuntu26.04/nvidia-driver b/ubuntu26.04/nvidia-driver index f532505cb..27ee863ac 100755 --- a/ubuntu26.04/nvidia-driver +++ b/ubuntu26.04/nvidia-driver @@ -330,9 +330,7 @@ _load_driver() { set +o xtrace -o nounset fi - _start_daemons - - return 0 + _start_daemons || return 1 } # Stop persistenced and unload the kernel modules if they are currently loaded. @@ -702,13 +700,13 @@ _start_daemons() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then _configure_fabric_manager echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi } diff --git a/ubuntu26.04/precompiled/nvidia-driver b/ubuntu26.04/precompiled/nvidia-driver index 54a926aed..37addfe0a 100755 --- a/ubuntu26.04/precompiled/nvidia-driver +++ b/ubuntu26.04/precompiled/nvidia-driver @@ -240,7 +240,7 @@ _load_driver() { --fm-config-file $fm_config_file \ --fm-pid-file $fm_pid_file \ --nvlsm-config-file $nvlsm_config_file \ - --nvlsm-pid-file $nvlsm_pid_file + --nvlsm-pid-file $nvlsm_pid_file || return 1 # If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches elif _assert_nvswitch_system; then @@ -248,12 +248,10 @@ _load_driver() { fabricmanager_install nscq_install imex_install - + echo "Starting NVIDIA fabric manager daemon..." - nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg || return 1 fi - - return 0 } # Stop persistenced and unload the kernel modules if they are currently loaded.