KVM: make storage heartbeat fence action configurable (graceful-reboot / restart-agent / log-only)

agent/conf/agent.properties

@@ -318,6 +318,29 @@ iscsi.session.cleanup.enabled=false
 # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat.
 #reboot.host.and.alert.management.on.heartbeat.timeout=false
 
+# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh / kvmsmpheartbeat.sh when a storage heartbeat
+# write fails persistently. Only consulted when fencing is enabled, i.e.
+# 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains
+# the global on/off switch for fencing, this setting only chooses WHAT the fence does.
+#
+# Allowed values:
+#   hard-reboot     - immediate sysrq-trigger reboot (default).
+#                     Required default for setups where a stale NFSv3 mount can prevent
+#                     a graceful shutdown from completing.
+#   graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly.
+#                     Use only if a stale storage mount cannot block shutdown.
+#   restart-agent   - restart cloudstack-agent only; running VMs are preserved.
+#   log-only        - only write a message to the host's syslog; take no action
+#                     (no reboot, no agent restart, no management server alert).
+#                     The operator must watch syslog and investigate.
+#
+# Any other value falls back to hard-reboot.
+#
+# The non-default values are recommended for setups using LINSTOR/DRBD or any local
+# storage with replication, where transient I/O contention can cause a heartbeat
+# write to time out without the host actually being unhealthy.
+#kvm.heartbeat.fence.action=hard-reboot
+
 # Enables manually setting CPU's topology on KVM's VM.
 #enable.manually.setting.cpu.topology.on.kvm.vm=true
 

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -622,6 +622,31 @@ public class AgentProperties{
     public static final Property<Boolean> REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT
         = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false);
 
+    /**
+     * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh
+     * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when
+     * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the
+     * on/off switch for fencing, this setting only chooses what the fence does. Allowed values:
+     * <ul>
+     *   <li>{@code hard-reboot} (default): immediate sysrq-trigger reboot. Required default for setups
+     *       where a stale NFSv3 mount can prevent a graceful shutdown from completing.</li>
+     *   <li>{@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs
+     *       to stop cleanly. Use only if a stale storage mount cannot block shutdown.</li>
+     *   <li>{@code restart-agent}: restart cloudstack-agent only; running VMs preserved.</li>
+     *   <li>{@code log-only}: only write a message to the host's syslog and take no action (no reboot,
+     *       no agent restart, no management server alert); the operator must investigate.</li>
+     * </ul>
+     * Any other value falls back to {@code hard-reboot}.<br>
+     * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated
+     * local storage, where transient I/O contention can cause a heartbeat write to time out
+     * without the host actually being unhealthy.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: String.<br>
+     * Default value: {@code hard-reboot}
+     */
+    public static final Property<String> KVM_HEARTBEAT_FENCE_ACTION
+        = new Property<>("kvm.heartbeat.fence.action", "hard-reboot");
+
     /**
      * Enables manually setting CPU's topology on KVM's VM. <br>
      * Data type: Boolean.<br>

scripts/vm/hypervisor/kvm/kvmha-fence.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Shared fence-action helper for kvmheartbeat.sh, kvmspheartbeat.sh and kvmsmpheartbeat.sh.
+# Sourced by those scripts; do not invoke directly.
+#
+# kvm.heartbeat.fence.action in agent.properties selects the action:
+#   hard-reboot (default), graceful-reboot, restart-agent, or log-only (only writes to
+#   syslog and takes no action). Any other value falls back to hard-reboot.
+#
+# Usage from caller:
+#   source "$(dirname "$0")/kvmha-fence.sh"
+#   fence_action "kvmheartbeat.sh"   # script name passed for log tagging
+
+AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}"
+
+fence_action() {
+  local source_script="${1:-kvmha}"
+  local FENCE_ACTION="hard-reboot"
+
+  if [ -r "$AGENT_PROPS" ]; then
+    local val
+    val=$(grep "^kvm.heartbeat.fence.action=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]')
+    [ -n "$val" ] && FENCE_ACTION="$val"
+  fi
+
+  case "$FENCE_ACTION" in
+    log-only)
+      # Only writes to syslog; no reboot, no agent restart and no management server alert.
+      /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected, taking no action. Operator must investigate."
+      exit 0
+      ;;
+    restart-agent)
+      /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent', restarting cloudstack-agent (running VMs preserved)."
+      sync &
+      sleep 2
+      systemctl restart cloudstack-agent
+      exit $?
+      ;;
+    graceful-reboot)
+      /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot', rebooting via systemctl (allows running VMs to stop cleanly)."
+      sync &
+      sleep 5
+      systemctl reboot
+      exit $?
+      ;;
+    hard-reboot|*)
+      # Default, and the fallback for any unrecognised value: identical to the pre-existing behaviour.
+      /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage."
+      sync &
+      sleep 5
+      echo b > /proc/sysrq-trigger
+      exit $?
+      ;;
+  esac
+}

scripts/vm/hypervisor/kvm/kvmheartbeat.sh

@@ -156,11 +156,9 @@ then
   exit 0
 elif [ "$cflag" == "1" ]
 then
-  /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage."
-  sync &
-  sleep 5
-  echo b > /proc/sysrq-trigger
-  exit $?
+  # shellcheck disable=SC1091
+  . "$(dirname "$0")/kvmha-fence.sh"
+  fence_action "kvmheartbeat.sh"
 else
   write_hbLog
   exit $?

scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh

@@ -207,11 +207,13 @@ then
   exit 0
 elif [ "$cflag" == "1" ]
 then
-  /usr/bin/logger -t heartbeat "kvmsmpheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage."
-  sync &
-  sleep 5
-  echo b > /proc/sysrq-trigger
-  exit $?
+  # Delegate the fence action to the shared helper so this script honours the
+  # kvm.heartbeat.fence.action property (hard-reboot / graceful-reboot /
+  # restart-agent / log-only) like kvmheartbeat.sh and kvmspheartbeat.sh.
+  # Default (no property set) remains hard-reboot via sysrq-trigger.
+  # shellcheck disable=SC1091
+  . "$(dirname "$0")/kvmha-fence.sh"
+  fence_action "kvmsmpheartbeat.sh"
 else
   write_hbLog
   exit $?

scripts/vm/hypervisor/kvm/kvmspheartbeat.sh

@@ -58,9 +58,7 @@ deleteVMs() {
 
 if [ "$cflag" == "1" ]
 then
-  /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage."
-  sync &
-  sleep 5
-  echo b > /proc/sysrq-trigger
-  exit $?
+  # shellcheck disable=SC1091
+  . "$(dirname "$0")/kvmha-fence.sh"
+  fence_action "kvmspheartbeat.sh"
 fi