check and fix

Author: liangxin1300

crmsh/bootstrap.py

@@ -46,7 +46,7 @@
 from .sh import ShellUtils
 from .ui_node import NodeMgmt
 from .user_of_host import UserOfHost, UserNotFoundError
-from .sbd import SBDUtils, SBDManager, SBDTimeout
+from .sbd import SBDUtils, SBDManager, SBDTimeout, SBDTimeoutChecker
 from . import watchdog
 import crmsh.healthcheck
 
@@ -2743,7 +2743,7 @@ def adjust_stonith_timeout(with_sbd: bool = False):
     Adjust stonith-timeout for sbd and other scenarios
     """
     if ServiceManager().service_is_active(constants.SBD_SERVICE) or with_sbd:
-        SBDTimeout.adjust_sbd_timeout_related_cluster_configuration()
+        SBDTimeoutChecker(fix=True, warn=False).check_and_fix()
     else:
         value = get_stonith_timeout_generally_expected()
         if value:

crmsh/sbd.py

@@ -10,6 +10,7 @@
 from . import xmlutil
 from . import watchdog
 from . import parallax
+from . import healthcheck
 from .service_manager import ServiceManager
 from .sh import ShellUtils
 
@@ -196,6 +197,7 @@ def __init__(self, context=None):
         self.sbd_watchdog_timeout = self.SBD_WATCHDOG_TIMEOUT_DEFAULT
         self.stonith_watchdog_timeout = None
         self.two_node_without_qdevice = False
+        self.qdevice_sync_timeout = None
         if self.context:
             self._initialize_timeout_in_bootstrap()
 
@@ -292,6 +294,8 @@ def _load_configurations(self):
             self.disk_based = False
             self.sbd_watchdog_timeout = SBDTimeout.get_sbd_watchdog_timeout()
             self.stonith_watchdog_timeout = SBDTimeout.get_stonith_watchdog_timeout_expected()
+            if corosync.is_qdevice_configured() and ServiceManager().service_is_active("corosync-qdevice.service")
+                self.qdevice_sync_timeout = utils.get_qdevice_sync_timeout()
         self.sbd_delay_start_value_expected = self.get_sbd_delay_start_expected() if utils.detect_virt() else "no"
         self.sbd_delay_start_value_from_config = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START")
 
@@ -359,53 +363,187 @@ def get_sbd_systemd_start_timeout() -> int:
         out = sh.cluster_shell().get_stdout_or_raise_error(SBDTimeout.SHOW_SBD_START_TIMEOUT_CMD)
         return utils.get_systemd_timeout_start_in_sec(out)
 
-    def adjust_systemd_start_timeout(self):
+
+class SBDTimeoutChecker(SBDTimeout):
+
+    def __init__(self, fix=False, warn=True):
+        super().__init__()
+        self.fix = fix
+        self.warning_during_check = warn
+
+    def check_and_fix(self) -> bool:
+        '''
+        Return True if all checks pass (after optional fixes),
+        otherwise False or raise healthcheck.FixFailure if fixing fails
+        '''
+        checks_and_fixes = [
+            # failure name, check function, fix function
+            ("SBD disk metadata",
+             self._check_sbd_disk_metadata, self._adjust_sbd_disk_metadata),
+            ("SBD_WATCHDOG_TIMEOUT",
+             self._check_sbd_watchdog_timeout, self._adjust_sbd_watchdog_timeout),
+            ("SBD_DELAY_START",
+             self._check_sbd_delay_start, self._adjust_sbd_delay_start),
+            ("systemd start timeout for sbd.service",
+             self._check_sbd_systemd_start_timeout, self._adjust_sbd_systemd_start_timeout),
+            ("stonith-watchdog-timeout property",
+             self._check_stonith_watchdog_timeout, self._adjust_stonith_watchdog_timeout),
+            ("stonith-timeout property",
+             self._check_stonith_timeout, self._adjust_stonith_timeout)
+        ]
+
+        self._load_configurations()
+        for name, check_func, fix_func in checks_and_fixes:
+            if check_func(warn=self.warning_during_check):
+                continue
+            if not self.fix:
+                return False
+            fix_func()
+            self._load_configurations()
+            if not check_func(warn=False):
+                raise healthcheck.FixFailure(f"Failed to fix: {name}")
+
+        return True
+
+    def _check_sbd_disk_metadata(self, warn=True) -> bool:
         '''
-        Adjust start timeout for sbd when set SBD_DELAY_START
+        Check msgwait and watchdog timeout for disk-based sbd
         '''
-        sbd_delay_start_value = SBDUtils.get_sbd_value_from_config("SBD_DELAY_START")
-        if sbd_delay_start_value == "no":
-            return
+        if self.disk_based and self.sbd_msgwait < 2*self.sbd_watchdog_timeout:
+            if warn:
+                logger.warning("It's recommended that msgwait(now %d) >= 2*watchdog timeout(now %d)",
+                               self.sbd_msgwait, self.sbd_watchdog_timeout)
+            return False
+        return True
 
-        start_timeout = SBDTimeout.get_sbd_systemd_start_timeout()
-        if start_timeout > int(sbd_delay_start_value):
-            return
+    def _adjust_sbd_disk_metadata(self):
+        '''
+        Adjust msgwait to 2*watchdog timeout for disk-based sbd
+        '''
+        advised_msgwait = 2*self.sbd_watchdog_timeout
+        logger.info("Adjusting sbd msgwait to %d", advised_msgwait)
+        cmd = f"crm sbd configure msgwait-timeout={advised_msgwait} watchdog-timeout={self.sbd_watchdog_timeout}"
+        output = sh.cluster_shell().get_stdout_or_raise_error(cmd)
+        if output:
+            print(output)
 
-        utils.mkdirp(SBDManager.SBD_SYSTEMD_DELAY_START_DIR)
-        sbd_delay_start_file = "{}/sbd_delay_start.conf".format(SBDManager.SBD_SYSTEMD_DELAY_START_DIR)
-        utils.str2file("[Service]\nTimeoutSec={}".format(int(1.2*int(sbd_delay_start_value))), sbd_delay_start_file)
-        bootstrap.sync_path(SBDManager.SBD_SYSTEMD_DELAY_START_DIR)
-        utils.cluster_run_cmd("systemctl daemon-reload")
+    def _check_sbd_watchdog_timeout(self, warn=True) -> bool:
+        '''
+        Check SBD_WATCHDOG_TIMEOUT for disk-less sbd
+        '''
+        if self.disk_based or not self.qdevice_sync_timeout:
+            return True
+        if self.sbd_watchdog_timeout < self.qdevice_sync_timeout:
+            if warn:
+                logger.warning("It's recommended that SBD_WATCHDOG_TIMEOUT(now %d) > qdevice sync timeout(now %d)",
+                               self.sbd_watchdog_timeout, self.qdevice_sync_timeout)
+            return False
+        return True
 
-    def adjust_stonith_timeout(self):
+    def _adjust_sbd_watchdog_timeout(self):
         '''
-        Adjust stonith-timeout property
+        Adjust SBD_WATCHDOG_TIMEOUT for disk-less sbd
         '''
-        utils.set_property("stonith-timeout", self.get_stonith_timeout_expected(), conditional=True)
+        advised_watchdog_timeout = self.qdevice_sync_timeout + SBDTimeout.QDEVICE_SYNC_TIMEOUT_MARGIN
+        SBDManager.update_sbd_configuration({"SBD_WATCHDOG_TIMEOUT": str(advised_watchdog_timeout)})
+        logger.info("Adjusting SBD_WATCHDOG_TIMEOUT to %d", advised_watchdog_timeout)
 
-    def adjust_sbd_delay_start(self):
+    def _check_sbd_delay_start(self, warn=True) -> bool:
         '''
-        Adjust SBD_DELAY_START in /etc/sysconfig/sbd
+        Check SBD_DELAY_START
         '''
         expected_value = str(self.sbd_delay_start_value_expected)
         config_value = self.sbd_delay_start_value_from_config
         if expected_value == config_value:
-            return
+            return True
         if expected_value == "no" \
                 or (not re.search(r'\d+', config_value)) \
                 or (int(expected_value) > int(config_value)):
-            SBDManager.update_sbd_configuration({"SBD_DELAY_START": expected_value})
+            if warn:
+                logger.warning("It's recommended that SBD_DELAY_START is set to %s, current value is %s",
+                               expected_value, config_value)
+            return False
 
-    @classmethod
-    def adjust_sbd_timeout_related_cluster_configuration(cls):
+    def _adjust_sbd_delay_start(self):
         '''
-        Adjust sbd timeout related configurations
+        Adjust SBD_DELAY_START
         '''
-        cls_inst = cls()
-        cls_inst._load_configurations()
-        cls_inst.adjust_sbd_delay_start()
-        cls_inst.adjust_stonith_timeout()
-        cls_inst.adjust_systemd_start_timeout()
+        SBDManager.update_sbd_configuration({"SBD_DELAY_START": str(self.sbd_delay_start_value_expected)})
+
+    def _check_sbd_systemd_start_timeout(self, warn=True) -> bool:
+        '''
+        Check systemd start timeout for sbd.service
+        '''
+        if not self.sbd_delay_start_value_from_config or self.sbd_delay_start_value_from_config == "no":
+            return True
+        systemd_start_timeout = SBDTimeout.get_sbd_systemd_start_timeout()
+        if systemd_start_timeout > int(self.sbd_delay_start_value_from_config):
+            return True
+        if warn:
+            logger.warning("It's recommended that systemd start timeout for sbd.service is set to %d, current value is %d",
+                           int(1.2*int(self.sbd_delay_start_value_from_config)), systemd_start_timeout)
+        return False
+
+    def _adjust_sbd_systemd_start_timeout(self):
+        '''
+        Adjust systemd start timeout for sbd.service
+        '''
+        systemd_start_timeout = int(1.2*int(self.sbd_delay_start_value_from_config))
+        utils.mkdirp(SBDManager.SBD_SYSTEMD_DELAY_START_DIR)
+        sbd_delay_start_file = f"{SBDManager.SBD_SYSTEMD_DELAY_START_DIR}/sbd_delay_start.conf"
+        utils.str2file(f"[Service]\nTimeoutSec={systemd_start_timeout}", sbd_delay_start_file)
+        bootstrap.sync_path(SBDManager.SBD_SYSTEMD_DELAY_START_DIR)
+        utils.cluster_run_cmd("systemctl daemon-reload")
+        logger.info("Adjusted systemd start timeout for sbd.service to %d", systemd_start_timeout)
+
+    def _check_stonith_watchdog_timeout(self, warn=True) -> bool:
+        '''
+        Check stonith-watchdog-timeout property
+        '''
+        value = utils.get_property("stonith-watchdog-timeout", get_default=False)
+        if self.disk_based:
+            if value and warn:
+                logger.warning("It's recommended to remove stonith-watchdog-timeout property when using disk-based SBD")
+                return False
+        elif int(value) < 2*self.sbd_watchdog_timeout:
+            if warn:
+                logger.warning("It's recommended that stonith-watchdog-timeout(now %s) >= 2*SBD_WATCHDOG_TIMEOUT(now %d)",
+                               value, self.sbd_watchdog_timeout)
+            return False
+        return True
+
+    def _adjust_stonith_watchdog_timeout(self):
+        '''
+        Adjust stonith-watchdog-timeout property
+        '''
+        if self.disk_based:
+            utils.delete_property("stonith-watchdog-timeout")
+            logger.info("Removed stonith-watchdog-timeout property")
+        else:
+            adviced_value = SBDTimeout.get_stonith_watchdog_timeout_expected()
+            utils.set_property("stonith-watchdog-timeout", adviced_value)
+            logger.info("Adjusted stonith-watchdog-timeout to %d", adviced_value)
+
+    def _check_stonith_timeout(self, warn=True) -> bool:
+        '''
+        Check stonith-timeout property
+        '''
+        value = utils.get_property("stonith-timeout", get_default=False)
+        expected_value = self.get_stonith_timeout_expected()
+        if not value or int(value) < expected_value:
+            if warn:
+                logger.warning("It's recommended that stonith-timeout is set to %d, current value is %s",
+                               expected_value, value)
+            return False
+        return True
+
+    def _adjust_stonith_timeout(self):
+        '''
+        Adjust stonith-timeout property
+        '''
+        expected_value = self.get_stonith_timeout_expected()
+        utils.set_property("stonith-timeout", expected_value)
+        logger.info("Adjusted stonith-timeout to %d", expected_value)
 
 
 class SBDManager:
@@ -497,7 +635,6 @@ def update_configuration(self) -> None:
         utils.sysconfig_set(self.SYSCONFIG_SBD, **self.update_dict)
         if self.cluster_is_running:
             bootstrap.sync_path(self.SYSCONFIG_SBD)
-            logger.info("Already synced %s to all nodes", self.SYSCONFIG_SBD)
 
     @classmethod
     def update_sbd_configuration(cls, update_dict: typing.Dict[str, str]) -> None:

crmsh/ui_cluster.py

@@ -28,7 +28,7 @@
 from .sh import ShellUtils
 from .ui_node import parse_option_for_nodes
 from . import constants
-
+from . import sbd
 
 from . import log
 from .utils import TerminateSubCommand
@@ -793,18 +793,17 @@ def do_geo_init_arbitrator(self, context, *args):
         bootstrap.bootstrap_arbitrator(geo_context)
         return True
 
-    @command.completers(compl.choice([
-        'hawk2',
-        'sles16',
-    ]))
+    HEALTH_COMPONENTS = ['hawk2', 'sles16', 'sbd']
+
+    @command.completers(compl.choice(HEALTH_COMPONENTS))
     def do_health(self, context, *args):
         '''
         Extensive health check.
         '''
         if not args:
             return Cluster._do_health_legacy()
         parser = argparse.ArgumentParser('health')
-        parser.add_argument('component', choices=['hawk2', 'sles16'])
+        parser.add_argument('component', choices=Cluster.HEALTH_COMPONENTS)
         parser.add_argument('-f', '--fix', action='store_true')
         parsed_args, remaining_args = parser.parse_known_args(args)
         match parsed_args.component:
@@ -837,8 +836,23 @@ def do_health(self, context, *args):
                         return True
                     else:
                         logger.error("hawk2: passwordless ssh authentication: FAIL.")
-                        logger.warning('Please run "crm cluster health hawk2 --fix"')
+                        logger.warning('Please run "crm cluster health hawk2 --fix".')
                         return False
+            case 'sbd':
+                fix = parsed_args.fix
+                try:
+                    result = sbd.SBDTimeoutChecker(fix=fix).check_and_fix()
+                except healthcheck.FixFailure as e:
+                    logger.error('%s', e)
+                    return False
+                if result:
+                    logger.info("SBD: Check sbd timeout settings: OK.")
+                    return True
+                else:
+                    logger.error("SBD: Check sbd timeout settings: FAIL.")
+                    if not fix:
+                        logger.warning('Please run "crm cluster health sbd --fix".')
+                    return False
             case 'sles16':
                 try:
                     if parsed_args.fix:

crmsh/ui_sbd.py

@@ -253,6 +253,9 @@ def _configure_show(self, args) -> None:
             print()
             self._show_property()
 
+        print()
+        sbd.SBDTimeoutChecker().check_and_fix()
+
     def _parse_args(self, args: tuple[str, ...]) -> dict[str, int|str]:
         '''
         Parse arguments and verify them