Finalize fast-reboot in warmboot finalizer (#14238)

- Why I did it
To solve an issue with upgrade with fast-reboot including FW upgrade which has been introduced since moving to fast-reboot over warm-reboot infrastructure.
As well, this introduces fast-reboot finalizing logic to determine fast-reboot is done.

- How I did it
Added logic to finalize-warmboot script to handle fast-reboot as well, this makes sense as using fast-reboot over warm-reboot this script will be invoked. The script will clear fast-reboot entry from state-db instead of previous implementation that relied on timer. The timer could expire in some scenarios between fast-reboot finished causing fallback to cold-reboot and possible crashes.

As well this PR updates all services/scripts reading fast-reboot state-db entry to look for the updated value representing fast-reboot is active.

- How to verify it
Run fast-reboot and check that fast-reboot entry exists in state-db right after startup and being cleared as warm-reboot is finalized and not due to a timer.

files/image_config/ntp/ntp-config.sh

@@ -8,11 +8,11 @@ reboot_type='cold'
 function get_database_reboot_type()
 {
     SYSTEM_WARM_START=`sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
-    SYSTEM_FAST_START=`sonic-db-cli STATE_DB get "FAST_REBOOT|system"`
+    SYSTEM_FAST_START=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
 
     if [[ x"${SYSTEM_WARM_START}" == x"true" ]]; then
         reboot_type='warm'
-    elif [[ x"${SYSTEM_FAST_START}" == x"1" ]]; then
+    elif [[ x"${SYSTEM_FAST_START}" == x"true" ]]; then
         reboot_type='fast'
     fi
 }

files/image_config/warmboot-finalizer/finalize-warmboot.sh

@@ -52,6 +52,17 @@ function check_warm_boot()
     WARM_BOOT=`sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
 }
 
+function check_fast_reboot()
+{
+    debug "Checking if fast-reboot is enabled..."
+    FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${FAST_REBOOT}" == x"true" ]]; then
+       debug "Fast-reboot is enabled..."
+    else
+       debug "Fast-reboot is disabled..."
+    fi
+}
+
 
 function wait_for_database_service()
 {
@@ -97,6 +108,12 @@ function finalize_warm_boot()
     sudo config warm_restart disable
 }
 
+function finalize_fast_reboot()
+{
+    debug "Finalizing fast-reboot..."
+    sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null
+}
+
 function stop_control_plane_assistant()
 {
     if [[ -x ${ASSISTANT_SCRIPT} ]]; then
@@ -119,14 +136,20 @@ function restore_counters_folder()
 
 wait_for_database_service
 
+check_fast_reboot
 check_warm_boot
 
 if [[ x"${WARM_BOOT}" != x"true" ]]; then
     debug "warmboot is not enabled ..."
-    exit 0
+    if [[ x"${FAST_REBOOT}" != x"true" ]]; then
+	    debug "fastboot is not enabled ..."
+	    exit 0
+    fi
 fi
 
-restore_counters_folder
+if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
+    restore_counters_folder
+fi
 
 get_component_list
 
@@ -143,14 +166,22 @@ for i in `seq 60`; do
     sleep 5
 done
 
-stop_control_plane_assistant
+if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
+   stop_control_plane_assistant
+fi
 
 # Save DB after stopped control plane assistant to avoid extra entries
-debug "Save in-memory database after warm reboot ..."
+debug "Save in-memory database after warm/fast reboot ..."
 config save -y
 
 if [[ -n "${list}" ]]; then
     debug "Some components didn't finish reconcile: ${list} ..."
 fi
 
-finalize_warm_boot
+if [ x"${FAST_REBOOT}" == x"true" ]; then
+    finalize_fast_reboot
+fi
+
+if [ x"${WARM_BOOT}" == x"true" ]; then
+    finalize_warm_boot
+fi

files/scripts/bgp.sh

@@ -30,7 +30,8 @@ function validate_restore_count()
 
 function check_fast_boot ()
 {
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
         FAST_BOOT="false"

files/scripts/service_mgmt.sh

@@ -19,7 +19,8 @@ function check_warm_boot()
 
 function check_fast_boot ()
 {
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
         FAST_BOOT="false"

files/scripts/swss.sh

@@ -60,7 +60,8 @@ function check_warm_boot()
 
 function check_fast_boot()
 {
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
         FAST_BOOT="false"
@@ -284,8 +285,8 @@ stop() {
     # encountered error, e.g. syncd crashed. And swss needs to
     # be restarted.
     if [[ x"$FAST_BOOT" != x"true" ]]; then
-        debug "Clearing FAST_REBOOT flag..."
-        clean_up_tables STATE_DB "'FAST_REBOOT*'"
+        debug "Clearing FAST_RESTART_ENABLE_TABLE flag..."
+        sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false"
     fi
     # Unlock has to happen before reaching out to peer service
     unlock_service_state_change

files/scripts/syncd_common.sh

@@ -50,7 +50,8 @@ function check_warm_boot()
 
 function check_fast_boot()
 {
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
         FAST_BOOT="false"
@@ -82,7 +83,8 @@ function getBootType()
         ;;
     *SONIC_BOOT_TYPE=fast*|*fast-reboot*)
         # check that the key exists
-        if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+        SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+        if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
             TYPE='fast'
         else
             TYPE='cold'

files/scripts/teamd.sh

@@ -32,7 +32,8 @@ function validate_restore_count()
 
 function check_fast_boot ()
 {
-    if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then
+    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
         FAST_BOOT="false"

src/sonic-py-common/sonic_py_common/device_info.py

@@ -699,14 +699,16 @@ def is_warm_restart_enabled(container_name):
 
 # Check if System fast reboot is enabled.
 def is_fast_reboot_enabled():
-    fb_system_state = 0
-    cmd = ['sonic-db-cli', 'STATE_DB', 'get', "FAST_REBOOT|system"]
-    proc = subprocess.Popen(cmd, universal_newlines=True, stdout=subprocess.PIPE)
-    (stdout, stderr) = proc.communicate()
-
-    if proc.returncode != 0:
-        log.log_error("Error running command '{}'".format(cmd))
-    elif stdout:
-        fb_system_state = stdout.rstrip('\n')
+    state_db = SonicV2Connector(host='127.0.0.1')
+    state_db.connect(state_db.STATE_DB, False)
+
+    TABLE_NAME_SEPARATOR = '|'
+    prefix = 'FAST_RESTART_ENABLE_TABLE' + TABLE_NAME_SEPARATOR
+
+    # Get the system warm reboot enable state
+    _hash = '{}{}'.format(prefix, 'system')
+    fb_system_state = state_db.get(state_db.STATE_DB, _hash, "enable")
+    fb_enable_state = True if fb_system_state == "true" else False
 
-    return fb_system_state
+    state_db.close(state_db.STATE_DB)
+    return fb_enable_state