Merge pull request #30 from QualiSystemsLab/teardowndebug_part2

Teardowndebug part2

.gitignore

@@ -66,4 +66,5 @@ qpm.ini
 # zip files
 *.zip
 SandboxOrchestration/environment_scripts/env_setup/data.json
-
+sandbox_scripts\QualiEnvironmentUtils\tests\DebugInteractive_setup_resources.py
+sandbox_scripts\QualiEnvironmentUtils\tests\DebugInteractive_teardown_resources.py

pack.bat

@@ -1,4 +1,6 @@
 @echo off
-python -m pip install qpm --no-cache-dir --upgrade
+Path = %Path%;C:\Python27
+echo %Path%
+python.exe -m pip install qpm --no-cache-dir --upgrade
 copy version.txt SandboxOrchestrationPackage/version.txt /Y
-python -m qpm pack --package_name SandboxOrchestration
+python.exe -m qpm pack --package_name SandboxOrchestration

sandbox_scripts/QualiEnvironmentUtils/Resource.py

@@ -5,9 +5,9 @@
 from cloudshell.api.cloudshell_api import *
 from cloudshell.api.common_cloudshell_api import *
 from QualiUtils import *
-import datetime
+import datetime, time
 import json
-
+from time import sleep
 
 class ResourceBase(object):
     def __init__(self, resource_name, resource_alias=''):
@@ -96,23 +96,28 @@ def get_neighbors(self, reservation_id):
 
     # ----------------------------------
     # ----------------------------------
-    def health_check(self,reservation_id):
+    def health_check(self,reservation_id, health_check_attempts=1):
         """
-        Run the healthCheck command on all the devices
+        Run the healthCheck command on the device
         :param str reservation_id:  Reservation id.
         """
         if self.has_command('health_check'):
-            try:
-                # Return a detailed description in case of a failure
-                out = self.execute_command(reservation_id, 'health_check', printOutput=False)
-                if out.Output.find(' passed') == -1:
-                    err = "Health check did not pass for device " + self.name + ". " + out.Output
+            for attempts in range(0, int(health_check_attempts)):
+                try:
+                    # Return a detailed description in case of a failure
+                    out = self.execute_command(reservation_id, 'health_check', printOutput=True) #.Output()
+                    if out.Output.find(' passed') == -1 and attempts == (int(health_check_attempts) -1):
+                        err = "Health check did not pass for device " + self.name + ". " + out.Output
+                        return err
+                    if out.Output.find(' passed') == -1:
+                        time.sleep(30)
+                    else:
+                        return ""
+                except QualiError as qe:
+                    err = "Health check did not pass for device " + self.name + ". " + str(qe)
                     return err
-
-            except QualiError as qe:
-                err = "Health check did not pass for device " + self.name + ". " + str(qe)
-                return err
-        return ""
+        else:
+            return ""
 
     # -----------------------------------------
     # -----------------------------------------

sandbox_scripts/QualiEnvironmentUtils/Sandbox.py

@@ -27,7 +27,7 @@ def __init__(self, reservation_id, logger):
             self.owner = context.owner_user
             self.Blueprint_name = context.environment_name
             if self.Blueprint_name == '':
-                raise QualiError("Blueprint name empty (from env name)")
+                raise QualiError("NameError","Blueprint name empty (from env name)")
 
             full_path = None
             tp = self.api_session.GetActiveTopologyNames()
@@ -244,6 +244,7 @@ def clear_all_resources_live_status(self):
         """
             Clear the live status from all the devices
         """
+        #TODO change to honor ignor_models
         root_resources = self.get_root_resources()
         for resource in root_resources:
             self.api_session.SetResourceLiveStatus(resource.name, liveStatusName="Info",

sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py

@@ -1,10 +1,13 @@
 import cloudshell.helpers.scripts.cloudshell_dev_helpers as dev_helpers
 from sandbox_scripts.environment.setup.setup_resources import *
 
-dev_helpers.attach_to_cloudshell_as(user="admin", password="admin", domain="Global",
-                                    reservation_id="c04d3da4-8025-4efe-9f4d-820ba19d20af",
-                                    server_address="localhost")
-os.environ["environment_name"] = "Abstract-ALL"
+# change line 155....in helpers to environment_name= os.environ["environmentName"],
+os.environ["environmentName"] = "Just1Res"
+
+
+dev_helpers.attach_to_cloudshell_as(user="admin", password="xxxx", domain="Global",
+                                    reservation_id="ad024811-7528-42eb-b2c4-d50003951278",
+                                    server_address="svl-dev-quali")
 
 x = EnvironmentSetupResources()
 x.execute()

sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_teardown_resources.py

@@ -1,8 +1,12 @@
 import cloudshell.helpers.scripts.cloudshell_dev_helpers as dev_helpers
 from sandbox_scripts.environment.teardown.teardown_resources import *
+import os
 
-dev_helpers.attach_to_cloudshell_as(user="admin", password="xx", domain="Global",
-                                    reservation_id="bc0517e5-7240-4184-b04d-19e755f9c9a7",
+
+os.environ["environmentName"] = "Just1Res"
+
+dev_helpers.attach_to_cloudshell_as(user="admin", password="xxxx", domain="Global",
+                                    reservation_id="2bc2e45f-63e6-41cb-a8f5-fe34e042a75e",
                                     server_address="svl-dev-quali")
 
 x = EnvironmentTeardownResources()

sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py

@@ -176,6 +176,39 @@ def test_health_check_failed(self):
         ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905")
         self.assertEqual('Health check did not pass for device r1. Health check failed',ret)
 
+    @patch('time.sleep')
+    def test_health_check_failed_twice(self, mock_time):
+        command1 = Mock()
+        command1.Name = 'health_check'
+        self.resource.commands = [command1]
+
+        rd = Mock()
+        rd.Output = "Health check failed"
+        self.mock_api_session.return_value.ExecuteCommand = Mock(return_value=rd)
+        ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905", health_check_attempts=2)
+        self.assertEqual('Health check did not pass for device r1. Health check failed',ret)
+        self.assertEqual(mock_time.call_count,1)
+
+    @patch('time.sleep')
+    def test_health_check_failed_once_succeeds_on_second(self, mock_time):
+        command1 = Mock()
+        command1.Name = 'health_check'
+        self.resource.commands = [command1]
+        self.count = 0
+        def execute_command_return_value(res_id, resource_name, resource_type, command_name, inputs, print_output):
+            rd = Mock()
+            if self.count == 0:
+                rd.Output = "Health check failed"
+                self.count += 1
+            else:
+                rd.Output = "Health check passed"
+            return rd
+
+        self.mock_api_session.return_value.ExecuteCommand.side_effect = execute_command_return_value
+        ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905", health_check_attempts=2)
+        self.assertEqual('',ret, "command was expected to be pass but wasn't")
+        self.assertEqual(mock_time.call_count,1)
+
     def test_health_check_not_found(self):
         ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905")
         self.assertEqual('',ret, "command was expected to be found but wasn't")

sandbox_scripts/environment/setup/setup_script.py

@@ -1,12 +1,10 @@
 from multiprocessing.pool import ThreadPool
 from threading import Lock
-
 from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers
 from cloudshell.api.cloudshell_api import *
 from cloudshell.api.common_cloudshell_api import CloudShellAPIError
 from cloudshell.core.logger.qs_logger import get_qs_logger
 from remap_child_resources_constants import *
-
 from sandbox_scripts.helpers.resource_helpers import *
 from sandbox_scripts.profiler.env_profiler import profileit
 
@@ -27,7 +25,7 @@ def execute(self):
         resource_details_cache = {}
 
         api.WriteMessageToReservationOutput(reservationId=self.reservation_id,
-                                            message='Beginning sandbox setup')
+                                            message= 'Beginning sandbox setup')
 
         self._prepare_connectivity(api, self.reservation_id)
 
@@ -67,7 +65,8 @@ def _prepare_connectivity(self, api, reservation_id):
         :param str reservation_id:
         """
         self.logger.info("Preparing connectivity for reservation {0}".format(self.reservation_id))
-        api.WriteMessageToReservationOutput(reservationId=self.reservation_id, message='Preparing connectivity')
+        api.WriteMessageToReservationOutput(reservationId=self.reservation_id,
+                                            message='Preparing connectivity')
         api.PrepareSandboxConnectivity(reservation_id)
 
     def _try_exeucte_autoload(self, api, deploy_result, resource_details_cache):
@@ -81,7 +80,8 @@ def _try_exeucte_autoload(self, api, deploy_result, resource_details_cache):
 
         if deploy_result is None:
             self.logger.info("No apps to discover")
-            api.WriteMessageToReservationOutput(reservationId=self.reservation_id, message='No apps to discover')
+            api.WriteMessageToReservationOutput(reservationId=self.reservation_id,
+                                                message='No apps to discover')
             return
 
         message_written = False
@@ -140,14 +140,14 @@ def _deploy_apps_in_reservation(self, api, reservation_details):
         if not apps or (len(apps) == 1 and not apps[0].Name):
             self.logger.info("No apps found in reservation {0}".format(self.reservation_id))
             api.WriteMessageToReservationOutput(reservationId=self.reservation_id,
-                                                message='No apps to deploy')
+                                                message= 'No apps to deploy')
             return None
 
         app_names = map(lambda x: x.Name, apps)
         app_inputs = map(lambda x: DeployAppInput(x.Name, "Name", x.Name), apps)
 
         api.WriteMessageToReservationOutput(reservationId=self.reservation_id,
-                                            message='Apps deployment started')
+                                            message= 'Apps deployment started')
         self.logger.info(
             "Deploying apps for reservation {0}. App names: {1}".format(reservation_details, ", ".join(app_names)))
 
@@ -248,8 +248,8 @@ def _configure_apps(self, api, reservation_id):
                     failed_apps.append(conf_res.AppName)
 
             if not failed_apps:
-                api.WriteMessageToReservationOutput(reservationId=reservation_id, message=
-                'Apps were configured successfully.')
+                api.WriteMessageToReservationOutput(reservationId=reservation_id,
+                                                    message='Apps were configured successfully.')
             else:
                 api.WriteMessageToReservationOutput(reservationId=reservation_id, message=
                 'Apps: {0} configuration failed. See logs for more details'.format(

sandbox_scripts/environment/teardown/teardown_VM.py

@@ -1,10 +1,9 @@
 from cloudshell.core.logger import qs_logger
 from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager
 from sandbox_scripts.helpers.Networking.NetworkingSaveNRestore import *
-from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers
 from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase
-from cloudshell.api.common_cloudshell_api import CloudShellAPIError
-from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase
+from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers
+from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError
 
 
 class EnvironmentTeardownVM:
@@ -15,15 +14,20 @@ def __init__(self):
         self.logger = qs_logger.get_qs_logger(log_file_prefix="CloudShell Sandbox Teardown",
                                               log_group=self.reservation_id,
                                               log_category='Teardown')
-        self.sandbox = SandboxBase(self.reservation_id, self.logger)
+        self.sandbox = None
 
-    # ---------------------------
-    # ---------------------------
     def execute(self):
+
+        self.sandbox = SandboxBase(self.reservation_id, self.logger)
+
         self.sandbox.report_info("Beginning VMs cleanup")
+
+        reservation_details = self.sandbox.api_session.GetReservationDetails(self.reservation_id)
+
         saveNRestoreTool = SaveRestoreManager(self.sandbox)
 
         filename = "Snapshot_"+self.reservation_id+".txt"
+
         is_snapshot = False
 
         # if the current reservation was saved as snapshot we look it by reservation_id
@@ -36,19 +40,20 @@ def execute(self):
                     is_snapshot = True
 
         if is_snapshot:
-            self.delete_VM_or_Power_off(to_delete=False)
+            self.delete_VM_or_Power_off(reservation_details, to_delete = False)
+
         else:
-            self.delete_VM_or_Power_off(to_delete=True)
+            self.delete_VM_or_Power_off(reservation_details, to_delete =True)
+
+    def delete_VM_or_Power_off(self, reservation_details, to_delete = False):
 
-    # ---------------------------
-    # ---------------------------
-    def delete_VM_or_Power_off(self, to_delete=False):
         """
-        :param bool to_delete:
+        :param GetReservationDescriptionResponseInfo reservation_details:
+        :param str reservation_id:
         :return:
         """
         # filter out resources not created in this reservation
-        resources = self.sandbox.get_root_vm_resources()
+        resources = reservation_details.ReservationDescription.Resources
 
         pool = ThreadPool()
         async_results = []
@@ -59,9 +64,12 @@ def delete_VM_or_Power_off(self, to_delete=False):
         }
 
         for resource in resources:
-            result_obj = pool.apply_async(self._power_off_or_delete_deployed_app,
-                                (resource, lock, message_status,to_delete))
-            async_results.append(result_obj)
+            resource_details = self.sandbox.api_session.GetResourceDetails(resource.Name)
+            vm_details = resource_details.VmDetails
+            if vm_details and hasattr(vm_details, "UID") and vm_details.UID:
+                result_obj = pool.apply_async(self._power_off_or_delete_deployed_app,
+                                    (resource_details, lock, message_status,to_delete))
+                async_results.append(result_obj)
 
         pool.close()
         pool.join()
@@ -76,36 +84,39 @@ def delete_VM_or_Power_off(self, to_delete=False):
         if resource_to_delete:
             try:
                 self.sandbox.api_session.RemoveResourcesFromReservation(self.reservation_id, resource_to_delete)
-            except CloudShellAPIError as exc:
+            except QualiError as exc:
                 if exc.code == EnvironmentTeardownVM.REMOVE_DEPLOYED_RESOURCE_ERROR:
                     self.sandbox.report_error(error_message=exc.message,
                               log_message="Error executing RemoveResourcesFromReservation command. "
                                           "Error: {0}".format(exc.message),
                               raise_error=True, write_to_output_window=True)
 
 
-    # ---------------------------
-    # ---------------------------
-    def _power_off_or_delete_deployed_app(self, resource, lock, message_status, to_delete):
+    def _power_off_or_delete_deployed_app(self, resource_info, lock, message_status, to_delete):
         """
         :param Lock lock:
         :param (dict of str: Boolean) message_status:
-        :param ResourceBase resource:
+        :param ResourceInfo resource_info:
         :return:
         """
-        if resource.model.lower() =="vcenter static vm":
+        resource_name = resource_info.Name
+
+        if resource_info.ResourceModelName.lower() =="vcenter static vm":
             to_delete = False
 
         try:
+
             if to_delete:
                 with lock:
                     if not message_status['delete']:
                         message_status['delete'] = True
                         self.sandbox.report_info("Apps are being powered off and deleted...",
                                                  write_to_output_window=True)
 
-                self.sandbox.report_info("Executing 'Delete' on deployed app {0}".format(resource.name))
-                return resource.name
+                self.logger.info("Executing 'Delete' on deployed app {0} in reservation {1}"
+                                 .format(resource_name, self.reservation_id))
+
+                return resource_name
 
             else:
                 with lock:
@@ -115,15 +126,15 @@ def _power_off_or_delete_deployed_app(self, resource, lock, message_status, to_d
                                                  write_to_output_window=True)
 
                 with lock:
-                    self.sandbox.report_info("Executing 'Power Off' on deployed app {0}"
-                                     .format(resource.name))
-                    resource.execute_connected_command(self.sandbox.id,"PowerOff", "power")
+                    self.logger.info("Executing 'Power Off' on deployed app {0}"
+                                     .format(resource_name, self.reservation_id))
+                    self.sandbox.api_session.ExecuteResourceConnectedCommand(self.reservation_id, resource_name,
+                                                                        "PowerOff", "power")
 
                 return None
 
         except Exception as exc:
-            err_msg = "Error deleting or powering off deployed app {0}. Error: {1}".format(
-                resource.name, str(exc))
-            self.sandbox.report_error(err_msg,raise_error=False)
+            self.logger.error("Error deleting or powering off deployed app {0} in reservation {1}. Error: {2}"
+                              .format(resource_name, self.reservation_id, str(exc)))
             return None