diff --git a/delfin/drivers/dell_emc/vmax/client.py b/delfin/drivers/dell_emc/vmax/client.py index c76727f6c..5e2839136 100644 --- a/delfin/drivers/dell_emc/vmax/client.py +++ b/delfin/drivers/dell_emc/vmax/client.py @@ -17,6 +17,7 @@ from delfin import exception from delfin.common import constants +from delfin.drivers.dell_emc.vmax import constants as consts from delfin.drivers.dell_emc.vmax import rest, perf_utils LOG = log.getLogger(__name__) @@ -164,7 +165,7 @@ def list_storage_pools(self, storage_id): if int(self.uni_version) < 90: total_cap = pool_info['total_usable_cap_gb'] * units.Gi used_cap = pool_info['total_allocated_cap_gb'] * units.Gi - subscribed_cap =\ + subscribed_cap = \ pool_info['total_subscribed_cap_gb'] * units.Gi else: srp_cap = pool_info['srp_capacity'] @@ -252,7 +253,7 @@ def list_volumes(self, storage_id): sg = vol['storageGroupId'][0] sg_info = self.rest.get_storage_group( self.array_id, self.uni_version, sg) - v['native_storage_pool_id'] =\ + v['native_storage_pool_id'] = \ sg_info.get('srp', default_srps[emulation_type]) v['compressed'] = sg_info.get('compression', False) else: @@ -386,29 +387,86 @@ def clear_alert(self, sequence_number): return self.rest.clear_alert(sequence_number, version=self.uni_version, array=self.array_id) - def get_array_performance_metrics(self, storage_id, start_time, end_time): + def get_storage_metrics(self, storage_id, metrics, start_time, end_time): """Get performance metrics.""" try: - # Fetch VMAX Array Performance data from REST client - # TODO : - # Check whether array is registered for performance collection - # in unisphere - perf_data = self.rest.get_array_performance_metrics( - self.array_id, start_time, end_time) - # parse VMAX REST response to metric->values map - metrics_value_map = perf_utils.parse_performance_data(perf_data) - # prepare labels required for array_leval performance data - labels = {'storage_id': storage_id, 'resource_type': 'array'} - # map to unified delifn metrics - delfin_metrics = perf_utils.\ - map_array_perf_metrics_to_delfin_metrics(metrics_value_map) + perf_list = self.rest.get_storage_metrics( + self.array_id, metrics, start_time, end_time) + + return perf_utils.construct_metrics(storage_id, + consts.STORAGE_METRICS, + consts.STORAGE_CAP, + perf_list) + except Exception: + LOG.error("Failed to get STORAGE metrics for VMAX") + raise + + def get_pool_metrics(self, storage_id, metrics, start_time, end_time): + """Get performance metrics.""" + try: + perf_list = self.rest.get_pool_metrics( + self.array_id, metrics, start_time, end_time) + + metrics_array = perf_utils.construct_metrics( + storage_id, consts.POOL_METRICS, consts.POOL_CAP, perf_list) + + return metrics_array + except Exception: + LOG.error("Failed to get STORAGE POOL metrics for VMAX") + raise + + def get_port_metrics(self, storage_id, metrics, start_time, end_time): + """Get performance metrics.""" + try: + be_perf_list, fe_perf_list, rdf_perf_list = \ + self.rest.get_port_metrics(self.array_id, + metrics, start_time, end_time) + metrics_array = [] - for key in constants.DELFIN_ARRAY_METRICS: - m = constants.metric_struct(name=key, labels=labels, - values=delfin_metrics[key]) - metrics_array.append(m) + metrics_list = perf_utils.construct_metrics( + storage_id, consts.BEPORT_METRICS, + consts.PORT_CAP, be_perf_list) + metrics_array.extend(metrics_list) + + metrics_list = perf_utils.construct_metrics( + storage_id, consts.FEPORT_METRICS, + consts.PORT_CAP, fe_perf_list) + metrics_array.extend(metrics_list) + + metrics_list = perf_utils.construct_metrics( + storage_id, consts.RDFPORT_METRICS, + consts.PORT_CAP, rdf_perf_list) + metrics_array.extend(metrics_list) return metrics_array + except Exception: + LOG.error("Failed to get PORT metrics for VMAX") + raise + + def get_controller_metrics(self, storage_id, + metrics, start_time, end_time): + """Get performance metrics.""" + try: + be_perf_list, fe_perf_list, rdf_perf_list = self.rest.\ + get_controller_metrics(self.array_id, + metrics, start_time, end_time) + + metrics_array = [] + metrics_list = perf_utils.construct_metrics( + storage_id, consts.BEDIRECTOR_METRICS, + consts.CONTROLLER_CAP, be_perf_list) + metrics_array.extend(metrics_list) + metrics_list = perf_utils.construct_metrics( + storage_id, consts.FEDIRECTOR_METRICS, + consts.CONTROLLER_CAP, fe_perf_list) + metrics_array.extend(metrics_list) + + metrics_list = perf_utils.construct_metrics( + storage_id, consts.RDFDIRECTOR_METRICS, + consts.CONTROLLER_CAP, rdf_perf_list) + metrics_array.extend(metrics_list) + + return metrics_array except Exception: - LOG.error("Failed to get performance metrics data for VMAX") + LOG.error("Failed to get CONTROLLER metrics for VMAX") raise diff --git a/delfin/drivers/dell_emc/vmax/constants.py b/delfin/drivers/dell_emc/vmax/constants.py index 5edbe927b..7fb3a1510 100644 --- a/delfin/drivers/dell_emc/vmax/constants.py +++ b/delfin/drivers/dell_emc/vmax/constants.py @@ -17,12 +17,133 @@ # minimum interval supported by VMAX VMAX_PERF_MIN_INTERVAL = 5 -ARRAY_METRICS = ["HostIOs", - "HostMBWritten", - "ReadResponseTime", - "HostMBReads", - "HostReads", - "HostWrites", - "WriteResponseTime" - ] -VMAX_REST_TARGET_URI_ARRAY_PERF = '/performance/Array/metrics' +BEDIRECTOR_METRICS = { + 'iops': 'IOs', + 'throughput': 'MBs', + 'readThroughput': 'MBRead', + 'writeThroughput': 'MBWritten', +} +FEDIRECTOR_METRICS = { + 'iops': 'HostIOs', + 'throughput': 'HostMBs', +} +RDFDIRECTOR_METRICS = { + 'iops': 'IOs', + 'throughput': 'MBSentAndReceived', + 'readThroughput': 'MBRead', + 'writeThroughput': 'MBWritten', + 'responseTime': 'AverageIOServiceTime', +} +BEPORT_METRICS = { + 'iops': 'IOs', + 'throughput': 'MBs', + 'readThroughput': 'MBRead', + 'writeThroughput': 'MBWritten', +} +FEPORT_METRICS = { + 'iops': 'IOs', + 'throughput': 'MBs', + 'readThroughput': 'MBRead', + 'writeThroughput': 'MBWritten', + 'responseTime': 'ResponseTime', +} +RDFPORT_METRICS = { + 'iops': 'IOs', + 'throughput': 'MBs', + 'readThroughput': 'MBRead', + 'writeThroughput': 'MBWritten', +} +POOL_METRICS = { + 'iops': 'HostIOs', + 'readIops': 'HostReads', + 'writeIops': 'HostWrites', + 'throughput': 'HostMBs', + 'readThroughput': 'HostMBReads', + 'writeThroughput': 'HostMBWritten', + 'responseTime': 'ResponseTime', +} +STORAGE_METRICS = { + 'iops': 'HostIOs', + 'readIops': 'HostReads', + 'writeIops': 'HostWrites', + 'throughput': 'HostMBs', + 'readThroughput': 'HostMBReads', + 'writeThroughput': 'HostMBWritten', +} + +IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Input/output operations per second" +} +READ_IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Read input/output operations per second" +} +WRITE_IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Write input/output operations per second" +} +THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data is " + "successfully transferred in MB/s" +} +READ_THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data read is " + "successfully transferred in MB/s" +} +WRITE_THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data write is " + "successfully transferred in MB/s" +} +RESPONSE_TIME_DESCRIPTION = { + "unit": "ms", + "description": "Average time taken for an IO " + "operation in ms" +} +IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of IO requests in KB" +} +READ_IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of read IO requests in KB" +} +WRITE_IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of write IO requests in KB" +} +STORAGE_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION, +} +POOL_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION, +} +CONTROLLER_CAP = { + "iops": IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION, +} +PORT_CAP = { + "iops": IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION, +} diff --git a/delfin/drivers/dell_emc/vmax/perf_utils.py b/delfin/drivers/dell_emc/vmax/perf_utils.py index dce7bd356..fa0da6403 100644 --- a/delfin/drivers/dell_emc/vmax/perf_utils.py +++ b/delfin/drivers/dell_emc/vmax/perf_utils.py @@ -12,88 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time +from delfin.common import constants -from collections import Counter -from delfin.drivers.dell_emc.vmax import constants - - -def epoch_time_ms_now(): - """Get current time in epoch ms. - :returns: epoch time in milli seconds - """ - ms = int(time.time() * 1000) - return ms - - -def epoch_time_interval_ago(interval_seconds=constants.VMAX_PERF_MIN_INTERVAL): - """Get epoch time in milliseconds before an interval - :param interval_seconds: interval in seconds - :returns: epoch time in milliseconds - """ - return int(epoch_time_ms_now() - (interval_seconds * 1000)) - - -def generate_performance_payload(array, start_time, end_time, metrics): - """Generate request payload for VMAX performance POST request - :param array: symmetrixID - :param start_time: start time for collection - :param end_time: end time for collection - :param metrics: metrics to be collected - :returns: payload dictionary - """ - return {'symmetrixId': str(array), - "endDate": end_time, - "startDate": start_time, - "metrics": metrics, - "dataFormat": "Average"} - - -def parse_performance_data(response): +def parse_performance_data(metrics): """Parse metrics response to a map - :param response: response from unispshere REST API + :param metrics: metrics from unispshere REST API :returns: map with key as metric name and value as dictionary containing {timestamp: value} for a the timestamps available """ metrics_map = {} - for metrics in response["resultList"]["result"]: - timestamp = metrics["timestamp"] - for key, value in metrics.items(): - metrics_map[key] = metrics_map.get(key, {}) - metrics_map[key][timestamp] = value + timestamp = metrics["timestamp"] + for key, value in metrics.items(): + metrics_map[key] = metrics_map.get(key, {}) + metrics_map[key][timestamp] = value return metrics_map -def map_array_perf_metrics_to_delfin_metrics(metrics_value_map): - """map vmax array performance metrics values to delfin metrics values - :param metrics_value_map: metric to values map of vmax metrics - :returns: map with key as delfin metric name and value as dictionary - containing {timestamp: value} for a the timestamps available - """ - # read and write response_time - read_response_values_dict = metrics_value_map.get('ReadResponseTime') - write_response_values_dict = metrics_value_map.get('WriteResponseTime') - if read_response_values_dict or write_response_values_dict: - response_time_values_dict = \ - Counter(read_response_values_dict) + \ - Counter(write_response_values_dict) - # bandwidth metrics - read_bandwidth_values_dict = metrics_value_map.get('HostMBReads') - write_bandwidth_values_dict = metrics_value_map.get('HostMBWritten') - if read_bandwidth_values_dict or write_bandwidth_values_dict: - bandwidth_values_dict = \ - Counter(read_bandwidth_values_dict) +\ - Counter(write_bandwidth_values_dict) - throughput_values_dict = metrics_value_map.get('HostIOs') - read_throughput_values_dict = metrics_value_map.get('HostReads') - write_throughput_values_dict = metrics_value_map.get('HostWrites') - # map values to delfin metrics spec - delfin_metrics = {'responseTime': response_time_values_dict, - 'readThroughput': read_bandwidth_values_dict, - 'writeThroughput': write_bandwidth_values_dict, - 'requests': throughput_values_dict, - 'readRequests': read_throughput_values_dict, - 'writeRequests': write_throughput_values_dict, - 'throughput': bandwidth_values_dict} - return delfin_metrics +def construct_metrics(storage_id, resource_metrics, unit_map, perf_list): + metrics_list = [] + metrics_values = {} + for perf in perf_list: + collected_metrics_list = perf.get('metrics') + for collected_metrics in collected_metrics_list: + metrics_map = parse_performance_data(collected_metrics) + + for key, value in resource_metrics.items(): + metrics_map_value = metrics_map.get(value) + if metrics_map_value: + metrics_values[key] = metrics_values.get(key, {}) + for k, v in metrics_map_value.items(): + metrics_values[key][k] = v + + for resource_key, resource_value in metrics_values.items(): + labels = { + 'storage_id': storage_id, + 'resource_type': perf.get('resource_type'), + 'resource_id': perf.get('resource_id'), + 'resource_name': perf.get('resource_name'), + 'type': 'RAW', + 'unit': unit_map[resource_key]['unit'] + } + metrics_res = constants.metric_struct(name=resource_key, + labels=labels, + values=resource_value) + metrics_list.append(metrics_res) + return metrics_list diff --git a/delfin/drivers/dell_emc/vmax/rest.py b/delfin/drivers/dell_emc/vmax/rest.py index 976c15aec..0ecdcde41 100644 --- a/delfin/drivers/dell_emc/vmax/rest.py +++ b/delfin/drivers/dell_emc/vmax/rest.py @@ -28,10 +28,12 @@ from delfin import exception from delfin import ssl_utils from delfin.common import alert_util -from delfin.drivers.dell_emc.vmax import perf_utils, constants +from delfin.common import constants as delfin_const +from delfin.drivers.dell_emc.vmax import constants from delfin.i18n import _ LOG = logging.getLogger(__name__) +PERFORMANCE = 'performance' SLOPROVISIONING = 'sloprovisioning' SYSTEM = 'system' SYMMETRIX = 'symmetrix' @@ -115,7 +117,6 @@ def request(self, target_uri, method, params=None, request_object=None, :raises: StorageBackendException, Timeout, ConnectionError, HTTPError, SSLError """ - url, message, status_code, response = None, None, None, None if not self.session: self.establish_rest_session() @@ -200,7 +201,7 @@ def check_status_code_success(operation, status_code, message): exception_message = ( _("Error %(operation)s. The status code received is %(sc)s " "and the message is %(message)s.") % { - 'operation': operation, 'sc': status_code, + 'operation': operation, 'sc': str(status_code), 'message': message}) raise exception.StorageBackendException( message=exception_message) @@ -707,31 +708,416 @@ def post_request(self, target_uri, payload): :param payload: the payload :returns: status_code -- int, message -- string, server response """ - status_code, message = self.request(target_uri, POST, request_object=payload) - operation = 'POST request for URL' % {target_uri} + resource_object = None + if status_code == STATUS_200: + resource_object = message + resource_object = self.list_pagination(resource_object) + operation = 'POST request for URL' self.check_status_code_success( - operation, status_code, message) - return status_code, message + operation, status_code, resource_object) + + return status_code, resource_object + + def get_array_keys(self, array): + target_uri = '/performance/Array/keys' + + response = self.get_request(target_uri, PERFORMANCE, None) + if response is None: + err_msg = "Failed to get Array keys from VMAX: {0}"\ + .format(str(array)) + LOG.error(err_msg) + + return response - def get_array_performance_metrics(self, array, start_time, end_time): + def get_resource_keys(self, array, resource, payload=None): + if payload is None: + payload = {} + + payload['symmetrixId'] = str(array) + target_uri = '/performance/{0}/keys'.format(resource) + sc, response = self.post_request(target_uri, payload) + if response is None: + err_msg = "Failed to get {0} keys from VMAX: {1} status: {2}"\ + .format(resource, str(array), sc) + LOG.error(err_msg) + + return response + + def get_resource_metrics(self, array, start_time, + end_time, resource, metrics, + payload=None): + if payload is None: + payload = {} + + payload['symmetrixId'] = str(array) + payload['startDate'] = start_time + payload['endDate'] = end_time + payload['metrics'] = metrics + payload['dataFormat'] = 'Average' + target_uri = '/performance/{0}/metrics'.format(resource) + + status_code, response = self.post_request(target_uri, payload) + if status_code != STATUS_200: + err_msg = "Failed to get {0} metrics from VMAX: {1}" \ + .format(resource, str(array)) + LOG.error(err_msg) + return None + return response + + def get_storage_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + storage_metrics = [] + for k in metrics.keys(): + vmax_key = constants.STORAGE_METRICS.get(k) + if vmax_key: + storage_metrics.append(vmax_key) + + keys = self.get_array_keys(array) + keys_dict = None + if keys: + keys_dict = keys.get('arrayInfo', None) + + metrics_list = [] + for key_dict in keys_dict: + if key_dict.get('symmetrixId') == array: + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'Array', + storage_metrics, payload=None) + if metrics_res: + label = { + 'resource_id': key_dict.get('symmetrixId'), + 'resource_name': 'VMAX' + key_dict.get('symmetrixId'), + 'resource_type': delfin_const.ResourceType.STORAGE, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_pool_metrics(self, array, metrics, start_time, end_time): """Get a array performance metrics from VMAX unipshere REST API. :param array: the array serial number + :param metrics: required metrics :param start_time: start time for collection :param end_time: end time for collection :returns: message -- response from unipshere REST API """ + pool_metrics = [] + for k in metrics.keys(): + vmax_key = constants.POOL_METRICS.get(k) + if vmax_key: + pool_metrics.append(vmax_key) + + keys = self.get_resource_keys(array, 'SRP') + keys_dict = None + if keys: + keys_dict = keys.get('srpInfo', None) + + metrics_list = [] + for key_dict in keys_dict: + payload = {'srpId': key_dict.get('srpId')} + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'SRP', + pool_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('srpId'), + 'resource_name': key_dict.get('srpId'), + 'resource_type': delfin_const.ResourceType.STORAGE_POOL, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_fedirector_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + fedirector_metrics = [] + for k in metrics.keys(): + vmax_key = constants.FEDIRECTOR_METRICS.get(k) + if vmax_key: + fedirector_metrics.append(vmax_key) + + keys = self.get_resource_keys(array, 'FEDirector') + keys_dict = None + if keys: + keys_dict = keys.get('feDirectorInfo', None) + + metrics_list = [] + for key_dict in keys_dict: + payload = {'directorId': key_dict.get('directorId')} + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'FEDirector', + fedirector_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('directorId'), + 'resource_name': 'FEDirector_' + + key_dict.get('directorId'), + 'resource_type': delfin_const.ResourceType.CONTROLLER, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_bedirector_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + bedirector_metrics = [] + for k in metrics.keys(): + vmax_key = constants.BEDIRECTOR_METRICS.get(k) + if vmax_key: + bedirector_metrics.append(vmax_key) + + keys = self.get_resource_keys(array, 'BEDirector') + keys_dict = None + if keys: + keys_dict = keys.get('beDirectorInfo', None) + + metrics_list = [] + for key_dict in keys_dict: + payload = {'directorId': key_dict.get('directorId')} + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'BEDirector', + bedirector_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('directorId'), + 'resource_name': 'BEDirector_' + + key_dict.get('directorId'), + 'resource_type': delfin_const.ResourceType.CONTROLLER, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_rdfdirector_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + rdfdirector_metrics = [] + for k in metrics.keys(): + vmax_key = constants.RDFDIRECTOR_METRICS.get(k) + if vmax_key: + rdfdirector_metrics.append(vmax_key) + + keys = self.get_resource_keys(array, 'RDFDirector') + keys_dict = None + if keys: + keys_dict = keys.get('rdfDirectorInfo', None) + + metrics_list = [] + for key_dict in keys_dict: + payload = {'directorId': key_dict.get('directorId')} + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'RDFDirector', + rdfdirector_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('directorId'), + 'resource_name': 'RDFDirector_' + + key_dict.get('directorId'), + 'resource_type': delfin_const.ResourceType.CONTROLLER, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_controller_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + be_metrics = self.get_bedirector_metrics( + array, metrics, start_time, end_time) + fe_metrics = self.get_fedirector_metrics( + array, metrics, start_time, end_time) + rdf_metrics = self.get_rdfdirector_metrics( + array, metrics, start_time, end_time) - target_uri = constants.VMAX_REST_TARGET_URI_ARRAY_PERF - payload = perf_utils.generate_performance_payload( - array, start_time, end_time, constants.ARRAY_METRICS) + return be_metrics, fe_metrics, rdf_metrics - status_code, message = self.post_request(target_uri, payload) - # Expected 200 when POST request has metrics in response body - if status_code != STATUS_200: - raise exception.StoragePerformanceCollectionFailed(message) - return message + def get_feport_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + feport_metrics = [] + for k in metrics.keys(): + vmax_key = constants.RDFDIRECTOR_METRICS.get(k) + if vmax_key: + feport_metrics.append(vmax_key) + + director_keys = self.get_resource_keys(array, 'FEDirector') + director_keys_dict = None + if director_keys: + director_keys_dict = director_keys.get('feDirectorInfo', None) + + metrics_list = [] + for director_key_dict in director_keys_dict: + payload = {'directorId': director_key_dict.get('directorId')} + keys = self.get_resource_keys(array, 'FEPort', payload=payload) + keys_dict = None + if keys: + keys_dict = keys.get('fePortInfo', None) + + for key_dict in keys_dict: + payload['portId'] = key_dict.get('portId') + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'FEPort', + feport_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('portId'), + 'resource_name': 'FEPort_' + + director_key_dict.get('directorId') + + '_' + key_dict.get('portId'), + 'resource_type': delfin_const.ResourceType.PORT, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_beport_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + + beport_metrics = [] + for k in metrics.keys(): + vmax_key = constants.BEPORT_METRICS.get(k) + if vmax_key: + beport_metrics.append(vmax_key) + + director_keys = self.get_resource_keys(array, 'BEDirector') + director_keys_dict = None + if director_keys: + director_keys_dict = director_keys.get('beDirectorInfo', None) + + metrics_list = [] + for director_key_dict in director_keys_dict: + payload = {'directorId': director_key_dict.get('directorId')} + keys = self.get_resource_keys(array, 'BEPort', payload=payload) + keys_dict = None + if keys: + keys_dict = keys.get('bePortInfo', None) + + for key_dict in keys_dict: + payload['portId'] = key_dict.get('portId') + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'BEPort', + beport_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('portId'), + 'resource_name': 'BEPort_' + + director_key_dict.get('directorId') + + '_' + key_dict.get('portId'), + 'resource_type': delfin_const.ResourceType.PORT, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_rdfport_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + rdfport_metrics = [] + for k in metrics.keys(): + vmax_key = constants.RDFPORT_METRICS.get(k) + if vmax_key: + rdfport_metrics.append(vmax_key) + + director_keys = self.get_resource_keys(array, 'RDFDirector') + director_keys_dict = None + if director_keys: + director_keys_dict = director_keys.get('rdfDirectorInfo', None) + + metrics_list = [] + for director_key_dict in director_keys_dict: + payload = {'directorId': director_key_dict.get('directorId')} + keys = self.get_resource_keys(array, 'RDFPort', payload=payload) + keys_dict = None + if keys: + keys_dict = keys.get('rdfPortInfo', None) + + for key_dict in keys_dict: + payload['portId'] = key_dict.get('portId') + metrics_res = self.get_resource_metrics( + array, start_time, end_time, 'RDFPort', + rdfport_metrics, payload=payload) + if metrics_res: + label = { + 'resource_id': key_dict.get('portId'), + 'resource_name': 'BEPort_' + + director_key_dict.get('directorId') + + '_' + key_dict.get('portId'), + 'resource_type': delfin_const.ResourceType.PORT, + 'metrics': metrics_res + } + metrics_list.append(label) + + return metrics_list + + def get_port_metrics(self, array, metrics, start_time, end_time): + """Get a array performance metrics from VMAX unipshere REST API. + :param array: the array serial number + :param metrics: required metrics + :param start_time: start time for collection + :param end_time: end time for collection + :returns: message -- response from unipshere REST API + """ + + be_metrics = self.get_beport_metrics( + array, metrics, start_time, end_time) + fe_metrics = self.get_feport_metrics( + array, metrics, start_time, end_time) + rdf_metrics = self.get_rdfport_metrics( + array, metrics, start_time, end_time) + return be_metrics, fe_metrics, rdf_metrics def list_pagination(self, list_info): """Process lists under or over the maxPageSize diff --git a/delfin/drivers/dell_emc/vmax/vmax.py b/delfin/drivers/dell_emc/vmax/vmax.py index b02b12567..da97bb7d3 100644 --- a/delfin/drivers/dell_emc/vmax/vmax.py +++ b/delfin/drivers/dell_emc/vmax/vmax.py @@ -17,6 +17,7 @@ from delfin.common import constants from delfin.drivers import driver from delfin.drivers.dell_emc.vmax import client +from delfin.drivers.dell_emc.vmax import constants as consts from delfin.drivers.dell_emc.vmax.alert_handler import snmp_alerts from delfin.drivers.dell_emc.vmax.alert_handler import unisphere_alerts @@ -104,8 +105,45 @@ def list_alerts(self, context, query_para): def collect_perf_metrics(self, context, storage_id, resource_metrics, start_time, end_time): - return self.client.get_array_performance_metrics(self.storage_id, - start_time, end_time) + metrics = [] + try: + # storage metrics + if resource_metrics.get(constants.ResourceType.STORAGE): + storage_metrics = self.client.get_storage_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.STORAGE), + start_time, end_time) + metrics.extend(storage_metrics) + + # storage-pool metrics + if resource_metrics.get(constants.ResourceType.STORAGE_POOL): + pool_metrics = self.client.get_pool_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.STORAGE_POOL), + start_time, end_time) + metrics.extend(pool_metrics) + + # controller metrics + if resource_metrics.get(constants.ResourceType.CONTROLLER): + controller_metrics = self.client.get_controller_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.CONTROLLER), + start_time, end_time) + metrics.extend(controller_metrics) + + # port metrics + if resource_metrics.get(constants.ResourceType.PORT): + port_metrics = self.client.get_port_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.PORT), + start_time, end_time) + metrics.extend(port_metrics) + + except Exception: + LOG.error("Failed to collect metrics from VMAX") + raise + + return metrics @staticmethod def get_capabilities(context): @@ -113,39 +151,9 @@ def get_capabilities(context): return { 'is_historic': True, 'resource_metrics': { - "storage": { - "throughput": { - "unit": "MB/s", - "description": "Represents how much data is " - "successfully transferred in MB/s" - }, - "responseTime": { - "unit": "ms", - "description": "Average time taken for an IO " - "operation in ms" - }, - "requests": { - "unit": "IOPS", - "description": "Input/output operations per second" - }, - "readThroughput": { - "unit": "MB/s", - "description": "Represents how much data read is " - "successfully transferred in MB/s" - }, - "writeThroughput": { - "unit": "MB/s", - "description": "Represents how much data write is " - "successfully transferred in MB/s" - }, - "readRequests": { - "unit": "IOPS", - "description": "Read requests per second" - }, - "writeRequests": { - "unit": "IOPS", - "description": "Write requests per second" - }, - } + constants.ResourceType.STORAGE: consts.STORAGE_CAP, + constants.ResourceType.STORAGE_POOL: consts.POOL_CAP, + constants.ResourceType.CONTROLLER: consts.CONTROLLER_CAP, + constants.ResourceType.PORT: consts.PORT_CAP, } } diff --git a/delfin/exporter/example.py b/delfin/exporter/example.py index 2009ff6cf..5747189d8 100644 --- a/delfin/exporter/example.py +++ b/delfin/exporter/example.py @@ -20,9 +20,9 @@ class AlertExporterExample(base_exporter.BaseExporter): def dispatch(self, ctxt, data): - LOG.info("AlertExporterExample, report data: %s" % data) + LOG.debug("AlertExporterExample, report data: %s" % data) class PerformanceExporterExample(base_exporter.BaseExporter): def dispatch(self, ctxt, data): - LOG.info("PerformanceExporterExample, report data: %s" % data) + LOG.debug("PerformanceExporterExample, report data: %s" % data) diff --git a/delfin/tests/unit/drivers/dell_emc/vmax/test_vmax.py b/delfin/tests/unit/drivers/dell_emc/vmax/test_vmax.py index a80d4dc5b..d868b301b 100644 --- a/delfin/tests/unit/drivers/dell_emc/vmax/test_vmax.py +++ b/delfin/tests/unit/drivers/dell_emc/vmax/test_vmax.py @@ -19,8 +19,7 @@ from delfin import context from delfin import exception -from delfin.common import config # noqa -from delfin.common import constants +from delfin.common import constants, config # noqa from delfin.drivers.dell_emc.vmax.rest import VMaxRest from delfin.drivers.dell_emc.vmax.vmax import VMAXStorageDriver @@ -569,271 +568,6 @@ def test_list_ports(self, mock_unisphere_version, self.assertIn('Exception from Storage Backend:', str(exc.exception)) - @mock.patch.object(VMaxRest, 'post_request') - @mock.patch.object(VMaxRest, 'get_vmax_array_details') - @mock.patch.object(VMaxRest, 'get_array_detail') - @mock.patch.object(VMaxRest, 'get_uni_version') - @mock.patch.object(VMaxRest, 'get_unisphere_version') - def test_get_storage_performance(self, mock_unisphere_version, - mock_version, mock_array, - mock_array_details, - mock_performnace): - vmax_array_perf_resp_historic = { - "expirationTime": 1600172441701, - "count": 4321, - "maxPageSize": 1000, - "id": "d495891f-1607-42b7-ba8d-44d0786bd335_0", - "resultList": { - "result": [ - { - "HostIOs": 296.1, - "HostMBWritten": 0.31862956, - "ReadResponseTime": 4.4177675, - "HostMBReads": 0.05016927, - "HostReads": 14.056666, - "HostWrites": 25.78, - "WriteResponseTime": 4.7228317, - "timestamp": 1598875800000 - }, - { - "HostIOs": 350.22998, - "HostMBWritten": 0.40306965, - "ReadResponseTime": 4.396796, - "HostMBReads": 0.043291014, - "HostReads": 13.213333, - "HostWrites": 45.97333, - "WriteResponseTime": 4.7806735, - "timestamp": 1598876100000 - }, - { - "HostIOs": 297.63333, - "HostMBWritten": 0.25046548, - "ReadResponseTime": 4.3915706, - "HostMBReads": 0.042753905, - "HostReads": 13.176666, - "HostWrites": 28.643333, - "WriteResponseTime": 4.8760557, - "timestamp": 1598876400000 - } - ] - } - } - vmax_array_perf_resp_real_time = { - "expirationTime": 1600172441701, - "count": 4321, - "maxPageSize": 1000, - "id": "d495891f-1607-42b7-ba8d-44d0786bd335_0", - "resultList": { - "result": [ - { - "HostIOs": 296.1, - "HostMBWritten": 0.31862956, - "ReadResponseTime": 4.4177675, - "HostMBReads": 0.05016927, - "HostReads": 14.056666, - "HostWrites": 25.78, - "WriteResponseTime": 4.7228317, - "timestamp": 1598875800000 - } - ] - } - } - - expected_historic = [ - constants.metric_struct(name='responseTime', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 9.1405992, - 1598876400000: - 9.2676263, - 1598876100000: - 9.1774695} - ), - constants.metric_struct(name='throughput', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 0.36879882999999997, - 1598876400000: - 0.293219385, - 1598876100000: - 0.446360664} - ), - constants.metric_struct(name='readThroughput', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 0.05016927, - 1598876100000: - 0.043291014, - 1598876400000: - 0.042753905} - ), - constants.metric_struct(name='writeThroughput', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 0.31862956, - 1598876100000: - 0.40306965, - 1598876400000: - 0.25046548} - ), - constants.metric_struct(name='requests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: 296.1, - 1598876100000: - 350.22998, - 1598876400000: - 297.63333} - ), - constants.metric_struct(name='readRequests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 14.056666, - 1598876100000: - 13.213333, - 1598876400000: - 13.176666} - ), - constants.metric_struct(name='writeRequests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: 25.78, - 1598876100000: - 45.97333, - 1598876400000: - 28.643333} - ) - ] - - expected_realtime = [ - constants.metric_struct(name='responseTime', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 9.1405992 - } - ), - constants.metric_struct(name='throughput', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 0.36879882999999997 - } - ), - constants.metric_struct(name='readThroughput', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 0.05016927 - } - ), - constants.metric_struct(name='writeThroughput', - labels={ - 'storage_id': '12345', - 'resource_type': 'array' - }, - values={ - 1598875800000: 0.31862956 - - } - ), - constants.metric_struct(name='requests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: 296.1 - } - ), - constants.metric_struct(name='readRequests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: - 14.056666 - } - ), - constants.metric_struct(name='writeRequests', - labels={ - 'storage_id': '12345', - 'resource_type': - 'array'}, - values={ - 1598875800000: 25.78 - } - ) - ] - - kwargs = VMAX_STORAGE_CONF - mock_version.return_value = ['V9.0.2.7', '90'] - mock_unisphere_version.return_value = ['V9.0.2.7', '90'] - mock_array.return_value = {'symmetrixId': ['00112233']} - mock_array_details.return_value = { - 'model': 'VMAX250F', - 'ucode': '5978.221.221', - 'display_name': 'VMAX250F-00112233'} - mock_performnace.return_value = 200, vmax_array_perf_resp_historic - - driver = VMAXStorageDriver(**kwargs) - self.assertEqual(driver.storage_id, "12345") - self.assertEqual(driver.client.array_id, "00112233") - - ret = driver.collect_perf_metrics(context, '12345', "", 10000000, - 10900000) - self.assertEqual(ret, expected_historic) - - mock_performnace.return_value = 200, vmax_array_perf_resp_real_time - ret = driver.collect_perf_metrics(context, '12345', "", 10900000, - 10900000) - self.assertEqual(ret, expected_realtime) - - mock_performnace.side_effect = \ - exception.StoragePerformanceCollectionFailed - with self.assertRaises(Exception) as exc: - ret = driver.collect_perf_metrics(context, '12345', "", 10000000, - 10900000) - - self.assertIn('Failed to collect performance metrics. Reason', - str(exc.exception)) - @mock.patch.object(Session, 'request') @mock.patch.object(VMaxRest, 'get_array_detail') @mock.patch.object(VMaxRest, 'get_uni_version') @@ -879,5 +613,266 @@ def test_get_capabilities(self, mock_unisphere_version, self.assertIsInstance(capabilities, dict) self.assertEqual(capabilities['is_historic'], True) self.assertIsInstance(capabilities['resource_metrics'], dict) - # Only support storage metrics - self.assertEqual(len(capabilities['resource_metrics']), 1) + # Support storage, storage_pool, controller and port metrics + self.assertEqual(len(capabilities['resource_metrics']), 4) + + @mock.patch.object(VMaxRest, 'get_resource_metrics') + @mock.patch.object(VMaxRest, 'get_resource_keys') + @mock.patch.object(VMaxRest, 'get_array_keys') + @mock.patch.object(VMaxRest, 'get_array_detail') + @mock.patch.object(VMaxRest, 'get_uni_version') + @mock.patch.object(VMaxRest, 'get_unisphere_version') + def test_collect_perf_metrics(self, mock_unisphere_version, + mock_version, + mock_array, mock_array_keys, + mock_r_keys, mock_r_metrics): + expected = [ + constants.metric_struct(name='iops', + labels={ + 'storage_id': '12345', + 'resource_type': 'storage', + 'resource_id': '00112233', + 'resource_name': 'VMAX00112233', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1566550500000: 417.42667} + ), + constants.metric_struct(name='iops', + labels={ + 'storage_id': '12345', + 'resource_type': 'storagePool', + 'resource_id': 'SRP_1', + 'resource_name': 'SRP_1', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1566550800000: 304.8} + ), + constants.metric_struct(name='iops', + labels={ + 'storage_id': '12345', + 'resource_type': 'controller', + 'resource_id': 'DF-1C', + 'resource_name': 'BEDirector_DF-1C', + 'type': 'RAW', + 'unit': 'IOPS' + }, + values={1566987000000: 248.40666} + ), + constants.metric_struct(name='iops', + labels={ + 'storage_id': '12345', + 'resource_type': 'port', + 'resource_id': '12', + 'resource_name': 'BEPort_DF-1C_12', + 'type': 'RAW', + 'unit': 'IOPS' + }, + values={1566987000000: 6.693333} + ), + ] + kwargs = VMAX_STORAGE_CONF + mock_version.return_value = ['V9.0.2.7', '90'] + mock_unisphere_version.return_value = ['V9.0.2.7', '90'] + mock_array.return_value = {'symmetrixId': ['00112233']} + + driver = VMAXStorageDriver(**kwargs) + self.assertEqual(driver.storage_id, "12345") + self.assertEqual(driver.client.array_id, "00112233") + ret_array_key = { + "arrayInfo": [{ + "symmetrixId": "00112233", + "firstAvailableDate": "1566146400000", + "lastAvailableDate": "1566550800000", + }] + } + ret_pool_key = { + "srpInfo": [ + { + "srpId": "SRP_1", + "firstAvailableDate": 1567065600000, + "lastAvailableDate": 1568130900000 + }, + ] + } + ret_be_dir_key = { + "beDirectorInfo": [ + { + "directorId": "DF-1C", + "firstAvailableDate": 1566557100000, + "lastAvailableDate": 1566987300000 + }, + ] + } + ret_fe_dir_key = { + "feDirectorInfo": [ + { + "directorId": "FA-1D", + "firstAvailableDate": 1567065600000, + "lastAvailableDate": 1567093200000 + }, + ] + } + ret_rdf_dir_key = { + "rdfDirectorInfo": [ + { + "directorId": "RF-1F", + "firstAvailableDate": 1567065600000, + "lastAvailableDate": 1567437900000 + }, + ] + } + ret_be_port_key = { + "bePortInfo": [ + { + "portId": "12", + "firstAvailableDate": 1566557100000, + "lastAvailableDate": 1566988500000 + }, + ] + } + ret_fe_port_key = { + "fePortInfo": [ + { + "firstAvailableDate": 1567065600000, + "lastAvailableDate": 1567162500000, + "portId": "4" + }, + ] + } + ret_rdf_port_key = { + "rdfPortInfo": [ + { + "portId": "7", + "firstAvailableDate": 1567065600000, + "lastAvailableDate": 1567439100000 + } + ] + } + mock_array_keys.return_value = ret_array_key + mock_r_keys.side_effect = [ + ret_pool_key, + ret_be_dir_key, ret_fe_dir_key, ret_rdf_dir_key, + ret_be_dir_key, ret_be_port_key, + ret_fe_dir_key, ret_fe_port_key, + ret_rdf_dir_key, ret_rdf_port_key, + ] + ret_array_metric = { + "HostIOs": 417.42667, + "HostMBs": 0.0018131511, + "FEReqs": 23.55, + "BEIOs": 25.216667, + "BEReqs": 5.55, + "PercentCacheWP": 0.031244868, + "timestamp": 1566550500000 + } + ret_pool_metric = { + "HostIOs": 304.8, + "HostMBs": 0.005192057, + "FEReqs": 23.04, + "BEIOs": 22.566668, + "BEReqs": 4.7733335, + "PercentCacheWP": 0.018810686, + "timestamp": 1566550800000 + } + ret_be_dir_metric = { + "PercentBusy": 0.025403459, + "IOs": 248.40666, + "Reqs": 3.91, + "MBRead": 1.7852213, + "MBWritten": 0.37213543, + "PercentNonIOBusy": 0.0, + "timestamp": 1566987000000 + } + ret_fe_dir_metric = { + "PercentBusy": 2.54652, + "HostIOs": 3436.9368, + "HostMBs": 51.7072, + "Reqs": 3330.5947, + "ReadResponseTime": 0.12916493, + "WriteResponseTime": 0.3310084, + "timestamp": 1567078200000 + } + ret_rdf_dir_metric = { + "PercentBusy": 4.8083158, + "IOs": 1474.2234, + "WriteReqs": 1189.76, + "MBWritten": 54.89597, + "MBRead": 0.4565983, + "MBSentAndReceived": 55.35257, + "AvgIOServiceTime": 0.89211756, + "CopyIOs": 0.0, + "CopyMBs": 0.0, + "timestamp": 1567161600000 + } + ret_be_port_metric = { + "Reads": 4.7, + "Writes": 1.9933333, + "IOs": 6.693333, + "MBRead": 0.43401042, + "MBWritten": 0.10486979, + "MBs": 0.5388802, + "AvgIOSize": 82.44224, + "PercentBusy": 0.013356605, + "timestamp": 1566987000000 + } + ret_fe_port_metric = { + "ResponseTime": 0.1263021, + "ReadResponseTime": 0.1263021, + "WriteResponseTime": 0.0, + "Reads": 0.32, + "Writes": 0.0, + "IOs": 0.32, + "MBRead": 4.296875E-4, + "MBWritten": 0.0, + "MBs": 4.296875E-4, + "AvgIOSize": 1.375, + "SpeedGBs": 16.0, + "PercentBusy": 2.6226044E-5, + "timestamp": 1567161600000 + } + ret_rdf_port_metric = { + "Reads": 0.0, + "Writes": 1216.7633, + "IOs": 1216.7633, + "MBRead": 0.0, + "MBWritten": 57.559597, + "MBs": 57.559597, + "AvgIOSize": 48.440834, + "SpeedGBs": 16.0, + "PercentBusy": 3.5131588, + "timestamp": 1567161600000 + } + mock_r_metrics.side_effect = [ + [ret_array_metric], + [ret_pool_metric], + [ret_be_dir_metric], + [ret_fe_dir_metric], + [ret_rdf_dir_metric], + [ret_be_port_metric], + [ret_fe_port_metric], + [ret_rdf_port_metric], + ] + resource_metrics = { + 'storage': {'iops': {'unit': 'IOPS'}}, + 'storagePool': {'iops': {'unit': 'IOPS'}}, + 'controller': {'iops': {'unit': 'IOPS'}}, + 'port': {'iops': {'unit': 'IOPS'}}, + } + ret = driver.collect_perf_metrics(context, + driver.storage_id, + resource_metrics, + 1000, 2000) + + self.assertEqual(ret[0], expected[0]) + self.assertEqual(ret[2], expected[1]) + self.assertEqual(ret[4], expected[2]) + self.assertEqual(ret[13], expected[3]) + + with self.assertRaises(Exception) as exc: + driver.collect_perf_metrics(context, + driver.storage_id, + resource_metrics, + 1000, 2000 + ) + + self.assertIn('', str(exc.exception))