Implement performance tests in nightly validation to compare read_model and ovc.convert with OVVP output

tests/model_hub_tests/models_hub_common/constants.py

@@ -10,11 +10,13 @@
 loading and heating and includes measurement only one of 2 models - got through convert and read_model.
 Both "converted" and "read_model" modes will be 2 * runtime_measure_duration 
 '''
-runtime_measure_duration = os.environ.get('RUNTIME_MEASURE_DURATION', '60')
+precommit_runtime_measure_duration = os.environ.get('PRECOMMIT_RUNTIME_MEASURE_DURATION', '60')
+nightly_runtime_measure_duration = os.environ.get('NIGHTLY_RUNTIME_MEASURE_DURATION', '15')
 '''
 @brief Time in seconds of heating before measurement
 '''
-runtime_heat_duration = os.environ.get('RUNTIME_HEAT_DURATION', '5')
+precommit_runtime_heat_duration = os.environ.get('PRECOMMIT_RUNTIME_HEAT_DURATION', '5')
+nigtly_runtime_heat_duration = os.environ.get('NIGHTLY_RUNTIME_HEAT_DURATION', '5')
 
 tf_hub_cache_dir = os.environ.get('TFHUB_CACHE_DIR',
                                   os.path.join(tempfile.gettempdir(), "tfhub_modules"))

tests/model_hub_tests/models_hub_common/test_performance_model.py

@@ -61,6 +61,9 @@ def __init__(self):
         self.infer_time_ratio = 0.0
         self.error_message = ''
         self.status = None
+        self.model_name = ''
+        self.model_link = ''
+        self.ie_device = ''
 
 
 class TestModelPerformance:
@@ -120,14 +123,12 @@ def get_read_model(self, model_path: str):
         core = ov.Core()
         return core.read_model(model=model_path)
 
-    def heat_hardware(self, ov_model, inputs) -> None:
-        _, heat_n_repeats, _ = utils.measure(utils.nano_secs(const.runtime_heat_duration), ov_model, (inputs,))
+    def heat_hardware(self, ov_model, inputs, conf) -> None:
+        _, heat_n_repeats, _ = utils.measure(conf.runtime_heat_duration, ov_model, (inputs,))
         print('heat done in {} repeats'.format(heat_n_repeats))
 
-    def measure_inference(self, ov_model, inputs) -> ModelResults:
-        time_slices, infer_n_repeats, real_runtime = utils.measure(utils.nano_secs(const.runtime_measure_duration),
-                                                                   ov_model,
-                                                                   (inputs,))
+    def measure_inference(self, ov_model, inputs, conf) -> ModelResults:
+        time_slices, infer_n_repeats, real_runtime = utils.measure(conf.runtime_measure_duration, ov_model, (inputs,))
         print('measurement done in {} repeats'.format(infer_n_repeats))
         infer_throughput = float(infer_n_repeats * (10 ** 9)) / real_runtime
         infer_mean_time_ns = np.mean(time_slices)
@@ -141,16 +142,19 @@ def measure_inference(self, ov_model, inputs) -> ModelResults:
         results.infer_variance = infer_variance
         return results
 
-    def infer_model(self, ov_model, inputs) -> ModelResults:
-        self.heat_hardware(ov_model, inputs)
-        return self.measure_inference(ov_model, inputs)
+    def infer_model(self, ov_model, inputs, conf) -> ModelResults:
+        self.heat_hardware(ov_model, inputs, conf)
+        return self.measure_inference(ov_model, inputs, conf)
 
     def compile_model(self, model, ie_device):
         core = ov.Core()
         return core.compile_model(model, ie_device)
 
-    def __run(self, model_name, model_link, ie_device):
+    def __run(self, model_name, model_link, ie_device, conf):
         results = Results()
+        results.model_name = model_name
+        results.model_link = model_link
+        results.ie_device = ie_device
         results.status = None
         try:
             results.status = Status.LOAD_MODEL
@@ -168,11 +172,11 @@ def __run(self, model_name, model_link, ie_device):
             results.status = Status.INFER_CONVERTED_MODEL
             results.converted_model_results = utils.call_with_timer('Infer converted model',
                                                                     self.infer_model,
-                                                                    (converted_model, inputs))
+                                                                    (converted_model, inputs, conf))
             results.status = Status.INFER_READ_MODEL
             results.read_model_results = utils.call_with_timer('Infer read model',
                                                                self.infer_model,
-                                                               (read_model, inputs))
+                                                               (read_model, inputs, conf))
 
             infer_time_ratio = (results.converted_model_results.infer_mean_time /
                                 results.read_model_results.infer_mean_time)
@@ -196,10 +200,10 @@ def __run(self, model_name, model_link, ie_device):
                                                                          ex_type=ex_type.__name__, ex_value=ex_value)
         return results
 
-    def run(self, model_name, model_link, ie_device):
+    def run(self, model_name, model_link, ie_device, conf):
         self.result = Results()
         t0 = time.time()
-        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device], model_name,
+        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device, conf], model_name,
                                           self.infer_timeout)
         t1 = time.time()
         utils.print_stat('test run time {} secs', (t1 - t0))

tests/model_hub_tests/models_hub_common/utils.py

@@ -32,6 +32,28 @@ def get_models_list(file_name: str):
     return models
 
 
+def get_skipped_model_links(filename: str):
+    links = set()
+    if not os.path.exists(filename):
+        return links
+    with open(filename) as f:
+        for model_info in f:
+            model_info = model_info.strip()
+            model_name, model_link = model_info.split(',')
+            links.add(model_link)
+    return links
+
+
+def get_models_list_not_skipped(model_list_file: str, skip_list_file: str):
+    skipped_links = get_skipped_model_links(skip_list_file)
+    not_skipped_models = []
+    for model_name, model_link, mark, reason in get_models_list(model_list_file):
+        if model_link in skipped_links:
+            continue
+        not_skipped_models.append((model_name, model_link, mark, reason))
+    return not_skipped_models
+
+
 def compare_two_tensors(ov_res, fw_res, eps):
     is_ok = True
     if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):

tests/model_hub_tests/performance_tests/conftest.py

@@ -7,6 +7,39 @@
 from models_hub_common.utils import get_params
 from py.xml import html
 from models_hub_common.utils import round_num
+import json
+import requests
+
+
+def pytest_sessionstart(session):
+    print('pytest_sessionstart')
+    session.results = []
+
+
+def pytest_sessionfinish(session, exitstatus):
+    print('pytest_sessionfinish')
+    dump_data = []
+    for item in session.results:
+        dump_item = dict()
+        dump_item['version'] = '1'
+        dump_item['status'] = str(item.status)
+        dump_item['ie_device'] = item.ie_device
+        dump_item['model_name'] = item.model_name
+        dump_item['model_link'] = item.model_link
+        dump_item['time_infer_conv'] = str(item.converted_model_results.infer_mean_time)
+        dump_item['time_infer_read'] = str(item.read_model_results.infer_mean_time)
+        dump_item['infer_time_ratio'] = str(item.infer_time_ratio)
+        dump_data.append(dump_item)
+    data = json.dumps(dump_data)
+    url = 'https://openvino-validation-portal.toolbox.iotg.sclab.intel.com/api/v1/Seeder_Tf_Hub_Compile_Read/seed'
+    headers = {
+        'accept': 'application/json',
+        'Content-Type': 'application/json'
+    }
+    try:
+        requests.post(url, headers=headers, data=data)
+    except:
+        print('error while sending data to OVVP')
 
 
 def pytest_generate_tests(metafunc):
@@ -22,6 +55,8 @@ def pytest_runtest_makereport(item, call):
     if getattr(item.obj.__self__, 'result', None) is not None:
         results = item.obj.__self__.result
         report._results = results
+        if report.when == 'call':
+            item.session.results.append(results)
 
 
 @pytest.mark.optionalhook