update ydb/test/olap from main

ydb/tests/olap/lib/allure_utils.py

@@ -3,54 +3,105 @@
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.results_processor import ResultsProcessor
 from urllib.parse import urlencode
+from datetime import datetime
+from copy import deepcopy
+from pytz import timezone
+
+
+def _set_monitoring(test_info: dict[str, str], start_time: float, end_time: float) -> None:
+    monitoring_start = int((start_time) * 1000)
+    monitoring_end = int((end_time) * 1000)
+    database = '/' + test_info.get('database', '*')
+    # monitoring does not show intervals less 1 minute.
+    monitoring_addition = 60000 - (monitoring_end - monitoring_start)
+    if monitoring_addition > 0:
+        monitoring_start -= monitoring_addition
+        monitoring_end += monitoring_addition
+
+    if len(YdbCluster.get_monitoring_urls()) > 0:
+        test_info['monitoring'] = ', '.join([
+            f"<a target='_blank' href='{monitoring.url.format(
+                database=database,
+                start_time=monitoring_start,
+                end_time=monitoring_end
+            )}'>{monitoring.caption}</a>"
+            for monitoring in YdbCluster.get_monitoring_urls()
+        ])
+
+
+def _set_coredumps(test_info: dict[str, str]) -> None:
+    if test_info['name'].startswith('ydb-k8s'):
+        core_link = f"https://coredumps.n.yandex-team.ru/index?itype=kikimr&host_list={test_info['nodes_wilcard']}&show_fixed=True"
+    else:
+        core_link = f"https://kikimr-cores.n.yandex-team.ru/show?server={test_info['nodes_wilcard']}%2A"
+    test_info['coredumps'] = f"<a target='_blank' href='{core_link}'>link</a>"
+
+
+def _set_results_plot(test_info: dict[str, str], suite: str, test: str, refference_set: str) -> None:
+    if not ResultsProcessor.send_results:
+        return
+    params = urlencode({
+        'tab': 'o8',
+        'suite_b2rp': suite,
+        'test_ehyw': test,
+        'db_fmdl': ResultsProcessor.get_cluster_id(),
+        'cluster_dufr': refference_set
+    })
+    test_info['results_plot'] = f"<a target='_blank' href='https://datalens.yandex-team.ru/iqnd4b1miaz27-testy-ydb?{params}'>link</a>"
+
+
+def _set_logs_command(test_info: dict[str, str], start_time: float, end_time: float):
+    hosts = []
+    for node in YdbCluster.get_cluster_nodes():
+        if node.role == YdbCluster.Node.Role.STORAGE:
+            hosts.append(node.host)
+    hosts_cmd = ' '.join([f'-H {h}' for h in hosts])
+    tz = timezone('Europe/Moscow')
+    start = datetime.fromtimestamp(start_time, tz).isoformat()
+    end = datetime.fromtimestamp(end_time, tz).isoformat()
+    time_cmd = f'-S "{start}" -U "{end}"'
+    cmd = f"parallel-ssh {hosts_cmd} -o . 'ulimit -n 100500;unified_agent select {time_cmd} -s kikimr'"
+    test_info['logs_command'] = f'<code>{cmd}</code>'
 
 
 def allure_test_description(
     suite: str,
     test: str,
+    start_time: float,
+    end_time: float,
     addition_table_strings: dict[str, any] = {},
     attachments: tuple[str, str, allure.attachment_type] = [],
-    refference_set: str = ''
+    refference_set: str = '',
 ):
     def _pretty_str(s):
         return ' '.join(s.split('_')).capitalize()
 
     allure.dynamic.title(f'{suite}.{test}')
     for body, name, type in attachments:
         allure.attach(body, name, type)
-    test_info = YdbCluster.get_cluster_info()
-    if test_info['name'].startswith('ydb-k8s'):
-        core_link = f"https://coredumps.n.yandex-team.ru/index?itype=kikimr&host_list={test_info['nodes_wilcard']}&show_fixed=True"
-    else:
-        core_link = f"https://kikimr-cores.n.yandex-team.ru/show?server={test_info['nodes_wilcard']}%2A"
-    monitoring_cluster = (
-        YdbCluster.monitoring_cluster if YdbCluster.monitoring_cluster is not None else test_info['name']
-    )
+
+    test_info = deepcopy(YdbCluster.get_cluster_info())
     test_info.update(addition_table_strings)
+
+    _set_monitoring(test_info, start_time, end_time)
+    _set_coredumps(test_info)
+    _set_results_plot(test_info, suite, test, refference_set)
+    _set_logs_command(test_info, start_time, end_time)
+
+    service_url = YdbCluster._get_service_url()
+    db = test_info['database']
     test_info.update(
         {
             'table_path': YdbCluster.tables_path,
-            'monitoring': (
-                f"<a target='_blank' href='https://monitoring.yandex-team.ru/projects/kikimr/dashboards/mone0310v4dbc6kui89v?"
-                f"p.cluster={monitoring_cluster}&p.database=/{test_info['database']}'>link</a>"
-            ),
-            'coredumps': f"<a target='_blank' href='{core_link}'>link</a>",
             'db_admin': (
-                f"<a target='_blank' href='{test_info['service_url']}/monitoring/tenant?"
-                f"schema=/{test_info['database']}/{YdbCluster.tables_path}&tenantPage=query"
-                f"&diagnosticsTab=nodes&name=/{test_info['database']}'>link</a>"
+                f"<a target='_blank' href='{service_url}/monitoring/tenant?"
+                f"schema=/{db}/{YdbCluster.tables_path}&tenantPage=query"
+                f"&diagnosticsTab=nodes&name=/{db}'>{service_url}</a>"
             ),
+            'time': f"{datetime.fromtimestamp(start_time).strftime('%a %d %b %y %H:%M:%S')} - {datetime.fromtimestamp(end_time).strftime('%H:%M:%S')}",
         }
     )
-    if ResultsProcessor.send_results:
-        params = urlencode({
-            'tab': 'o8',
-            'suite_b2rp': suite,
-            'test_ehyw': test,
-            'db_fmdl': ResultsProcessor.get_cluster_id(),
-            'cluster_dufr': refference_set
-        })
-        test_info['results_plot'] = f"<a target='_blank' href='https://datalens.yandex-team.ru/iqnd4b1miaz27-testy-ydb?{params}'>link</a>"
+    del test_info['nodes_wilcard']
     table_strings = '\n'.join([f'<tr><td>{_pretty_str(k)}</td><td>{v}</td></tr>' for k, v in test_info.items()])
     allure.dynamic.description_html(
         f'''<table border='1' cellpadding='4px'><tbody>

ydb/tests/olap/lib/results_processor.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
+import allure
 import json
 import ydb
 import os
+import logging
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.utils import external_param_is_true, get_external_param
 from time import time_ns
@@ -15,9 +17,14 @@ def __init__(self, ep: str, db: str, table: str, key: str, iam_file: str) -> Non
             self._table = table
 
         def send_data(self, data):
-            self._driver.table_client.bulk_upsert(
-                os.path.join(self._db, self._table), [data], ResultsProcessor._columns_types
-            )
+            try:
+                ydb.retry_operation_sync(
+                    lambda: self._driver.table_client.bulk_upsert(
+                        os.path.join(self._db, self._table), [data], ResultsProcessor._columns_types
+                    )
+                )
+            except BaseException as e:
+                logging.error(f'Exception while send results: {e}')
 
     _endpoints : list[ResultsProcessor.Endpoint] = None
     _run_id : int = None
@@ -73,6 +80,7 @@ def get_cluster_id():
         return os.path.join(YdbCluster.ydb_endpoint, YdbCluster.ydb_database, run_id)
 
     @classmethod
+    @allure.step
     def upload_results(
         cls,
         kind: str,

ydb/tests/olap/lib/ya.make

@@ -11,6 +11,7 @@ PY3_LIBRARY()
     PEERDIR(
         contrib/python/allure-pytest
         contrib/python/allure-python-commons
+        contrib/python/pytz
         contrib/python/requests
         library/python/testing/yatest_common
         ydb/public/api/client/yc_public/iam