[load] baseline trends | sas download

src/load/HISTORY.rst

@@ -10,8 +10,9 @@ Release History
 * Add CLI parameter `--report` to 'az load test-run download-files' to download the dashboard reports.
 * Enable debug level logging using `--debug-mode` argument in 'az load test-run create' command .
 * Return the SAS URL to copy artifacts to storage accounts using command 'az load test-run get-artifacts-url'.
-* Add config for high-scale load tests and appropriate messaging in the 'az load test-run download-files' command for such tests.
+* Add config for high-scale load tests and extend 'az load test-run download-files' to support download of logs and results from artifacts container for such tests.
 * Add command 'az load test convert-to-jmx' to convert URL type tests to JMX tests.
+* Add commands 'az load test set-baseline' to set the baseline for a test and 'az load test compare-to-baseline' to compare recent test runs to the baseline test run.
 
 
 1.3.1

src/load/azext_load/data_plane/load_test/commands.py

@@ -6,6 +6,7 @@
 from azext_load.data_plane.utils import validators
 from azext_load.data_plane.utils.constants import LoadCommandsConstants
 from azure.cli.core.commands import CliCommandType
+from .transformers import trends_output_transformer
 
 admin_custom_sdk = CliCommandType(
     operations_tmpl="azext_load.data_plane.load_test.custom#{}"
@@ -31,6 +32,8 @@ def load_test_commands(self, _):
             "convert_to_jmx",
             confirmation=LoadCommandsConstants.CONVERT_TO_JMX_CONFIRM_PROMPT
         )
+        g.custom_command("set-baseline", "set_baseline")
+        g.custom_command("compare-to-baseline", "compare_to_baseline", table_transformer=trends_output_transformer)
 
     with self.command_group(
         "load test app-component",

src/load/azext_load/data_plane/load_test/custom.py

@@ -9,17 +9,20 @@
 
 from azext_load.data_plane.utils.utils import (
     convert_yaml_to_test,
+    create_autostop_criteria_from_args,
     create_or_update_test_with_config,
     create_or_update_test_without_config,
     download_file,
+    generate_trends_row,
     get_admin_data_plane_client,
+    get_testrun_data_plane_client,
     load_yaml,
     upload_file_to_test,
     upload_files_helper,
-    create_autostop_criteria_from_args,
 )
 from azext_load.data_plane.utils.models import (
     AllowedTestTypes,
+    AllowedTrendsResponseTimeAggregations,
 )
 from azure.cli.core.azclierror import InvalidArgumentValueError, FileOperationError
 from azure.core.exceptions import ResourceNotFoundError
@@ -302,6 +305,90 @@ def convert_to_jmx(
     return response.as_dict()
 
 
+def set_baseline(
+    cmd,
+    load_test_resource,
+    test_id,
+    test_run_id,
+    resource_group_name=None,
+):
+    logger.info("Setting baseline for test with test ID: %s", test_id)
+    test_client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test_run_client = get_testrun_data_plane_client(cmd, load_test_resource, resource_group_name)
+    existing_test = test_client.get_test(test_id)
+    existing_test_run = test_run_client.get_test_run(test_run_id)
+    if existing_test_run.get("testId") != test_id:
+        raise InvalidArgumentValueError(
+            f"Test run with ID: {test_run_id} is not associated with test ID: {test_id}"
+        )
+    if existing_test_run.get("status") not in ["CANCELLED", "DONE"]:
+        raise InvalidArgumentValueError(
+            f"Test run with ID: {test_run_id} does not have a valid "
+            f"test run status {existing_test_run.get('status')}. "
+            "Valid test run status are: CANCELLED, DONE"
+        )
+    if existing_test_run.get("testRunStatistics", {}).get("Total") is None:
+        raise InvalidArgumentValueError(
+            f"Sampler statistics are not yet available for test run ID {test_run_id}. "
+            "Please try again later."
+        )
+    body = create_or_update_test_without_config(
+        test_id=test_id,
+        body=existing_test,
+        baseline_test_run_id=test_run_id,
+    )
+    response = test_client.create_or_update_test(test_id=test_id, body=body)
+    logger.debug("Set test run %s as baseline for test: %s", test_run_id, test_id)
+    return response.as_dict()
+
+
+def compare_to_baseline(
+    cmd,
+    load_test_resource,
+    test_id,
+    resource_group_name=None,
+    response_time_aggregate=AllowedTrendsResponseTimeAggregations.MEAN.value,
+):
+    logger.info("Showing test trends for test with test ID: %s", test_id)
+    test_client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test_run_client = get_testrun_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test = test_client.get_test(test_id)
+    if test.get("baselineTestRunId") is None:
+        raise InvalidArgumentValueError(
+            f"Test with ID: {test_id} does not have a baseline test run associated with it."
+        )
+    baseline_test_run_id = test.get("baselineTestRunId")
+    baseline_test_run = test_run_client.get_test_run(baseline_test_run_id)
+    all_test_runs = test_run_client.list_test_runs(
+        orderby="executedDateTime desc",
+        test_id=test_id,
+        status="CANCELLED,DONE",
+        maxpagesize=20,
+    )
+    all_test_runs = [run.as_dict() for run in all_test_runs]
+    logger.debug("Total number of test runs: %s", len(all_test_runs))
+    count = 0
+    recent_test_runs = []
+    for run in all_test_runs:
+        if (
+            run.get("testRunId") != baseline_test_run_id
+            and count < 10  # Show only 10 most recent test runs
+        ):
+            recent_test_runs.append(run)
+            count += 1
+
+    logger.debug("Number of recent test runs: %s", len(recent_test_runs))
+    rows = [
+        generate_trends_row(baseline_test_run, response_time_aggregate=response_time_aggregate)
+    ]
+    for run in recent_test_runs:
+        rows.append(
+            generate_trends_row(run, response_time_aggregate=response_time_aggregate)
+        )
+    logger.debug("Retrieved test trends: %s", rows)
+    return rows
+
+
 def add_test_app_component(
     cmd,
     load_test_resource,

src/load/azext_load/data_plane/load_test/help.py

@@ -118,6 +118,31 @@
         az load test convert-to-jmx --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id
 """
 
+helps[
+    "load test set-baseline"
+] = """
+type: command
+short-summary: Set a test run as the baseline for comparison with other runs in the test.
+examples:
+    - name: Set baseline test run.
+      text: |
+        az load test set-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --test-run-id sample-associated-test-run-id
+"""
+
+helps[
+    "load test compare-to-baseline"
+] = """
+type: command
+short-summary: Compare the sampler statistics from recent test runs with those of the baseline test run.
+examples:
+    - name: Compare recent test runs to baseline.
+      text: |
+        az load test compare-to-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id -o table
+    - name: Compare recent test runs to baseline with specific aggregation.
+      text: |
+        az load test compare-to-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --aggregation P95 -o table
+"""
+
 helps[
     "load test download-files"
 ] = """

src/load/azext_load/data_plane/load_test/params.py

@@ -56,6 +56,12 @@ def load_arguments(self, _):
         c.argument("autostop_error_rate_time_window", argtypes.autostop_error_rate_time_window)
         c.argument("regionwise_engines", argtypes.regionwise_engines)
 
+    with self.argument_context("load test set-baseline") as c:
+        c.argument("test_run_id", argtypes.test_run_id)
+
+    with self.argument_context("load test compare-to-baseline") as c:
+        c.argument("response_time_aggregate", argtypes.response_time_aggregate)
+
     with self.argument_context("load test download-files") as c:
         c.argument("path", argtypes.dir_path)
         c.argument("force", argtypes.force)

src/load/azext_load/data_plane/load_test/transformers.py

@@ -0,0 +1,14 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from collections import OrderedDict
+from azext_load.data_plane.utils.constants import LoadTestTrendsKeys
+
+
+def trends_output_transformer(result):
+    table = []
+    for row in result:
+        table.append(OrderedDict([(k, row.get(k)) for k in LoadTestTrendsKeys.ORDERED_HEADERS]))
+    return table

src/load/azext_load/data_plane/load_test_run/custom.py

@@ -5,12 +5,15 @@
 
 from azext_load.data_plane.utils.utils import (
     create_or_update_test_run_body,
+    download_from_storage_container,
     get_file_info_and_download,
     get_testrun_data_plane_client,
 )
 from azext_load.data_plane.utils.constants import HighScaleThreshold
 from azure.cli.core.azclierror import InvalidArgumentValueError
 from azure.core.exceptions import ResourceNotFoundError
+from urllib.parse import urlparse
+
 from knack.log import get_logger
 
 logger = get_logger(__name__)
@@ -226,6 +229,36 @@ def _is_high_scale_test_run(test_run_data):
     return False
 
 
+def _download_from_artifacts_container(artifacts_container, path, logs=False, results=False):
+    logger.info(
+        "Downloading %s from artifacts container for high scale test run",
+        {"logs" if logs else "results" if results else "files"}
+    )
+    if artifacts_container is not None and artifacts_container.get("url") is not None:
+        artifacts_container_url = artifacts_container.get("url")
+        artifacts_container_url = _update_artifacts_container_path(artifacts_container_url, logs, results)
+        download_from_storage_container(artifacts_container_url, path)
+        logger.info(
+            "%s from artifacts container downloaded to %s",
+            {"Logs" if logs else "Results" if results else "Files"},
+            path
+        )
+    else:
+        logger.warning("No artifacts container found")
+
+
+def _update_artifacts_container_path(artifacts_container_url, logs, results):
+    artifacts_container_path = urlparse(artifacts_container_url).path
+    artifacts_container_path_updated = (
+        artifacts_container_path
+        + f"{'' if artifacts_container_path.endswith('/') else '/'}"
+        + f"{'logs' if logs else 'results' if results else ''}"
+    )
+    return artifacts_container_url.replace(
+        artifacts_container_path, artifacts_container_path_updated,
+    )
+
+
 def download_test_run_files(
     cmd,
     load_test_resource,
@@ -250,28 +283,26 @@ def download_test_run_files(
         _download_input_file(test_run_input_artifacts, test_run_id, path)
 
     is_high_scale_test_run = _is_high_scale_test_run(test_run_data)
-    high_scale_test_run_message = ""
+    artifacts_container = (
+        test_run_output_artifacts.get("artifactsContainerInfo")
+        if test_run_output_artifacts
+        else None
+    )
     if test_run_log:
         if is_high_scale_test_run:
-            high_scale_test_run_message += f"Logs file for high-scale test {test_run_id} "\
-                "is not available for download. "
+            _download_from_artifacts_container(artifacts_container, path, logs=True)
         else:
             _download_logs_file(test_run_output_artifacts, test_run_id, path)
 
     if test_run_results:
         if is_high_scale_test_run:
-            high_scale_test_run_message += f"Results file for high-scale test {test_run_id} "\
-                "is not available for download. "
+            _download_from_artifacts_container(artifacts_container, path, results=True)
         else:
             _download_results_file(test_run_output_artifacts, test_run_id, path)
 
     if test_run_report:
         _download_reports_file(test_run_output_artifacts, test_run_id, path)
 
-    if high_scale_test_run_message:
-        high_scale_test_run_message += "Use the 'get-artifacts-url' command to fetch the SAS URL and access the file."
-        return high_scale_test_run_message
-
 
 # app components
 def add_test_run_app_component(

src/load/azext_load/data_plane/utils/argtypes.py

@@ -391,3 +391,10 @@
     nargs="+",
     help="Specify the engine count for each region in the format: region1=engineCount1 region2=engineCount2 .... Use region names in the format accepted by Azure Resource Manager (ARM). Ensure the regions are supported by Azure Load Testing. Multi-region load tests can only target public endpoints.",
 )
+
+response_time_aggregate = CLIArgumentType(
+    options_list=["--aggregation"],
+    type=str,
+    choices=utils.get_enum_values(models.AllowedTrendsResponseTimeAggregations),
+    help="Specify the aggregation method for response time.",
+)

src/load/azext_load/data_plane/utils/constants.py

@@ -4,6 +4,7 @@
 # --------------------------------------------------------------------------------------------
 
 from dataclasses import dataclass
+from .models import AllowedTrendsResponseTimeAggregations
 
 
 @dataclass
@@ -39,3 +40,34 @@ class HighScaleThreshold:
 class LoadCommandsConstants:
     CONVERT_TO_JMX_CONFIRM_PROMPT = "Once the test is converted, the process cannot be reversed.\n" \
         "Do you want to continue?"
+
+
+@dataclass
+class LoadTestTrendsKeys:
+    NAME = "Name"
+    DESCRIPTION = "Description"
+    DURATION = "Duration (in minutes)"
+    VUSERS = "Virtual users"
+    TOTAL_REQUESTS = "Total requests"
+    RESPONSE_TIME = "Response time"
+    ERROR_PCT = "Error percentage"
+    THROUGHPUT = "Throughput"
+    STATUS = "Status"
+
+    ORDERED_HEADERS = [NAME, DESCRIPTION, DURATION, VUSERS, TOTAL_REQUESTS,
+                       RESPONSE_TIME, ERROR_PCT, THROUGHPUT, STATUS]
+
+    RESPONSE_TIME_METRICS = {
+        AllowedTrendsResponseTimeAggregations.MEAN.value: "meanResTime",
+        AllowedTrendsResponseTimeAggregations.MEDIAN.value: "medianResTime",
+        AllowedTrendsResponseTimeAggregations.MAX.value: "maxResTime",
+        AllowedTrendsResponseTimeAggregations.MIN.value: "minResTime",
+        AllowedTrendsResponseTimeAggregations.P75.value: "pct75ResTime",
+        AllowedTrendsResponseTimeAggregations.P90.value: "pct1ResTime",
+        AllowedTrendsResponseTimeAggregations.P95.value: "pct2ResTime",
+        AllowedTrendsResponseTimeAggregations.P96.value: "pct96ResTime",
+        AllowedTrendsResponseTimeAggregations.P98.value: "pct98ResTime",
+        AllowedTrendsResponseTimeAggregations.P99.value: "pct3ResTime",
+        AllowedTrendsResponseTimeAggregations.P999.value: "pct999ResTime",
+        AllowedTrendsResponseTimeAggregations.P9999.value: "pct9999ResTime",
+    }

src/load/azext_load/data_plane/utils/models.py

@@ -41,3 +41,18 @@ class AllowedTestTypes(str, Enum):
 class AllowedTestPlanFileExtensions(str, Enum):
     JMX = ".jmx"
     URL = ".json"
+
+
+class AllowedTrendsResponseTimeAggregations(str, Enum):
+    MEAN = "MEAN"
+    MEDIAN = "MEDIAN"
+    MAX = "MAX"
+    MIN = "MIN"
+    P75 = "P75"
+    P90 = "P90"
+    P95 = "P95"
+    P96 = "P96"
+    P98 = "P98"
+    P99 = "P99"
+    P999 = "P999"
+    P9999 = "P9999"