diff --git a/.openpublishing.publish.config.json b/.openpublishing.publish.config.json index b6778e379f..24b2024930 100644 --- a/.openpublishing.publish.config.json +++ b/.openpublishing.publish.config.json @@ -61,6 +61,12 @@ "url": "https://github.com/Microsoft/templates.docs.msft.pdf", "branch": "main", "branch_mapping": {} + }, + { + "path_to_root": "reusable-content", + "url": "https://github.com/MicrosoftDocs/reusable-content", + "branch": "main", + "branch_mapping": {} } ], "branch_target_mapping": { diff --git a/data-explorer/.openpublishing.redirection.json b/data-explorer/.openpublishing.redirection.json index 8fa2683914..6bbe301fbd 100644 --- a/data-explorer/.openpublishing.redirection.json +++ b/data-explorer/.openpublishing.redirection.json @@ -444,6 +444,16 @@ "source_path": "query-exported-azure-monitor-data.md", "redirect_url": "/azure/data-explorer/query-monitor-data", "redirect_document_id": false + }, + { + "source_path": "using-metrics.md", + "redirect_url": "/azure/data-explorer/monitor-data-explorer", + "redirect_document_id": true + }, + { + "source_path": "using-diagnostic-logs.md", + "redirect_url": "/azure/data-explorer/monitor-data-explorer", + "redirect_document_id": false } ] } diff --git a/data-explorer/monitor-data-explorer-reference.md b/data-explorer/monitor-data-explorer-reference.md new file mode 100644 index 0000000000..d25b67bcbc --- /dev/null +++ b/data-explorer/monitor-data-explorer-reference.md @@ -0,0 +1,83 @@ +--- +title: Monitoring data reference for Azure Data Explorer +description: This article contains important reference material you need when you monitor Azure Data Explorer by using Azure Monitor. +ms.date: 12/09/2024 +ms.custom: horz-monitor +ms.topic: reference +author: shsagir +ms.author: shsagir +ms.service: azure-data-explorer +--- + +# Azure Data Explorer monitoring data reference + +[!INCLUDE [horz-monitor-ref-intro](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-intro.md)] + +See [Monitor Azure Data Explorer](monitor-data-explorer.md) for details on the data you can collect for Azure Data Explorer and how to use it. + +[!INCLUDE [horz-monitor-ref-metrics-intro](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-metrics-intro.md)] + +### Supported metrics for Microsoft.Kusto/clusters + +The following table lists the metrics available for the Microsoft.Kusto/clusters resource type. + +[!INCLUDE [horz-monitor-ref-metrics-tableheader](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-metrics-tableheader.md)] + +[!INCLUDE [Microsoft.Kusto/clusters](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/reference/metrics/microsoft-kusto-clusters-metrics-include.md)] + +[!INCLUDE [horz-monitor-ref-metrics-dimensions-intro](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-metrics-dimensions-intro.md)] + +[!INCLUDE [horz-monitor-ref-metrics-dimensions](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-metrics-dimensions.md)] + +- CommandType +- ComponentName +- ComponentType +- ContinuousExportName +- Database +- FailureKind +- IngestionKind +- IngestionResultDetails +- Kind +- MaterializedViewName +- QueryStatus +- Result +- RoleInstance +- SealReason +- State +- Table + +[!INCLUDE [horz-monitor-ref-resource-logs](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-resource-logs.md)] + +### Supported resource logs for Microsoft.Kusto/clusters + +[!INCLUDE [Microsoft.Kusto/clusters](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/reference/logs/microsoft-kusto-clusters-logs-include.md)] + +[!INCLUDE [horz-monitor-ref-logs-tables](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-logs-tables.md)] + +### Azure Data Explorer Microsoft.Kusto/Clusters + +- [AzureActivity](/azure/azure-monitor/reference/tables/azureactivity#columns) +- [AzureMetrics](/azure/azure-monitor/reference/tables/azuremetrics#columns) +- [FailedIngestion](/azure/azure-monitor/reference/tables/failedingestion#columns) + + For information about error codes, see [Ingestion error codes](error-codes.md). + +- [SucceededIngestion](/azure/azure-monitor/reference/tables/succeededingestion#columns) +- [ADXIngestionBatching](/azure/azure-monitor/reference/tables/adxingestionbatching#columns) + + For information about batching types, see [Batching policy](/kusto/management/batching-policy?view=azure-data-explorer&preserve-view=true#sealing-a-batch). + +- [ADXCommand](/azure/azure-monitor/reference/tables/adxcommand#columns) +- [ADXQuery](/azure/azure-monitor/reference/tables/adxquery#columns) +- [ADXTableUsageStatistics](/azure/azure-monitor/reference/tables/adxtableusagestatistics#columns) +- [ADXTableDetails](/azure/azure-monitor/reference/tables/adxtabledetails#columns) +- [ADXJournal](/azure/azure-monitor/reference/tables/adxjournal#columns) + +[!INCLUDE [horz-monitor-ref-activity-log](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/horz-monitor-ref-activity-log.md)] + +- [Analytics resource provider operations](/azure/role-based-access-control/resource-provider-operations#analytics) + +## Related content + +- See [Monitor Azure Data Explorer](monitor-data-explorer.md) for a description of monitoring Azure Data Explorer. +- See [Monitor Azure resources with Azure Monitor](/azure/azure-monitor/essentials/monitor-azure-resource) for details on monitoring Azure resources. diff --git a/data-explorer/monitor-data-explorer.md b/data-explorer/monitor-data-explorer.md new file mode 100644 index 0000000000..81a0033bd4 --- /dev/null +++ b/data-explorer/monitor-data-explorer.md @@ -0,0 +1,146 @@ +--- +title: Monitor Azure Data Explorer +description: Learn how to monitor Azure Data Explorer using Azure Monitor, including data collection, analysis, and alerting. +ms.date: 12/09/2024 +ms.custom: horz-monitor +ms.topic: conceptual +author: shsagir +ms.author: shsagir +ms.service: azure-data-explorer +--- + +# Monitor Azure Data Explorer + +[!INCLUDE [azmon-horz-intro](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-intro.md)] + +## Collect data with Azure Monitor + +This table describes how you can collect data to monitor your service, and what you can do with the data once collected: + +|Data to collect|Description|How to collect and route the data|Where to view the data|Supported data| +|---------|---------|---------|---------|---------| +|Metric data|Metrics are numerical values that describe an aspect of a system at a particular point in time. Metrics can be aggregated using algorithms, compared to other metrics, and analyzed for trends over time.|- Collected automatically at regular intervals.
- You can route some platform metrics to a Log Analytics workspace to query with other data. Check the **DS export** setting for each metric to see if you can use a diagnostic setting to route the metric data.|[Metrics explorer](/azure/azure-monitor/essentials/metrics-getting-started)| [Azure Data Explorer metrics supported by Azure Monitor](monitor-data-explorer-reference.md#metrics)| +|Resource log data|Logs are recorded system events with a timestamp. Logs can contain different types of data, and be structured or free-form text. You can route resource log data to Log Analytics workspaces for querying and analysis.|[Create a diagnostic setting](/azure/azure-monitor/essentials/create-diagnostic-settings) to collect and route resource log data.| [Log Analytics](/azure/azure-monitor/learn/quick-create-workspace)|[Azure Data Explorer resource log data supported by Azure Monitor](monitor-data-explorer-reference.md#resource-logs) | +|Activity log data|The Azure Monitor activity log provides insight into subscription-level events. The activity log includes information like when a resource is modified or a virtual machine is started.|- Collected automatically.
- [Create a diagnostic setting](/azure/azure-monitor/essentials/create-diagnostic-settings) to a Log Analytics workspace at no charge.|[Activity log](/azure/azure-monitor/essentials/activity-log)| | + +[!INCLUDE [azmon-horz-supported-data](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-supported-data.md)] + +## Built in monitoring for Azure Data Explorer + +Azure Data Explorer offers metrics and logs to monitor the service. + +### Monitor Azure Data Explorer performance, health, and usage with metrics + +Azure Data Explorer metrics provide key indicators as to the health and performance of the Azure Data Explorer cluster resources. Use the metrics to monitor Azure Data Explorer cluster usage, health, and performance in your specific scenario as standalone metrics. You can also use metrics as the basis for operational [Azure Dashboards](/azure/azure-portal/azure-portal-dashboards) and [Azure Alerts](/azure/azure-monitor/alerts/alerts-types#metric-alerts). + +To use metrics to monitor your Azure Data Explorer resources in the Azure portal: + +1. Sign in to the [Azure portal](https://portal.azure.com/). +1. In the left-hand pane of your Azure Data Explorer cluster, search for *metrics*. +1. Select **Metrics** to open the metrics pane and begin analysis on your cluster. + + +In the metrics pane, select specific metrics to track, choose how to aggregate your data, and create metric charts to view on your dashboard. + +The **Resource** and **Metric Namespace** pickers are preselected for your Azure Data Explorer cluster. The numbers in the following image correspond to the numbered list. They guide you through different options in setting up and viewing your metrics. + + :::image type="content" source="media/using-metrics/metrics-pane.png" alt-text="Screenshot shows different options for viewing metrics."::: + +1. To create a metric chart, select **Metric** name and relevant **Aggregation** per metric. For more information about different metrics, see [supported Azure Data Explorer metrics](monitor-data-explorer-reference.md#metrics). +1. Select **Add metric** to see multiple metrics plotted in the same chart. +1. Select **+ New chart** to see multiple charts in one view. +1. Use the time picker to change the time range (default: past 24 hours). +1. Use [**Add filter** and **Apply splitting**](/azure/azure-monitor/platform/metrics-getting-started#apply-dimension-filters-and-splitting) for metrics that have dimensions. +1. Select **Pin to dashboard** to add your chart configuration to the dashboards so that you can view it again. +1. Set **New alert rule** to visualize your metrics using the set criteria. The new alerting rule includes your target resource, metric, splitting, and filter dimensions from your chart. Modify these settings in the [alert rule creation pane](/azure/azure-monitor/platform/metrics-charts#create-alert-rules). + +### Monitor Azure Data Explorer ingestion, commands, queries, and tables using diagnostic logs + +Azure Data Explorer is a fast, fully managed data analytics service for real-time analysis on large volumes of data streaming from applications, websites, IoT devices, and more. [Azure Monitor diagnostic logs](/azure/azure-monitor/platform/diagnostic-logs-overview) provide data about the operation of Azure resources. Azure Data Explorer uses diagnostic logs for insights on ingestion, commands, query, and tables. You can export operation logs to Azure Storage, event hub, or Log Analytics to monitor ingestion, commands, and query status. Logs from Azure Storage and Azure Event Hubs can be routed to a table in your Azure Data Explorer cluster for further analysis. + +> [!IMPORTANT] +> Diagnostic log data may contain sensitive data. Restrict permissions of the logs destination according to your monitoring needs. + +[!INCLUDE [azure-monitor-vs-log-analytics](includes/azure-monitor-vs-log-analytics.md)] + +Diagnostic logs can be used to configure the collection of the following log data: + +### [Ingestion](#tab/ingestion) + +> [!NOTE] +> +> - Ingestion logs are supported for queued ingestion to the **Data ingestion URI** using [Kusto client libraries](/kusto/api/client-libraries?view=azure-data-explorer&preserve-view=true) and [data connectors](integrate-data-overview.md). +> - Ingestion logs aren't supported for streaming ingestion, direct ingestion to the **Cluster URI**, ingestion from query, or `.set-or-append` commands. + +> [!NOTE] +> +> Failed ingestion logs are only reported for the final state of an ingest operation, unlike the [Ingestion result](using-metrics.md#ingestion-metrics) metric, which is emitted for transient failures that are retried internally. + +- **Successful ingestion operations**: These logs have information about successfully completed ingestion operations. +- **Failed ingestion operations**: These logs have detailed information about failed ingestion operations including error details. +- **Ingestion batching operations**: These logs have detailed statistics of batches ready for ingestion (duration, batch size, blobs count, and [batching types](/kusto/management/batching-policy?view=azure-data-explorer&preserve-view=true#sealing-a-batch)). + +### [Commands and Queries](#tab/commands-and-queries) + +- **Commands**: These logs have information about admin commands that have reached a final state. +- **Queries**: These logs have detailed information about queries that have reached a final state. + + > [!NOTE] + > The command and query log data contains the query text. + +### [Tables](#tab/tables) + +- **TableUsageStatistics**: These logs have detailed information about the tables whose extents were scanned during query execution. This log doesn't record statistics for queries that are part of commands, such as the [.set-or-append](/kusto/management/data-ingestion/ingest-from-query?view=azure-data-explorer&preserve-view=true) command. + + > [!NOTE] + > The `TableUsageStatistics` log data doesn't contain the command or query text. + +- **TableDetails**: These logs have detailed information about the cluster's tables. + +### [Journal](#tab/journal) + +- **Journal**: These logs have detailed information about metadata operations. + +--- + +You can choose to send the log data to a Log Analytics workspace, a storage account, or stream it to an event hub. + +Diagnostic logs are disabled by default. Use the following steps to enable diagnostic logs for your cluster: + +1. In the [Azure portal](https://portal.azure.com), select the cluster resource that you want to monitor. +1. Under **Monitoring**, select **Diagnostic settings**. + + :::image type="content" source="media/using-diagnostic-logs/add-diagnostic-logs.png" alt-text="Screenshot shows the Diagnostic settings page where you can add a setting."::: + +1. Select **Add diagnostic setting**. +1. In the **Diagnostic settings** window: + + :::image type="content" source="media/using-diagnostic-logs/configure-diagnostics-settings.png" alt-text="Screenshot of the Diagnostic settings screen, on which you configure which monitoring data to collect for your Azure Data Explorer cluster."::: + + 1. Enter a **Diagnostic setting name**. + 1. Select one or more destination targets: a Log Analytics workspace, a storage account, or an event hub. + 1. Select logs to be collected: **Succeeded ingestion**, **Failed ingestion**, **Ingestion batching**, **Command**, **Query**, **Table usage statistics**, **Table details**, or **Journal**. + 1. Select [metrics](using-metrics.md#supported-azure-data-explorer-metrics) to be collected (optional). + 1. Select **Save** to save the new diagnostic logs settings and metrics. + +Once the settings are ready, logs start to appear in the configured destination targets: a storage account, an event hub, or Log Analytics workspace. + +> [!NOTE] +> If you send logs to a Log Analytics workspace, the `SucceededIngestion`, `FailedIngestion`, `IngestionBatching`, `Command`, `Query`, `TableUsageStatistics`, `TableDetails`, and `Journal` logs are stored in Log Analytics tables named: `SucceededIngestion`, `FailedIngestion`, `ADXIngestionBatching`, `ADXCommand`, `ADXQuery`, `ADXTableUsageStatistics`, `ADXTableDetails`, and `ADXJournal` respectively. + +[!INCLUDE [azmon-horz-tools](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-tools.md)] + +[!INCLUDE [azmon-horz-export-data](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-export-data.md)] + +[!INCLUDE [azmon-horz-kusto](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-kusto.md)] + +[!INCLUDE [azmon-horz-alerts-part-one](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-alerts-part-one.md)] + +[!INCLUDE [azmon-horz-alerts-part-two](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-alerts-part-two.md)] + +[!INCLUDE [azmon-horz-advisor](~/../reusable-content/ce-skilling/azure/includes/azure-monitor/horizontals/azmon-horz-advisor.md)] + +## Related content + +- [Azure Data Explorer monitoring data reference](monitor-data-explorer-reference.md) +- [Monitoring Azure resources with Azure Monitor](/azure/azure-monitor/essentials/monitor-azure-resource) diff --git a/data-explorer/toc.yml b/data-explorer/toc.yml index 37576c43ca..8f25bc4061 100644 --- a/data-explorer/toc.yml +++ b/data-explorer/toc.yml @@ -504,11 +504,9 @@ items: href: dealing-with-duplicates.md - name: Monitor items: - - name: Monitor Azure Data Explorer with metrics + - name: Monitor Azure Data Explorer displayName: health, performance - href: using-metrics.md - - name: Use diagnostic logs to monitor ingestion, commands, queries, and tables - href: using-diagnostic-logs.md + href: monitor-data-explorer.md - name: Use resource health to monitor cluster health href: monitor-with-resource-health.md - name: Use Azure Data Explorer Clusters Insights @@ -658,6 +656,8 @@ items: href: policy-reference.md - name: Bicep and ARM template resource types href: /azure/templates/microsoft.kusto/allversions + - name: Monitoring data reference + href: monitor-data-explorer-reference.md - name: PowerShell Az.Kusto items: - name: Use Kusto cmdlets in Azure PowerShell diff --git a/data-explorer/using-diagnostic-logs.md b/data-explorer/using-diagnostic-logs.md deleted file mode 100644 index 6eb03e0ffb..0000000000 --- a/data-explorer/using-diagnostic-logs.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -title: Monitor Azure Data Explorer ingestion, commands, and queries using diagnostic logs -description: Learn how to set up diagnostic logs for Azure Data Explorer to monitor ingestion commands, and query operations. -ms.reviewer: guregini -ms.topic: how-to -ms.date: 08/09/2023 ---- - -# Monitor Azure Data Explorer ingestion, commands, queries, and tables using diagnostic logs - -Azure Data Explorer is a fast, fully managed data analytics service for real-time analysis on large volumes of data streaming from applications, websites, IoT devices, and more. [Azure Monitor diagnostic logs](/azure/azure-monitor/platform/diagnostic-logs-overview) provide data about the operation of Azure resources. Azure Data Explorer uses diagnostic logs for insights on ingestion, commands, query, and tables. You can export operation logs to Azure Storage, event hub, or Log Analytics to monitor ingestion, commands, and query status. Logs from Azure Storage and Azure Event Hubs can be routed to a table in your Azure Data Explorer cluster for further analysis. - -> [!IMPORTANT] -> Diagnostic log data may contain sensitive data. Restrict permissions of the logs destination according to your monitoring needs. - -[!INCLUDE [azure-monitor-vs-log-analytics](includes/azure-monitor-vs-log-analytics.md)] - -## Prerequisites - -* An Azure subscription. Create a [free Azure account](https://azure.microsoft.com/free/). -* Sign in to the [Azure portal](https://portal.azure.com/). -* Create [a cluster and database](create-cluster-and-database.md). - -## Set up diagnostic logs for an Azure Data Explorer cluster - -Diagnostic logs can be used to configure the collection of the following log data: - -### [Ingestion](#tab/ingestion) - -> [!NOTE] -> * Ingestion logs are supported for queued ingestion to the **Data ingestion URI** using [Kusto client libraries](/kusto/api/client-libraries?view=azure-data-explorer&preserve-view=true) and [data connectors](integrate-data-overview.md). -> * Ingestion logs aren't supported for streaming ingestion, direct ingestion to the **Cluster URI**, ingestion from query, or `.set-or-append` commands. - -> [!NOTE] -> Failed ingestion logs are only reported for the final state of an ingest operation, unlike the [Ingestion result](using-metrics.md#ingestion-metrics) metric, which is emitted for transient failures that are retried internally. - -* **Successful ingestion operations**: These logs have information about successfully completed ingestion operations. -* **Failed ingestion operations**: These logs have detailed information about failed ingestion operations including error details. -* **Ingestion batching operations**: These logs have detailed statistics of batches ready for ingestion (duration, batch size, blobs count, and [batching types](/kusto/management/batching-policy?view=azure-data-explorer&preserve-view=true#sealing-a-batch)). - -### [Commands and Queries](#tab/commands-and-queries) - -* **Commands**: These logs have information about admin commands that have reached a final state. -* **Queries**: These logs have detailed information about queries that have reached a final state. - - > [!NOTE] - > The command and query log data contains the query text. - -### [Tables](#tab/tables) - -* **TableUsageStatistics**: These logs have detailed information about the tables whose extents were scanned during query execution. Note that this log doesn't record statistics for queries that are part of commands, such as the [.set-or-append](/kusto/management/data-ingestion/ingest-from-query?view=azure-data-explorer&preserve-view=true) command. - - > [!NOTE] - > The `TableUsageStatistics` log data doesn't contain the command or query text. - -* **TableDetails**: These logs have detailed information about the cluster's tables. - -### [Journal](#tab/journal) - -* **Journal**: These logs have detailed information about metadata operations. - ---- - -You can choose to send the log data to a Log Analytics workspace, a storage account, or stream it to an event hub. - -### Enable diagnostic logs - -Diagnostic logs are disabled by default. Use the following steps to enable diagnostic logs for your cluster: - -1. In the [Azure portal](https://portal.azure.com), select the cluster resource that you want to monitor. -1. Under **Monitoring**, select **Diagnostic settings**. - - ![Add diagnostics logs.](media/using-diagnostic-logs/add-diagnostic-logs.png) - -1. Select **Add diagnostic setting**. -1. In the **Diagnostic settings** window: - - :::image type="content" source="media/using-diagnostic-logs/configure-diagnostics-settings.png" alt-text="Screenshot of the Diagnostic settings screen, on which you configure which monitoring data to collect for your Azure Data Explorer cluster."::: - - 1. Enter a **Diagnostic setting name**. - 1. Select one or more destination targets: a Log Analytics workspace, a storage account, or an event hub. - 1. Select logs to be collected: **Succeeded ingestion**, **Failed ingestion**, **Ingestion batching**, **Command**, **Query**, **Table usage statistics**, **Table details**, or **Journal**. - 1. Select [metrics](using-metrics.md#supported-azure-data-explorer-metrics) to be collected (optional). - 1. Select **Save** to save the new diagnostic logs settings and metrics. - -Once the settings are ready, logs will start to appear in the configured destination targets (a storage account, an event hub, or Log Analytics workspace). - -> [!NOTE] -> If you send logs to a Log Analytics workspace, the `SucceededIngestion`, `FailedIngestion`, `IngestionBatching`, `Command`, `Query`, `TableUsageStatistics`, `TableDetails`, and `Journal` logs will be stored in Log Analytics tables named: `SucceededIngestion`, `FailedIngestion`, `ADXIngestionBatching`, `ADXCommand`, `ADXQuery`, `ADXTableUsageStatistics`, `ADXTableDetails`, and `ADXJournal` respectively. - -## Diagnostic logs schema - -All [Azure Monitor diagnostic logs share a common top-level schema](/azure/azure-monitor/platform/diagnostic-logs-schema). Azure Data Explorer events have their own unique properties that are described in the following schema references: - -* [SucceededIngestion](/azure/azure-monitor/reference/tables/succeededingestion) -* [FailedIngestion](/azure/azure-monitor/reference/tables/failedingestion) - * For information about error codes, see [Ingestion error codes](error-codes.md) -* [ADXIngestionBatching](/azure/azure-monitor/reference/tables/adxingestionbatching) - * For information about batching types, see [Batching policy](/kusto/management/batching-policy?view=azure-data-explorer&preserve-view=true#sealing-a-batch) -* [ADXCommand](/azure/azure-monitor/reference/tables/adxcommand) -* [ADXQuery](/azure/azure-monitor/reference/tables/adxquery) -* [ADXTableUsageStatistics](/azure/azure-monitor/reference/tables/adxtableusagestatistics) -* [ADXTableDetails](/azure/azure-monitor/reference/tables/adxtabledetails) -* [ADXJournal](/azure/azure-monitor/reference/tables/adxjournal) - -## Related content - -* [Use metrics to monitor cluster health](using-metrics.md) -* [Tutorial: Ingest and query monitoring data in Azure Data Explorer](ingest-data-no-code.md) for ingestion diagnostic logs diff --git a/data-explorer/using-metrics.md b/data-explorer/using-metrics.md deleted file mode 100644 index 44f37b3cce..0000000000 --- a/data-explorer/using-metrics.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: Monitor Azure Data Explorer performance, health & usage with metrics -description: Learn how to use Azure Data Explorer metrics to monitor the cluster's performance, health, and usage. -ms.reviewer: gabil -ms.topic: how-to -ms.date: 11/20/2024 ---- - -# Monitor Azure Data Explorer performance, health, and usage with metrics - -Azure Data Explorer metrics provide key indicators as to the health and performance of the Azure Data Explorer cluster resources. Use the metrics that are detailed in this article to monitor Azure Data Explorer cluster usage, health, and performance in your specific scenario as standalone metrics. You can also use metrics as the basis for operational [Azure Dashboards](/azure/azure-portal/azure-portal-dashboards) and [Azure Alerts](/azure/azure-monitor/alerts/alerts-types#metric-alerts). - -For more information about Azure Metrics Explorer, see [Metrics Explorer](/azure/azure-monitor/platform/metrics-getting-started). - -## Prerequisites - -* An Azure subscription. Create a [free Azure account](https://azure.microsoft.com/free/). -* An Azure Data Explorer cluster and database. [Create a cluster and database](create-cluster-and-database.md). - -## Use metrics to monitor your Azure Data Explorer resources - -1. Sign in to the [Azure portal](https://portal.azure.com/). -1. In the left-hand pane of your Azure Data Explorer cluster, search for *metrics*. -1. Select **Metrics** to open the metrics pane and begin analysis on your cluster. - :::image type="content" source="media/using-metrics/select-metrics.gif" alt-text="Search and select metrics in the Azure portal."::: - -## Work in the metrics pane - -In the metrics pane, select specific metrics to track, choose how to aggregate your data, and create metric charts to view on your dashboard. - -The **Resource** and **Metric Namespace** pickers are pre-selected for your Azure Data Explorer cluster. The numbers in the following image correspond to the numbered list below. They guide you through different options in setting up and viewing your metrics. - -![Metrics pane.](media/using-metrics/metrics-pane.png) - -1. To create a metric chart, select **Metric** name and relevant **Aggregation** per metric. For more information about different metrics, see [supported Azure Data Explorer metrics](#supported-azure-data-explorer-metrics). -1. Select **Add metric** to see multiple metrics plotted in the same chart. -1. Select **+ New chart** to see multiple charts in one view. -1. Use the time picker to change the time range (default: past 24 hours). -1. Use [**Add filter** and **Apply splitting**](/azure/azure-monitor/platform/metrics-getting-started#apply-dimension-filters-and-splitting) for metrics that have dimensions. -1. Select **Pin to dashboard** to add your chart configuration to the dashboards so that you can view it again. -1. Set **New alert rule** to visualize your metrics using the set criteria. The new alerting rule will include your target resource, metric, splitting, and filter dimensions from your chart. Modify these settings in the [alert rule creation pane](/azure/azure-monitor/platform/metrics-charts#create-alert-rules). - -## Supported Azure Data Explorer metrics - -The Azure Data Explorer metrics give insight into both overall performance and use of your resources, as well as information about specific actions, such as ingestion or query. The metrics in this article have been grouped by usage type. - -The types of metrics are: -* [Cluster metrics](#cluster-metrics) -* [Export metrics](#export-metrics) -* [Ingestion metrics](#ingestion-metrics) -* [Streaming ingest metrics](#streaming-ingest-metrics) -* [Query metrics](#query-metrics) -* [Materialized view metrics](#materialized-view-metrics) - -For an alphabetical list of Azure Monitor's metrics for Azure Data Explorers, see [supported Azure Data Explorer cluster metrics](/azure/azure-monitor/platform/metrics-supported#microsoftkustoclusters). - -## Cluster metrics - -The cluster metrics track the general health of the cluster. For example, resource and ingestion use and responsiveness. - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -| Cache utilization (deprecated) | Percent | Avg, Max, Min | The percentage of allocated cache resources currently in use by the cluster. Cache is the size of SSD allocated for user activity according to the defined cache policy.

An average cache utilization of 80% or less is a sustainable state for a cluster. If the average cache utilization is above 80%, the cluster should be
[scaled up](manage-cluster-vertical-scaling.md) to a storage-optimized pricing tier or
[scaled out](manage-cluster-horizontal-scaling.md) to more instances. Alternatively, adapt the cache policy to fewer days in cache. If cache utilization is over 100%, the size of data to be cached is larger than the total size of cache on the cluster.
This metric is deprecated and presented for backward compatibility only. Use the ‘Cache utilization factor’ metric instead. | None -| Cache utilization factor | Percent | Avg, Max, Min | The percentage of utilized disk space dedicated for hot cache in the cluster.
100% means that the disk space assigned to hot data is optimally utilized. No action is needed, and the cluster is completely fine.
Less than 100% means that the disk space assigned for hot data is not fully utilized.
More than 100% means that the cluster's disk space is not large enough to accommodate the hot data, as defined by your caching policies. To ensure that sufficient space is available for all the hot data, the amount of hot data needs to be reduced or the cluster needs to be scaled out. We recommend enabling auto scale. | None | -| CPU | Percent | Avg, Max, Min | The percentage of allocated compute resources currently in use by machines in the cluster.

An average CPU of 80% or less is sustainable for a cluster. The maximum value of CPU is 100%, which means there are no additional compute resources to process data.
When a cluster isn't performing well, check the maximum value of the CPU to determine if there are specific CPUs that are blocked. | None | -| Ingestion utilization | Percent | Avg, Max, Min | The percentage of actual resources used to ingest data from the total resources allocated, in the capacity policy, to perform ingestion. The default capacity policy is no more than 512 concurrent ingestion operations or 75% of the cluster resources invested in ingestion.

Average ingestion utilization of 80% or less is a sustainable state for a cluster. Maximum value of ingestion utilization is 100%, which means all cluster ingestion ability is used and an ingestion queue may result. | None | -| InstanceCount | Count | Avg | The total instance count. | -| Keep alive | Count | Avg | Tracks the responsiveness of the cluster.

A fully responsive cluster returns value 1 and a blocked or disconnected cluster returns 0. | -| Total number of throttled commands | Count | Avg, Max, Min, Sum | The number of throttled (rejected) commands in the cluster, since the maximum allowed number of concurrent (parallel) commands was reached. | None | -| Total number of extents | Count | Avg, Max, Min, Sum | The total number of data extents in the cluster.

Changes in this metric can imply massive data structure changes and high load on the cluster, since merging data extents is a CPU-heavy activity. | None | -| Follower latency | Milliseconds | Avg, Max, Min | The follower databases synchronize changes in the leader databases. Because of the synchronization, there’s a data lag of a few seconds to a few minutes in data availability.

This metric measures the length of the time lag. The time lag depends on several factors like: the overall size and rate of the ingested data to the leader, the number of databases followed, the rate of internal operations performed on the leader (merge/rebuild operations).

This is a cluster level metrics: the followers catch metadata of all databases that are followed. This metric represents the latency of the process. | None | - -## Export metrics - -Export metrics track the general health and performance of export operations like lateness, results, number of records, and utilization. - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -Continuous export number of exported records | Count | Sum | The number of exported records in all continuous export jobs. | ContinuousExportName | -Continuous export max lateness | Count | Max | The lateness (in minutes) reported by the continuous export jobs in the cluster. | None | -Continuous export pending count | Count | Max | The number of pending continuous export jobs. These jobs are ready to run but waiting in a queue, possibly due to insufficient capacity). -Continuous export result | Count | Count | The Failure/Success result of each continuous export run. | ContinuousExportName | -Export utilization | Percent | Max | The export capacity used, out of the total export capacity in the cluster (between 0 and 100). | None | - -## Ingestion metrics - -Ingestion metrics track the general health and performance of ingestion operations like latency, results, and volume. -To refine your analysis: -* [Apply filters to charts](/azure/azure-monitor/platform/metrics-charts#apply-filters-to-charts) to plot partial data by dimensions. For example, explore ingestion to a specific `Database`. -* [Apply splitting to a chart](/azure/azure-monitor/platform/metrics-charts#apply-splitting-to-a-chart) to visualize data by different components. This process is useful for analyzing metrics that are reported by each step of the ingestion pipeline, for example `Blobs received`. - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -| Batch blob count | Count | Avg, Max, Min | The number of data sources in a completed batch for ingestion. | Database | -| Batch duration | Seconds | Avg, Max, Min | The duration of the batching phase in the ingestion flow. | Database | -| Batch size | Bytes | Avg, Max, Min | The uncompressed expected data size in an aggregated batch for ingestion. | Database | -| Batches processed | Count | Sum, Max, Min | The number of batches completed for ingestion.
`Batching Type`: The trigger for sealing a batch.
For a complete list of batching types, see [Batching types](/kusto/management/batching-policy?view=azure-data-explorer&preserve-view=true#sealing-a-batch). | Database, Batching Type | -| Blobs received | Count | Sum, Max, Min | The number of blobs received from input stream by a component.

Use **apply splitting** to analyze each component. | Database, Component Type, Component Name | -| Blobs processed | Count | Sum, Max, Min | The number of blobs processed by a component.

Use **apply splitting** to analyze each component. | Database, Component Type, Component Name | -| Blobs dropped | Count | Sum, Max, Min | The number of blobs permanently dropped by a component. For each such blob, an `Ingestion result` metric with a failure reason is sent.

Use **apply splitting** to analyze each component. | Database, Component Type, Component Name | -| Discovery latency | Seconds | Avg | Time from data enqueue until discovery by data connections. This time isn't included in the **Stage latency** or in the **Ingestion latency** metrics.

Discovery latency might increase in the following situations:
| Component Type, Component Name | -| Events received | Count | Sum, Max, Min | The number of events received by data connections from input stream. | Component Type, Component Name | -| Events processed | Count | Sum, Max, Min | The number of events processed by data connections. | Component Type, Component Name | -| Events dropped | Count | Sum, Max, Min | The number of events permanently dropped by data connections. For each such event, an `Ingestion result` metric with a failure reason is sent. | Component Type, Component Name | -| Ingestion latency | Seconds | Avg, Max, Min | The latency of data ingested, from the time the data was received in the cluster until it's ready for query. The ingestion latency period depends on the ingestion scenario.
`Ingestion Kind`: Streaming Ingestion or Queued Ingestion | Ingestion Kind | -| Ingestion result | Count | Sum | The total number of sources that either failed or succeeded to be ingested.
`Status`: **Success** for successful ingestion or the failure category for failures. For a complete list of possible failure categories see [Ingestion error codes in Azure Data Explorer](error-codes.md).
`Failure Status Type`: Whether the failure is permanent or transient. For successful ingestion, this dimension is `None`.

**Note:**
| Status, Failure Status Type | -| Ingestion volume (in bytes) | Count | Max, Sum | The total size of data ingested to the cluster (in bytes) before compression. | Database | -| Queue length | Count | Avg | The number of pending messages in a component's input queue. The batching manager component has one message per blob. The ingestion manager component has one message per batch. A batch is a single ingest command with one or more blobs. | Component Type | -| Queue oldest message | Seconds | Avg | The time in seconds from when the oldest message in a component's input queue has been inserted. | Component Type | -| Received data size bytes | Bytes | Avg, Sum | The size of data received by data connections from input stream. | Component Type, Component Name | -| Stage latency | Seconds | Avg | The time from when a message is accepted by Azure Data Explorer, until its content is received by an ingestion component for processing.

Use **apply filters** and select **Component Type > StorageEngine** to show the total ingestion latency.| Database, Component Type | - -## Streaming ingest metrics - -Streaming ingest metrics track streaming ingestion data and request rate, duration, and results. - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -Streaming Ingest Data Rate | Count | RateRequestsPerSecond | The total volume of data ingested to the cluster. | None | -Streaming Ingest Duration | Milliseconds | Avg, Max, Min | The total duration of all streaming ingestion requests. | None | -Streaming Ingest Request Rate | Count | Count, Avg, Max, Min, Sum | The total number of streaming ingestion requests. | None | -Streaming Ingest Result | Count | Avg | The total number of streaming ingestion requests by result type. | Result | - -## Query metrics - -Query performance metrics track query duration and total number of concurrent or throttled queries. - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -| Query duration | Milliseconds | Avg, Min, Max, Sum | The total time until query results are received (doesn't include network latency). | QueryStatus | -| QueryResult | Count | Count | The total number of queries. | QueryStatus | -| Total number of concurrent queries | Count | Avg, Max, Min, Sum | The number of queries run in parallel in the cluster. This metric is a good way to estimate the load on the cluster. | None | -| Total number of throttled queries | Count | Avg, Max, Min, Sum | The number of throttled (rejected) queries in the cluster. The maximum number of concurrent (parallel) queries allowed is defined in the request rate limit policy. | None | -| Weak consistency latency | Milliseconds | Avg, Max, Min | The age of the metadata snapshot used by nodes running the weak consistency service. | Database, Node | - -## Materialized view metrics - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -|MaterializedViewHealth | 1, 0 | Avg | The value is 1 if the view is considered healthy, otherwise 0. | Database, MaterializedViewName | -|MaterializedViewAgeSeconds | Seconds | Avg | The `age` of the view is defined by the current time minus the last ingestion time processed by the view. Metric value is time in seconds (the lower the value is, the view is "healthier"). | Database, MaterializedViewName | -|MaterializedViewResult | 1 | Avg | The metric includes a `Result` dimension indicating the result of the last materialization cycle (see the [MaterializedViewResult metric](/kusto/management/materialized-views/materialized-views-monitoring?view=azure-data-explorer&preserve-view=true#materializedviewresult-metric) for details about possible values). Metric value always equals 1. | Database, MaterializedViewName, Result | -|MaterializedViewRecordsInDelta | Records count | Avg | The number of records currently in the non-processed part of the source table. For more information, see [how materialized views work](/kusto/management/materialized-views/materialized-view-overview?view=azure-data-explorer&preserve-view=true#how-materialized-views-work)| Database, MaterializedViewName | -|MaterializedViewExtentsRebuild | Extents count | Avg | The number of extents that required updates in the materialization cycle. | Database, MaterializedViewName| -|MaterializedViewDataLoss | 1 | Max | The metric is fired when unprocessed source data is approaching retention. Indicates that the materialized view is unhealthy. | Database, MaterializedViewName, Kind | - -## Partitioning metrics - -Partitioning metrics monitor the partitioning process for tables with a [partitioning policy](/azure/data-explorer/kusto/management/partitioning-policy?view=azure-data-explorer&preserve-view=true). - -|**Metric** | **Unit** | **Aggregation** | **Metric description** | **Dimensions** | -|---|---|---|---|---| -|PartitioningPercentage | Percent| Avg, Min, Max | The percentage of records partitioned relative to the total number of records. | Database, Table | -|PartitioningPercentageHot | Percent | Avg, Min, Max | The percentage of records partitioned related to the total number of records (in 'hot' cache only). | Database, Table | -|ProcessedPartitionedRecords | Percent | Avg, Min, Max, Sum | The number of records partitioned in the measured time window. | Database, Table | - -## Related content - -* [Tutorial: Ingest and query monitoring data in Azure Data Explorer](ingest-data-no-code.md) -* [Monitor Azure Data Explorer ingestion operations using diagnostic logs](using-diagnostic-logs.md) -* [Quickstart: Query data in Azure Data Explorer](web-query-data.md)