From 311d50c1835f4d1bfbbce4a8018ed82c96f6a871 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 10 Oct 2022 22:06:57 +0200 Subject: [PATCH 01/10] added static routes per vpc tracking --- .../network-dashboard/cloud-function/main.py | 3 + .../cloud-function/metrics.yaml | 12 +++ .../cloud-function/metrics/limits.py | 2 +- .../cloud-function/metrics/metrics.py | 7 +- .../cloud-function/metrics/routes.py | 91 +++++++++++++++++++ .../cloud-function/metrics/vpc_firewalls.py | 2 +- 6 files changed, 112 insertions(+), 5 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 67f3316fc6..d1aafe3b0d 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -176,6 +176,9 @@ def main(event, context): ilb_fwrules.get_forwarding_rules_data( config, metrics_dict, l7_forwarding_rules_dict, limits_dict['internal_forwarding_rules_l7_limit'], "L7") + + routes.get_static_routes_vpc(config, metrics_dict, project_quotas_dict) + peerings.get_vpc_peering_data(config, metrics_dict, limits_dict['number_of_vpc_peerings_limit']) dynamic_routes_dict = routes.get_dynamic_routes( diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index b87dc1c371..54acc43fdb 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -99,6 +99,18 @@ metrics_per_network: utilization: name: dynamic_routes_per_network_utilization description: Number of Dynamic routes per network - utilization. + static_routes_per_network: + usage: + name: static_routes_per_project_vpc_usage + description: Number of Static routes per project and network - usage. + limit: + name: static_routes_per_project_limit + description: Number of Static routes per project - limit. + values: + default_value: 250 + utilization: + name: static_routes_per_project_utilization + description: Number of Static routes per project - utilization. metrics_per_peering_group: l4_forwarding_rules_per_peering_group: usage: diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index 5c5bfc73a3..5ad1f34c59 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -42,7 +42,7 @@ def get_quotas_dict(quotas_list): def get_quota_project_limit(config, regions=["global"]): ''' - Retrieves limit for a specific project quota + Retrieves quotas for all monitored project in selected regions, default 'global' Parameters: project_link (string): Project link. Returns: diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 4792c27f80..3bb696978d 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -91,7 +91,8 @@ def create_metric(metric_name, description, monitoring_project): def append_data_to_series_buffer(config, metric_name, metric_value, metric_labels, timestamp=None): ''' - Writes data to Cloud Monitoring custom metrics. + Appends data to Cloud Monitoring custom metrics, using a buffer. buffer is flushed every BUFFER_LEN elements, + any unflushed series is discarded upon function closure Parameters: config (dict): The dict containing config like clients and limits metric_name (string): Name of the metric @@ -136,7 +137,7 @@ def append_data_to_series_buffer(config, metric_name, metric_value, def flush_series_buffer(config): ''' - writes buffered metrics to Google Cloud Monitoring, empties buffer upon failure + writes buffered metrics to Google Cloud Monitoring, empties buffer upon both failure/success config (dict): The dict containing config like clients and limits ''' try: @@ -237,7 +238,7 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): metric_dict["utilization"]["name"], limit_dict) print( - f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" + f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" ) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index 8bca349629..e26db04c57 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -14,9 +14,11 @@ # limitations under the License. # +import re import time from collections import defaultdict +from google.protobuf import field_mask_pb2 from . import metrics, networks, limits, peerings, routers @@ -177,3 +179,92 @@ def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict): print( f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}" ) + + +def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): + ''' + LOREM + + Parameters: + config (dict): The dict containing config like clients and limits + metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics + usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value + project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + Returns: + None + ''' + routes_per_vpc_dict = defaultdict() + usage_dict = defaultdict() + + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + response = config["clients"]["asset_client"].search_all_resources( + request={ + "scope": f"organizations/{config['organization']}", + "asset_types": ["compute.googleapis.com/Route"], + "read_mask": read_mask, + }) + + timestamp = time.time() + for resource in response: + for versioned in resource.versioned_resources: + static_route = dict() + for field_name, field_value in versioned.resource.items(): + static_route[field_name] = field_value + static_route["project_id"] = re.search("\/([^\/]*)(\/[^\/]*){3}$", + static_route["network"]).group(1) + static_route["network_name"] = re.search("\/([^\/]*)$", + static_route["network"]).group(1) + + #exclude default vpc and peering routes + if "nextHopPeering" not in static_route and "nextHopNetwork" not in static_route: + if static_route["project_id"] not in routes_per_vpc_dict: + routes_per_vpc_dict[static_route["project_id"]] = dict() + if static_route["network_name"] not in routes_per_vpc_dict[ + static_route["project_id"]]: + routes_per_vpc_dict[static_route["project_id"]][ + static_route["network_name"]] = dict() + + if static_route["destRange"] not in routes_per_vpc_dict[ + static_route["project_id"]][static_route["network_name"]]: + routes_per_vpc_dict[static_route["project_id"]][ + static_route["network_name"]][static_route["destRange"]] = {} + if "usage" not in routes_per_vpc_dict[static_route["project_id"]][ + static_route["network_name"]]: + routes_per_vpc_dict[static_route["project_id"]][ + static_route["network_name"]]["usage"] = 0 + routes_per_vpc_dict[static_route["project_id"]][ + static_route["network_name"]]["usage"] = routes_per_vpc_dict[ + static_route["project_id"]][ + static_route["network_name"]]["usage"] + 1 + + for project_id in routes_per_vpc_dict: + current_quota_limit = project_quotas_dict[project_id]['global']["routes"][ + "limit"] + if current_quota_limit is None: + print( + f"Could not determine static routes metric for projects/{project_id} due to missing quotas" + ) + continue + for network_name in routes_per_vpc_dict[project_id]: + metric_labels = {"project": project_id, "network_name": network_name} + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["static_routes_per_network"]["usage"]["name"], + routes_per_vpc_dict[project_id][network_name]["usage"], metric_labels, + timestamp=timestamp) + + # limit and utilization are calculted by project + metric_labels = {"project": project_id} + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + ["limit"]["name"], current_quota_limit, metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + ["utilization"]["name"], + routes_per_vpc_dict[project_id][network_name]["usage"] / + current_quota_limit, metric_labels, timestamp=timestamp) + + return \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py index e7abade5f0..b61d134d4d 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -83,7 +83,7 @@ def get_firewalls_data(config, metrics_dict, project_quotas_dict, current_quota_limit = project_quotas_dict[project_id]['global']["firewalls"] if current_quota_limit is None: print( - f"Could not determine VPC firewal rules to metric for projects/{project_id} due to missing quotas" + f"Could not determine VPC firewal rules metric for projects/{project_id} due to missing quotas" ) continue From 61917690e5bd0662fb7a5e46cd08816c165b0416 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Wed, 12 Oct 2022 14:51:03 +0200 Subject: [PATCH 02/10] added support for ppg static routes --- .../network-dashboard/README.md | 11 +- .../network-dashboard/cloud-function/main.py | 16 ++- .../cloud-function/metrics.yaml | 12 ++ .../metrics/firewall_policies.py | 11 +- .../cloud-function/metrics/routes.py | 135 ++++++++++-------- 5 files changed, 117 insertions(+), 68 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 5d5b59935b..5d02b22303 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -42,21 +42,24 @@ The Cloud Function currently tracks usage, limit and utilization of: - internal forwarding rules for internal L7 load balancers per VPC - internal forwarding rules for internal L4 load balancers per VPC peering group - internal forwarding rules for internal L7 load balancers per VPC peering group -- Dynamic routes per VPC -- Dynamic routes per VPC peering group +- Dynamic routes per VPC (note: assumes global routing is ON) +- Dynamic routes per VPC peering group (note: assumes custom routes importing/exporting is ON) +- Static routes per project (VPC drill down is available for usage) +- Static routes per VPC peering group (note: assumes custom routes sharing is ON for all peered networks) - IP utilization per subnet (% of IP addresses used in a subnet) - VPC firewall rules per project (VPC drill down is available for usage) - Tuples per Firewall Policy It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. -Note that metrics are created in the cloud-function/metrics.yaml file. +Note that metrics are created in the cloud-function/metrics.yaml file. also note that the Cloud Function assumes all VPCs in peering groups are within the same organization. You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. ## Next steps and ideas In a future release, we could support: -- Static routes per VPC / per VPC peering group - Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) +- Dynamic routes calculation for VPCs/PPGs with "global routing" set to OFF +- Static routes calculation for projects/PPGs with "custom routes importing/exporting" set to OFF If you are interested in this and/or would like to contribute, please contact legranda@google.com. \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index d1aafe3b0d..a04dcf81e5 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -158,6 +158,9 @@ def main(event, context): l4_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L4") l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) + static_routes_dict = routes.get_static_routes_dict(config) + dynamic_routes_dict = routes.get_dynamic_routes( + config, metrics_dict, limits_dict['dynamic_routes_per_network_limit']) try: @@ -177,12 +180,11 @@ def main(event, context): config, metrics_dict, l7_forwarding_rules_dict, limits_dict['internal_forwarding_rules_l7_limit'], "L7") - routes.get_static_routes_vpc(config, metrics_dict, project_quotas_dict) + routes.get_static_routes_data(config, metrics_dict, static_routes_dict, + project_quotas_dict) peerings.get_vpc_peering_data(config, metrics_dict, limits_dict['number_of_vpc_peerings_limit']) - dynamic_routes_dict = routes.get_dynamic_routes( - config, metrics_dict, limits_dict['dynamic_routes_per_network_limit']) # Per VPC peering group metrics metrics.get_pgg_data( @@ -205,7 +207,13 @@ def main(event, context): ["subnet_ranges_per_peering_group"], subnet_range_dict, config["limit_names"]["SUBNET_RANGES"], limits_dict['number_of_subnet_IP_ranges_ppg_limit']) - routes.get_dynamic_routes_ppg( + #static + routes.get_routes_ppg( + config, metrics_dict["metrics_per_peering_group"] + ["static_routes_per_peering_group"], static_routes_dict, + limits_dict['static_routes_per_peering_group_limit']) + #dynamic + routes.get_routes_ppg( config, metrics_dict["metrics_per_peering_group"] ["dynamic_routes_per_peering_group"], dynamic_routes_dict, limits_dict['dynamic_routes_per_peering_group_limit']) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 54acc43fdb..721fb9666f 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -172,6 +172,18 @@ metrics_per_peering_group: utilization: name: dynamic_routes_per_peering_group_utilization description: Number of Dynamic routes per peering group - utilization. + static_routes_per_peering_group: + usage: + name: static_routes_per_peering_group_usage + description: Number of Static routes per peering group - usage. + limit: + name: static_routes_per_peering_group_limit + description: Number of Static routes per peering group - limit. + values: + default_value: 250 + utilization: + name: static_routes_per_peering_group_utilization + description: Number of Static routes per peering group - utilization. metrics_per_project: firewalls: usage: diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py index 8c9e33a2fc..90928889fe 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py @@ -28,8 +28,8 @@ def get_firewall_policies_dict(config: dict): ''' - Calls the Asset Inventory API to get all Firewall Policies under the GCP organization - + Calls the Asset Inventory API to get all Firewall Policies under the GCP organization, including children + Ignores monitored projects list: returns all policies regardless of their parent resource Parameters: config (dict): The dict containing config like clients and limits Returns: @@ -57,8 +57,8 @@ def get_firewall_policies_dict(config: dict): def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict): ''' - Gets the data for VPC Firewall lorem ipsum - + Gets the data for VPC Firewall Policies in an organization, including children. All folders are considered, + only projects in the monitored projects list are considered. Parameters: config (dict): The dict containing config like clients and limits metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. @@ -93,6 +93,9 @@ def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict): parent_type = re.search("(^\w+)", firewall_policy["parent"]).group( 1) if "parent" in firewall_policy else "projects" + if parent_type == "projects" and parent not in config["monitored_projects"]: + continue + metric_labels = {'parent': parent, 'parent_type': parent_type} metric_labels["name"] = firewall_policy[ diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index e26db04c57..5f4c30d3af 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -14,7 +14,9 @@ # limitations under the License. # +from ast import AnnAssign import re +from sqlite3 import Timestamp import time from collections import defaultdict @@ -80,8 +82,8 @@ def get_routes_for_network(config, network_link, project_id, routers_dict): def get_dynamic_routes(config, metrics_dict, limits_dict): ''' - Writes all dynamic routes per VPC to custom metrics. - + This function gets the usage, limit and utilization for the dynamic routes per VPC + note: assumes global routing is ON for all VPCs Parameters: config (dict): The dict containing config like clients and limits metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. @@ -130,10 +132,10 @@ def get_dynamic_routes(config, metrics_dict, limits_dict): return dynamic_routes_dict -def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict): +def get_routes_ppg(config, metric_dict, usage_dict, limit_dict): ''' - This function gets the usage, limit and utilization for the dynamic routes per VPC peering group. - + This function gets the usage, limit and utilization for the static or dynamic routes per VPC peering group. + note: assumes global routing is ON for all VPCs for dynamic routes, assumes share custom routes is on for all peered networks Parameters: config (dict): The dict containing config like clients and limits metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics @@ -142,11 +144,11 @@ def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict): Returns: None ''' - for project in config["monitored_projects"]: - network_dict_list = peerings.gather_peering_data(config, project) + for project_id in config["monitored_projects"]: + network_dict_list = peerings.gather_peering_data(config, project_id) for network_dict in network_dict_list: - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}" + network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" limit = limits.get_ppg(network_link, limit_dict) @@ -171,27 +173,23 @@ def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict): peered_network_dict["usage"] = peered_usage peered_network_dict["limit"] = peered_limit - limits.count_effective_limit(config, project, network_dict, + limits.count_effective_limit(config, project_id, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_dict) print( - f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}" + f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" ) -def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): +def get_static_routes_dict(config): ''' - LOREM - - Parameters: - config (dict): The dict containing config like clients and limits - metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics - usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value - project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - Returns: - None + Calls the Asset Inventory API to get all static customr routes under the GCP organization. + Parameters: + config (dict): The dict containing config like clients and limits + Returns: + routes_per_vpc_dict (dictionary of string: int): Keys are the network links and values are the number of custom static routes per network. ''' routes_per_vpc_dict = defaultdict() usage_dict = defaultdict() @@ -203,10 +201,9 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): request={ "scope": f"organizations/{config['organization']}", "asset_types": ["compute.googleapis.com/Route"], - "read_mask": read_mask, + "read_mask": read_mask }) - timestamp = time.time() for resource in response: for versioned in resource.versioned_resources: static_route = dict() @@ -216,30 +213,64 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): static_route["network"]).group(1) static_route["network_name"] = re.search("\/([^\/]*)$", static_route["network"]).group(1) - - #exclude default vpc and peering routes + network_link = f"https://www.googleapis.com/compute/v1/projects/{static_route['project_id']}/global/networks/{static_route['network_name']}" + #exclude default vpc and peering routes, dynamic routes are not in Cloud Asset Inventory if "nextHopPeering" not in static_route and "nextHopNetwork" not in static_route: - if static_route["project_id"] not in routes_per_vpc_dict: - routes_per_vpc_dict[static_route["project_id"]] = dict() - if static_route["network_name"] not in routes_per_vpc_dict[ - static_route["project_id"]]: - routes_per_vpc_dict[static_route["project_id"]][ - static_route["network_name"]] = dict() - - if static_route["destRange"] not in routes_per_vpc_dict[ - static_route["project_id"]][static_route["network_name"]]: - routes_per_vpc_dict[static_route["project_id"]][ - static_route["network_name"]][static_route["destRange"]] = {} - if "usage" not in routes_per_vpc_dict[static_route["project_id"]][ - static_route["network_name"]]: - routes_per_vpc_dict[static_route["project_id"]][ - static_route["network_name"]]["usage"] = 0 - routes_per_vpc_dict[static_route["project_id"]][ - static_route["network_name"]]["usage"] = routes_per_vpc_dict[ - static_route["project_id"]][ - static_route["network_name"]]["usage"] + 1 - - for project_id in routes_per_vpc_dict: + if network_link not in routes_per_vpc_dict: + routes_per_vpc_dict[network_link] = dict() + routes_per_vpc_dict[network_link]["project_id"] = static_route[ + "project_id"] + routes_per_vpc_dict[network_link]["network_name"] = static_route[ + "network_name"] + if static_route["destRange"] not in routes_per_vpc_dict[network_link]: + routes_per_vpc_dict[network_link][static_route["destRange"]] = {} + if "usage" not in routes_per_vpc_dict[network_link]: + routes_per_vpc_dict[network_link]["usage"] = 0 + routes_per_vpc_dict[network_link][ + "usage"] = routes_per_vpc_dict[network_link]["usage"] + 1 + + #output a dict with network links and usage only + return { + network_link_out: routes_per_vpc_dict[network_link_out]["usage"] + for network_link_out in routes_per_vpc_dict + } + + +def get_static_routes_data(config, metrics_dict, static_routes_dict, + project_quotas_dict): + ''' + Determines and writes the number of static routes for each VPC in monitored projects, the per project limit and the per project utilization + note: assumes custom routes sharing is ON for all VPCs + Parameters: + config (dict): The dict containing config like clients and limits + metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics + static_routes_dict (dictionary of dictionary: int): Keys are the network links and values are the number of custom static routes per network. + project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + Returns: + None + ''' + timestamp = time.time() + project_usage = {project: 0 for project in config["monitored_projects"]} + + #usage is drilled down by network + for network_link in static_routes_dict: + + project_id = re.search("\/([^\/]*)(\/[^\/]*){3}$", network_link).group(1) + if (project_id not in config["monitored_projects"]): + continue + network_name = re.search("\/([^\/]*)$", network_link).group(1) + + project_usage[project_id] = project_usage[project_id] + static_routes_dict[ + network_link] + + metric_labels = {"project": project_id, "network_name": network_name} + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + ["usage"]["name"], static_routes_dict[network_link], metric_labels, + timestamp=timestamp) + + #limit and utilization are calculated by projec + for project_id in project_usage: current_quota_limit = project_quotas_dict[project_id]['global']["routes"][ "limit"] if current_quota_limit is None: @@ -247,14 +278,6 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): f"Could not determine static routes metric for projects/{project_id} due to missing quotas" ) continue - for network_name in routes_per_vpc_dict[project_id]: - metric_labels = {"project": project_id, "network_name": network_name} - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["static_routes_per_network"]["usage"]["name"], - routes_per_vpc_dict[project_id][network_name]["usage"], metric_labels, - timestamp=timestamp) - # limit and utilization are calculted by project metric_labels = {"project": project_id} metrics.append_data_to_series_buffer( @@ -264,7 +287,7 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict): metrics.append_data_to_series_buffer( config, metrics_dict["metrics_per_network"]["static_routes_per_network"] ["utilization"]["name"], - routes_per_vpc_dict[project_id][network_name]["usage"] / - current_quota_limit, metric_labels, timestamp=timestamp) + project_usage[project_id] / current_quota_limit, metric_labels, + timestamp=timestamp) - return \ No newline at end of file + return From 1fb93e1b64cbfe8b3ee4a4d083f5d6cf1d9c2419 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Wed, 12 Oct 2022 15:05:58 +0200 Subject: [PATCH 03/10] aligned timestamps for ppg data --- .../network-dashboard/cloud-function/metrics/limits.py | 6 ++++-- .../network-dashboard/cloud-function/metrics/metrics.py | 3 ++- .../network-dashboard/cloud-function/metrics/routes.py | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index 5ad1f34c59..8987b4cb8d 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -158,7 +158,7 @@ def get_quota_current_limit(config, project_link, metric_name): def count_effective_limit(config, project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, - limit_dict): + limit_dict, timestamp=None): ''' Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit @@ -171,11 +171,13 @@ def count_effective_limit(config, project_id, network_dict, usage_metric_name, limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group. utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group. limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit). + timestamp (time): timestamp to be recorded for all points Returns: None ''' - timestamp = time.time() + if timestamp == None: + timestamp = time.time() if network_dict['peerings'] == []: return diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 3bb696978d..901f28637e 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -186,6 +186,7 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): current_quota_limit_view = customize_quota_view(current_quota_limit) + timestamp = time.time() # For each network in this GCP project for network_dict in network_dict_list: if network_dict['network_id'] == 0: @@ -236,7 +237,7 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], - limit_dict) + limit_dict, timestamp) print( f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" ) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index 5f4c30d3af..cfc6f7fc58 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -15,6 +15,7 @@ # from ast import AnnAssign +from os import times import re from sqlite3 import Timestamp import time @@ -144,6 +145,7 @@ def get_routes_ppg(config, metric_dict, usage_dict, limit_dict): Returns: None ''' + timestamp = time.time() for project_id in config["monitored_projects"]: network_dict_list = peerings.gather_peering_data(config, project_id) @@ -177,7 +179,7 @@ def get_routes_ppg(config, metric_dict, usage_dict, limit_dict): metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], - limit_dict) + limit_dict, timestamp) print( f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" ) From de5bad7f194fee46a024a75f0156e530b318b231 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 14 Oct 2022 12:05:17 +0200 Subject: [PATCH 04/10] corrected default value for static routes --- .../network-dashboard/cloud-function/metrics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 721fb9666f..bc6177bc13 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -180,7 +180,7 @@ metrics_per_peering_group: name: static_routes_per_peering_group_limit description: Number of Static routes per peering group - limit. values: - default_value: 250 + default_value: 300 utilization: name: static_routes_per_peering_group_utilization description: Number of Static routes per peering group - utilization. From cebc9360f829584118c6db82d6ed6ef6339e0b57 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 21 Oct 2022 17:43:00 +0200 Subject: [PATCH 05/10] fixes to dashboard and statc routes category metric name --- .gitignore | 1 + .../cloud-function/metrics.yaml | 3 +- .../cloud-function/metrics/routes.py | 6 +- .../dashboards/quotas-utilization.json | 148 +++++++++++++++--- 4 files changed, 129 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index ec7ead32be..e0bfaac4a5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ examples/cloud-operations/adfs/ansible/vars/vars.yaml examples/cloud-operations/adfs/ansible/gssh.sh examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/vars.yaml examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/gssh.sh +blueprints/cloud-operations/network-dashboard/cloud-function.zip diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index bc6177bc13..0c5bb8cc36 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -99,7 +99,8 @@ metrics_per_network: utilization: name: dynamic_routes_per_network_utilization description: Number of Dynamic routes per network - utilization. - static_routes_per_network: + #static routes limit is per project, but usage is per network + static_routes_per_project: usage: name: static_routes_per_project_vpc_usage description: Number of Static routes per project and network - usage. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index cfc6f7fc58..6aa3627ee8 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -267,7 +267,7 @@ def get_static_routes_data(config, metrics_dict, static_routes_dict, metric_labels = {"project": project_id, "network_name": network_name} metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + config, metrics_dict["metrics_per_network"]["static_routes_per_project"] ["usage"]["name"], static_routes_dict[network_link], metric_labels, timestamp=timestamp) @@ -283,11 +283,11 @@ def get_static_routes_data(config, metrics_dict, static_routes_dict, # limit and utilization are calculted by project metric_labels = {"project": project_id} metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + config, metrics_dict["metrics_per_network"]["static_routes_per_project"] ["limit"]["name"], current_quota_limit, metric_labels, timestamp=timestamp) metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_network"] + config, metrics_dict["metrics_per_network"]["static_routes_per_project"] ["utilization"]["name"], project_usage[project_id] / current_quota_limit, metric_labels, timestamp=timestamp) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 0e71bb7f75..58958863d6 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -1,6 +1,6 @@ { "category": "CUSTOM", - "displayName": "quotas_utilization_updated", + "displayName": "quotas_utilization", "mosaicLayout": { "columns": 12, "tiles": [ @@ -22,13 +22,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -64,13 +62,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -106,13 +102,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -148,13 +142,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -190,13 +182,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_INTERPOLATE" } } @@ -232,13 +222,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -274,13 +262,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -316,13 +302,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -358,7 +342,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"" @@ -395,7 +378,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"" @@ -452,7 +434,7 @@ }, "width": 6, "xPos": 0, - "yPos": 24 + "yPos": 32 }, { "height": 4, @@ -492,7 +474,7 @@ }, "width": 6, "xPos": 6, - "yPos": 24 + "yPos": 32 }, { "height": 4, @@ -512,7 +494,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_utilization\" resource.type=\"global\"" @@ -528,7 +509,7 @@ } }, "width": 6, - "xPos": 0, + "xPos": 6, "yPos": 28 }, { @@ -549,7 +530,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" @@ -586,7 +566,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_peering_group_utilization\" resource.type=\"global\"" @@ -604,6 +583,125 @@ "width": 6, "xPos": 6, "yPos": 20 + }, + { + "height": 4, + "widget": { + "title": "static_routes_per_project_vpc_usage", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_vpc_usage\" resource.type=\"global\"" + } + } + } + ], + "thresholds": [], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 0, + "yPos": 24 + }, + { + "height": 4, + "widget": { + "title": "static_routes_per_ppg_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_peering_group_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } + } + } + } + ], + "thresholds": [], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 0, + "yPos": 28 + }, + { + "height": 4, + "widget": { + "title": "static_routes_per_project_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } + } + } + } + ], + "thresholds": [], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 24 } ] } From 4b1138c5bb6c033ecb48fb400e6b7a9570ca63b5 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 21 Oct 2022 18:44:53 +0200 Subject: [PATCH 06/10] pr comments fixed --- .../cloud-function/metrics/routes.py | 17 ++++++----------- .../dashboards/quotas-utilization.json | 10 ++-------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index 6aa3627ee8..07510ca523 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -14,10 +14,7 @@ # limitations under the License. # -from ast import AnnAssign -from os import times import re -from sqlite3 import Timestamp import time from collections import defaultdict @@ -187,7 +184,7 @@ def get_routes_ppg(config, metric_dict, usage_dict, limit_dict): def get_static_routes_dict(config): ''' - Calls the Asset Inventory API to get all static customr routes under the GCP organization. + Calls the Asset Inventory API to get all static custom routes under the GCP organization. Parameters: config (dict): The dict containing config like clients and limits Returns: @@ -211,10 +208,8 @@ def get_static_routes_dict(config): static_route = dict() for field_name, field_value in versioned.resource.items(): static_route[field_name] = field_value - static_route["project_id"] = re.search("\/([^\/]*)(\/[^\/]*){3}$", - static_route["network"]).group(1) - static_route["network_name"] = re.search("\/([^\/]*)$", - static_route["network"]).group(1) + static_route["project_id"] = static_route["network"].split('/')[6] + static_route["network_name"] = static_route["network"].split('/')[-1] network_link = f"https://www.googleapis.com/compute/v1/projects/{static_route['project_id']}/global/networks/{static_route['network_name']}" #exclude default vpc and peering routes, dynamic routes are not in Cloud Asset Inventory if "nextHopPeering" not in static_route and "nextHopNetwork" not in static_route: @@ -257,10 +252,10 @@ def get_static_routes_data(config, metrics_dict, static_routes_dict, #usage is drilled down by network for network_link in static_routes_dict: - project_id = re.search("\/([^\/]*)(\/[^\/]*){3}$", network_link).group(1) + project_id = network_link.split('/')[6] if (project_id not in config["monitored_projects"]): continue - network_name = re.search("\/([^\/]*)$", network_link).group(1) + network_name = network_link.split('/')[-1] project_usage[project_id] = project_usage[project_id] + static_routes_dict[ network_link] @@ -271,7 +266,7 @@ def get_static_routes_data(config, metrics_dict, static_routes_dict, ["usage"]["name"], static_routes_dict[network_link], metric_labels, timestamp=timestamp) - #limit and utilization are calculated by projec + #limit and utilization are calculated by project for project_id in project_usage: current_quota_limit = project_quotas_dict[project_id]['global']["routes"][ "limit"] diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 58958863d6..44b9081b0e 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -609,7 +609,6 @@ } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -632,7 +631,7 @@ "dataSets": [ { "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", + "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { "apiSource": "DEFAULT_CLOUD", @@ -682,16 +681,11 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_utilization\" resource.type=\"global\"" } } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", From 0a878454aaa39f4d336cb2011f8bf22709841c65 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 21 Oct 2022 18:48:53 +0200 Subject: [PATCH 07/10] dashboard fix --- .../dashboards/quotas-utilization.json | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 44b9081b0e..c9eb8bd123 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -595,20 +595,29 @@ "dataSets": [ { "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", + "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [ + "metric.label.\"project\"" + ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_vpc_usage\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_vpc_usage\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } } } } ], + "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -640,11 +649,7 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_peering_group_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } + "filter": "metric.type=\"custom.googleapis.com/static_routes_per_peering_group_utilization\" resource.type=\"global\"" } } } From c359c13d4a04a20cf2dd5ce2af06fb19ca790283 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 24 Oct 2022 13:49:31 +0200 Subject: [PATCH 08/10] removed unneeded import --- .../network-dashboard/cloud-function/metrics/routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index 07510ca523..a161454547 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -14,7 +14,6 @@ # limitations under the License. # -import re import time from collections import defaultdict From 946868062c9a76eaf39fbd40ae451d9d9e14b53f Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Tue, 25 Oct 2022 11:37:14 +0200 Subject: [PATCH 09/10] updated readme --- .../network-dashboard/README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 49a73de0ee..cc7b28ff1f 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -46,25 +46,32 @@ The Cloud Function currently tracks usage, limit and utilization of: - internal forwarding rules for internal L7 load balancers per VPC - internal forwarding rules for internal L4 load balancers per VPC peering group - internal forwarding rules for internal L7 load balancers per VPC peering group -- Dynamic routes per VPC (note: assumes global routing is ON) -- Dynamic routes per VPC peering group (note: assumes custom routes importing/exporting is ON) +- Dynamic routes per VPC +- Dynamic routes per VPC peering group - Static routes per project (VPC drill down is available for usage) -- Static routes per VPC peering group (note: assumes custom routes sharing is ON for all peered networks) +- Static routes per VPC peering group - IP utilization per subnet (% of IP addresses used in a subnet) - VPC firewall rules per project (VPC drill down is available for usage) - Tuples per Firewall Policy It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. -Note that metrics are created in the cloud-function/metrics.yaml file. also note that the Cloud Function assumes all VPCs in peering groups are within the same organization. +Note that metrics are created in the cloud-function/metrics.yaml file. You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. + +## Assumptions and limitations +- The CF assumes that all VPCs in peering groups are within the same organization, except for PSA peerings +- PSA peerings record only subnets data +- The CF assumes global routing is ON, this impacts dynamic routes usage calculation +- The CF assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation +- The CF assumes all networks in peering groups have the same global routing and custom routes sharing configuration -You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. ## Next steps and ideas In a future release, we could support: - Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) - Dynamic routes calculation for VPCs/PPGs with "global routing" set to OFF - Static routes calculation for projects/PPGs with "custom routes importing/exporting" set to OFF +- Calculations for cross Organization peering groups If you are interested in this and/or would like to contribute, please contact legranda@google.com. From 1b19587f0c76d2f463b3f9e295bfd5ad5d900279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Tue, 25 Oct 2022 11:50:40 +0200 Subject: [PATCH 10/10] Update README.md --- blueprints/cloud-operations/network-dashboard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index cc7b28ff1f..4d62ed5fc8 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -60,7 +60,7 @@ Note that metrics are created in the cloud-function/metrics.yaml file. You can a ## Assumptions and limitations - The CF assumes that all VPCs in peering groups are within the same organization, except for PSA peerings -- PSA peerings record only subnets data +- The CF will only fetch subnet utilization data from the PSA peerings (not the VMs, ILB or routes usage) - The CF assumes global routing is ON, this impacts dynamic routes usage calculation - The CF assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation - The CF assumes all networks in peering groups have the same global routing and custom routes sharing configuration